236 lines
8.7 KiB
Python
236 lines
8.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
|
|
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
|
|
#
|
|
# This file is part of cdist.
|
|
#
|
|
# cdist is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# cdist is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
#
|
|
|
|
import logging
|
|
import os
|
|
import glob
|
|
import multiprocessing
|
|
|
|
import cdist
|
|
|
|
'''
|
|
common:
|
|
runs only remotely, needs local and remote to construct paths
|
|
|
|
env:
|
|
__explorer: full qualified path to other global explorers on
|
|
remote side
|
|
-> remote.global_explorer_path
|
|
|
|
a global explorer is:
|
|
- a script
|
|
- executed on the remote side
|
|
- returns its output as a string
|
|
|
|
env:
|
|
|
|
creates: nothing, returns output
|
|
|
|
type explorer is:
|
|
- a script
|
|
- executed on the remote side for each object instance
|
|
- returns its output as a string
|
|
|
|
env:
|
|
__object: full qualified path to the object's remote dir
|
|
__object_id: the objects id
|
|
__object_fq: full qualified object id, iow: $type.name + / + object_id
|
|
__type_explorer: full qualified path to the other type explorers on
|
|
remote side
|
|
|
|
creates: nothing, returns output
|
|
|
|
'''
|
|
|
|
|
|
class Explorer(object):
|
|
"""Executes cdist explorers.
|
|
|
|
"""
|
|
def __init__(self, target_host, local, remote, jobs=None):
|
|
self.target_host = target_host
|
|
|
|
self._open_logger()
|
|
|
|
self.local = local
|
|
self.remote = remote
|
|
self.env = {
|
|
'__target_host': self.target_host,
|
|
'__explorer': self.remote.global_explorer_path,
|
|
}
|
|
self._type_explorers_transferred = []
|
|
self.jobs = jobs
|
|
|
|
def _open_logger(self):
|
|
self.log = logging.getLogger(self.target_host)
|
|
|
|
# global
|
|
|
|
def list_global_explorer_names(self):
|
|
"""Return a list of global explorer names."""
|
|
return glob.glob1(self.local.global_explorer_path, '*')
|
|
|
|
def run_global_explorers(self, out_path):
|
|
"""Run global explorers and save output to files in the given
|
|
out_path directory.
|
|
|
|
"""
|
|
self.log.info("Running global explorers")
|
|
self.transfer_global_explorers()
|
|
if self.jobs is None:
|
|
self._run_global_explorers_seq(out_path)
|
|
else:
|
|
self._run_global_explorers_parallel(out_path)
|
|
|
|
def _run_global_explorer(self, explorer, out_path):
|
|
output = self.run_global_explorer(explorer)
|
|
path = os.path.join(out_path, explorer)
|
|
with open(path, 'w') as fd:
|
|
fd.write(output)
|
|
|
|
def _run_global_explorers_seq(self, out_path):
|
|
self.log.info("Running global explorers sequentially")
|
|
for explorer in self.list_global_explorer_names():
|
|
self._run_global_explorer(explorer, out_path)
|
|
|
|
def _run_global_explorers_parallel(self, out_path):
|
|
self.log.info("Running global explorers in {} parallel jobs".format(
|
|
self.jobs))
|
|
self.log.debug("Multiprocessing start method is {}".format(
|
|
multiprocessing.get_start_method()))
|
|
self.log.info(("Starting multiprocessing Pool for global "
|
|
"explorers run"))
|
|
with multiprocessing.Pool(self.jobs) as pool:
|
|
self.log.info("Starting async for global explorer run")
|
|
results = [
|
|
pool.apply_async(self._run_global_explorer, (e, out_path,))
|
|
for e in self.list_global_explorer_names()
|
|
]
|
|
|
|
self.log.info("Waiting async results for global explorer runs")
|
|
for r in results:
|
|
r.get() # self._run_global_explorer returns None
|
|
self.log.info(("Multiprocessing run for global explorers "
|
|
"finished"))
|
|
|
|
# logger is not pickable, so remove it when we pickle
|
|
def __getstate__(self):
|
|
state = self.__dict__.copy()
|
|
if 'log' in state:
|
|
del state['log']
|
|
return state
|
|
|
|
# recreate logger when we unpickle
|
|
def __setstate__(self, state):
|
|
self.__dict__.update(state)
|
|
self._open_logger()
|
|
|
|
def transfer_global_explorers(self):
|
|
"""Transfer the global explorers to the remote side."""
|
|
self.remote.mkdir(self.remote.global_explorer_path)
|
|
if self.jobs is None:
|
|
self.remote.transfer(self.local.global_explorer_path,
|
|
self.remote.global_explorer_path)
|
|
else:
|
|
self.remote.transfer_dir_parallel(
|
|
self.local.global_explorer_path,
|
|
self.remote.global_explorer_path,
|
|
self.jobs)
|
|
self.remote.run(["chmod", "0700",
|
|
"%s/*" % (self.remote.global_explorer_path)])
|
|
|
|
def run_global_explorer(self, explorer):
|
|
"""Run the given global explorer and return it's output."""
|
|
script = os.path.join(self.remote.global_explorer_path, explorer)
|
|
return self.remote.run_script(script, env=self.env, return_output=True)
|
|
|
|
# type
|
|
|
|
def list_type_explorer_names(self, cdist_type):
|
|
"""Return a list of explorer names for the given type."""
|
|
source = os.path.join(self.local.type_path, cdist_type.explorer_path)
|
|
try:
|
|
return glob.glob1(source, '*')
|
|
except EnvironmentError:
|
|
return []
|
|
|
|
def run_type_explorers(self, cdist_object):
|
|
"""Run the type explorers for the given object and save their output
|
|
in the object.
|
|
|
|
"""
|
|
self.log.debug("Transfering type explorers for type: %s",
|
|
cdist_object.cdist_type)
|
|
self.transfer_type_explorers(cdist_object.cdist_type)
|
|
self.log.debug("Transfering object parameters for object: %s",
|
|
cdist_object.name)
|
|
self.transfer_object_parameters(cdist_object)
|
|
for explorer in self.list_type_explorer_names(cdist_object.cdist_type):
|
|
output = self.run_type_explorer(explorer, cdist_object)
|
|
self.log.debug("Running type explorer '%s' for object '%s'",
|
|
explorer, cdist_object.name)
|
|
cdist_object.explorers[explorer] = output
|
|
|
|
def run_type_explorer(self, explorer, cdist_object):
|
|
"""Run the given type explorer for the given object and return
|
|
it's output."""
|
|
cdist_type = cdist_object.cdist_type
|
|
env = self.env.copy()
|
|
env.update({
|
|
'__object': os.path.join(self.remote.object_path,
|
|
cdist_object.path),
|
|
'__object_id': cdist_object.object_id,
|
|
'__object_name': cdist_object.name,
|
|
'__object_fq': cdist_object.path,
|
|
'__type_explorer': os.path.join(self.remote.type_path,
|
|
cdist_type.explorer_path)
|
|
})
|
|
script = os.path.join(self.remote.type_path, cdist_type.explorer_path,
|
|
explorer)
|
|
return self.remote.run_script(script, env=env, return_output=True)
|
|
|
|
def transfer_type_explorers(self, cdist_type):
|
|
"""Transfer the type explorers for the given type to the
|
|
remote side."""
|
|
if cdist_type.explorers:
|
|
if cdist_type.name in self._type_explorers_transferred:
|
|
self.log.debug("Skipping retransfer of type explorers for: %s",
|
|
cdist_type)
|
|
else:
|
|
source = os.path.join(self.local.type_path,
|
|
cdist_type.explorer_path)
|
|
destination = os.path.join(self.remote.type_path,
|
|
cdist_type.explorer_path)
|
|
self.remote.mkdir(destination)
|
|
self.remote.transfer(source, destination)
|
|
self.remote.run(["chmod", "0700", "%s/*" % (destination)])
|
|
self._type_explorers_transferred.append(cdist_type.name)
|
|
|
|
def transfer_object_parameters(self, cdist_object):
|
|
"""Transfer the parameters for the given object to the remote side."""
|
|
if cdist_object.parameters:
|
|
source = os.path.join(self.local.object_path,
|
|
cdist_object.parameter_path)
|
|
destination = os.path.join(self.remote.object_path,
|
|
cdist_object.parameter_path)
|
|
self.remote.mkdir(destination)
|
|
self.remote.transfer(source, destination)
|