kjg/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

ugly->bad

Browse source
This commit is contained in:
Darko Poljak 2016-12-04 20:27:42 +01:00
parent 2999f12698
commit 609977b7ff
6 changed files with 54 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cdist/__init__.py
View file

@ -68,13 +68,13 @@ class CdistBetaRequired(cdist.Error):
err_msg = ("\'{}\' command is beta, but beta is "
"not enabled. If you want to use it please enable beta "
"functionalities by using the -b/--enable-beta command "
"line flag.")
"line flag or setting CDIST_BETA env var.")
fmt_args = [self.command, ]
else:
err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta "
"is not enabled. If you want to use it please enable "
"beta functionalities by using the -b/--enable-beta "
"command line flag.")
"command line flag or setting CDIST_BETA env var.")
fmt_args = [self.arg, self.command, ]
return err_msg.format(*fmt_args)

11
cdist/core/explorer.py
View file

@ -149,14 +149,9 @@ class Explorer(object):
def transfer_global_explorers(self):
"""Transfer the global explorers to the remote side."""
self.remote.mkdir(self.remote.global_explorer_path)
if self.jobs is None:
self.remote.transfer(self.local.global_explorer_path,
self.remote.global_explorer_path)
else:
self.remote.transfer_dir_parallel(
self.local.global_explorer_path,
self.remote.global_explorer_path,
self.jobs)
self.remote.transfer(self.local.global_explorer_path,
self.remote.global_explorer_path,
self.jobs)
self.remote.run(["chmod", "0700",
"%s/*" % (self.remote.global_explorer_path)])

80
cdist/exec/remote.py
View file

@ -118,57 +118,59 @@ class Remote(object):
self.log.debug("Remote mkdir: %s", path)
self.run(["mkdir", "-p", path])
def transfer(self, source, destination):
def transfer(self, source, destination, jobs=None):
"""Transfer a file or directory to the remote side."""
self.log.debug("Remote transfer: %s -> %s", source, destination)
self.rmdir(destination)
if os.path.isdir(source):
self.mkdir(destination)
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
self._run_command(command)
if jobs:
self._transfer_dir_parallel(source, destination, jobs)
else:
self._transfer_dir_sequential(source, destination)
elif jobs:
raise cdist.Error("Source {} is not a directory".format(source))
else:
command = self._copy.split()
command.extend([source, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
self._run_command(command)
def transfer_dir_parallel(self, source, destination, jobs):
"""Transfer a directory to the remote side in parallel mode."""
self.log.debug("Remote transfer: %s -> %s", source, destination)
self.rmdir(destination)
if os.path.isdir(source):
self.mkdir(destination)
self.log.info("Remote transfer in {} parallel jobs".format(
jobs))
self.log.debug("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.debug(("Starting multiprocessing Pool for parallel "
"remote transfer"))
with multiprocessing.Pool(jobs) as pool:
self.log.debug("Starting async for parallel transfer")
commands = []
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
commands.append(command)
results = [
pool.apply_async(self._run_command, (cmd,))
for cmd in commands
]
def _transfer_dir_sequential(self, source, destination):
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
self._run_command(command)
self.log.debug("Waiting async results for parallel transfer")
for r in results:
r.get() # self._run_command returns None
self.log.debug(("Multiprocessing for parallel transfer "
"finished"))
else:
raise cdist.Error("Source {} is not a directory".format(source))
def _transfer_dir_parallel(self, source, destination, jobs):
"""Transfer a directory to the remote side in parallel mode."""
self.log.info("Remote transfer in {} parallel jobs".format(
jobs))
self.log.debug("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.debug(("Starting multiprocessing Pool for parallel "
"remote transfer"))
with multiprocessing.Pool(jobs) as pool:
self.log.debug("Starting async for parallel transfer")
commands = []
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
commands.append(command)
results = [
pool.apply_async(self._run_command, (cmd,))
for cmd in commands
]
self.log.debug("Waiting async results for parallel transfer")
for r in results:
r.get() # self._run_command returns None
self.log.debug(("Multiprocessing for parallel transfer "
"finished"))
def run_script(self, script, env=None, return_output=False):
"""Run the given script with the given environment on the remote side.

4
cdist/test/exec/remote.py
View file

@ -136,8 +136,8 @@ class RemoteTestCase(test.CdistTestCase):
source_file_name = os.path.split(source_file)[-1]
filenames.append(source_file_name)
target = self.mkdtemp(dir=self.temp_dir)
self.remote.transfer_dir_parallel(source, target,
multiprocessing.cpu_count())
self.remote.transfer(source, target,
multiprocessing.cpu_count())
# test if the payload files are in the target directory
for filename in filenames:
self.assertTrue(os.path.isfile(os.path.join(target, filename)))

3
docs/src/cdist-reference.rst.sh
View file

@ -273,4 +273,7 @@ CDIST_REMOTE_EXEC
CDIST_REMOTE_COPY
Use this command for remote copy (should behave like scp).
CDIST_BETA
Enable beta functionalities.
eof

3
docs/src/man1/cdist.rst
View file

@ -236,6 +236,9 @@ CDIST_REMOTE_EXEC
CDIST_REMOTE_COPY
Use this command for remote copy (should behave like scp).
CDIST_BETA
Enable beta functionalities.
EXIT STATUS
-----------
The following exit values shall be returned: