evilham/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge remote-tracking branch 'ungleich/master' into fix-hostdir

Browse source
This commit is contained in:
Steven Armstrong 2016-07-07 14:41:09 +02:00
commit a3d15caaa1
211 changed files with 3852 additions and 1303 deletions
Download patch file Download diff file Expand all files Collapse all files

65
cdist/exec/local.py
View file

@ -34,6 +34,10 @@ import hashlib
import cdist
import cdist.message
from cdist import core
import cdist.exec.util as exec_util
CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
class Local(object):
"""Execute commands locally.
@ -82,7 +86,8 @@ class Local(object):
@property
def dist_conf_dir(self):
return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), "conf"))
return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__),
"conf"))
@property
def home_dir(self):
@ -107,15 +112,17 @@ class Local(object):
# Depending on out_path
self.bin_path = os.path.join(self.base_path, "bin")
self.conf_path = os.path.join(self.base_path, "conf")
self.global_explorer_out_path = os.path.join(self.base_path, "explorer")
self.global_explorer_out_path = os.path.join(self.base_path,
"explorer")
self.object_path = os.path.join(self.base_path, "object")
self.messages_path = os.path.join(self.base_path, "messages")
self.files_path = os.path.join(self.conf_path, "files")
# Depending on conf_path
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.manifest_path = os.path.join(self.conf_path, "manifest")
self.initial_manifest = (self.custom_initial_manifest or
os.path.join(self.manifest_path, "init"))
os.path.join(self.manifest_path, "init"))
self.type_path = os.path.join(self.conf_path, "type")
@ -161,8 +168,8 @@ class Local(object):
with open(self.object_marker_file, 'w') as fd:
fd.write("%s\n" % self.object_marker_name)
self.log.debug("Object marker %s saved in %s" % (self.object_marker_name, self.object_marker_file))
self.log.debug("Object marker %s saved in %s" % (
self.object_marker_name, self.object_marker_file))
def _init_cache_dir(self, cache_dir):
if cache_dir:
@ -171,7 +178,8 @@ class Local(object):
if self.home_dir:
self.cache_path = os.path.join(self.home_dir, "cache")
else:
raise cdist.Error("No homedir setup and no cache dir location given")
raise cdist.Error(
"No homedir setup and no cache dir location given")
def rmdir(self, path):
"""Remove directory on the local side."""
@ -189,7 +197,8 @@ class Local(object):
"""
self.log.debug("Local run: %s", command)
assert isinstance(command, (list, tuple)), "list or tuple argument expected, got: %s" % command
assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command)
if env is None:
env = os.environ.copy()
@ -204,28 +213,30 @@ class Local(object):
env.update(message.env)
try:
output, errout = exec_util.call_get_output(command, env=env)
self.log.debug("Local stdout: {}".format(output))
self.log.debug("Local stderr: {}".format(errout))
if return_output:
return subprocess.check_output(command, env=env).decode()
else:
subprocess.check_call(command, env=env)
except subprocess.CalledProcessError:
raise cdist.Error("Command failed: " + " ".join(command))
return output.decode()
except subprocess.CalledProcessError as e:
exec_util.handle_called_process_error(e, command)
except OSError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1])
finally:
if message_prefix:
message.merge_messages()
def run_script(self, script, env=None, return_output=False, message_prefix=None):
def run_script(self, script, env=None, return_output=False,
message_prefix=None):
"""Run the given script with the given environment.
Return the output as a string.
"""
command = [ os.environ.get('CDIST_LOCAL_SHELL',"/bin/sh") , "-e"]
command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"]
command.append(script)
return self.run(command=command, env=env, return_output=return_output, message_prefix=message_prefix)
return self.run(command=command, env=env, return_output=return_output,
message_prefix=message_prefix)
def save_cache(self):
destination = os.path.join(self.cache_path, self.target_host)
@ -235,7 +246,8 @@ class Local(object):
if os.path.exists(destination):
shutil.rmtree(destination)
except PermissionError as e:
raise cdist.Error("Cannot delete old cache %s: %s" % (destination, e))
raise cdist.Error(
"Cannot delete old cache %s: %s" % (destination, e))
shutil.move(self.base_path, destination)
@ -245,14 +257,14 @@ class Local(object):
pass
def _create_conf_path_and_link_conf_dirs(self):
# Link destination directories
for sub_dir in [ "explorer", "manifest", "type" ]:
# Create destination directories
for sub_dir in CONF_SUBDIRS_LINKED:
self.mkdir(os.path.join(self.conf_path, sub_dir))
# Iterate over all directories and link the to the output dir
for conf_dir in self.conf_dirs:
self.log.debug("Checking conf_dir %s ..." % (conf_dir))
for sub_dir in [ "explorer", "manifest", "type" ]:
for sub_dir in CONF_SUBDIRS_LINKED:
current_dir = os.path.join(conf_dir, sub_dir)
# Allow conf dirs to contain only partial content
@ -261,18 +273,21 @@ class Local(object):
for entry in os.listdir(current_dir):
rel_entry_path = os.path.join(sub_dir, entry)
src = os.path.abspath(os.path.join(conf_dir, sub_dir, entry))
src = os.path.abspath(os.path.join(conf_dir,
sub_dir,
entry))
dst = os.path.join(self.conf_path, sub_dir, entry)
# Already exists? remove and link
if os.path.exists(dst):
os.unlink(dst)
self.log.debug("Linking %s to %s ..." % (src, dst))
try:
os.symlink(src, dst)
except OSError as e:
raise cdist.Error("Linking %s %s to %s failed: %s" % (sub_dir, src, dst, e.__str__()))
raise cdist.Error("Linking %s %s to %s failed: %s" % (
sub_dir, src, dst, e.__str__()))
def _link_types_for_emulator(self):
"""Link emulator to types"""
@ -284,4 +299,6 @@ class Local(object):
try:
os.symlink(src, dst)
except OSError as e:
raise cdist.Error("Linking emulator from %s to %s failed: %s" % (src, dst, e.__str__()))
raise cdist.Error(
"Linking emulator from %s to %s failed: %s" % (
src, dst, e.__str__()))

29
cdist/exec/remote.py
View file

@ -26,9 +26,11 @@ import sys
import glob
import subprocess
import logging
import cdist.exec.util as exec_util
import cdist
class DecodeError(cdist.Error):
def __init__(self, command):
self.command = command
@ -74,7 +76,6 @@ class Remote(object):
os.environ['__remote_copy'] = self._copy
os.environ['__remote_exec'] = self._exec
def create_files_dirs(self):
self.rmdir(self.base_path)
self.mkdir(self.base_path)
@ -100,11 +101,13 @@ class Remote(object):
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(self.target_host, destination)])
command.extend([path, '{0}:{1}'.format(
self.target_host, destination)])
self._run_command(command)
else:
command = self._copy.split()
command.extend([source, '{0}:{1}'.format(self.target_host, destination)])
command.extend([source, '{0}:{1}'.format(
self.target_host, destination)])
self._run_command(command)
def run_script(self, script, env=None, return_output=False):
@ -113,7 +116,7 @@ class Remote(object):
"""
command = [ os.environ.get('CDIST_REMOTE_SHELL',"/bin/sh") , "-e"]
command = [os.environ.get('CDIST_REMOTE_SHELL', "/bin/sh"), "-e"]
command.append(script)
return self.run(command, env, return_output)
@ -147,8 +150,8 @@ class Remote(object):
# /bin/csh will execute this script in the right way.
if env:
remote_env = [" export %s=%s;" % item for item in env.items()]
string_cmd = ("/bin/sh -c '" + " ".join(remote_env)
+ " ".join(command) + "'")
string_cmd = ("/bin/sh -c '" + " ".join(remote_env) +
" ".join(command) + "'")
cmd.append(string_cmd)
else:
cmd.extend(command)
@ -159,7 +162,8 @@ class Remote(object):
Return the output as a string.
"""
assert isinstance(command, (list, tuple)), "list or tuple argument expected, got: %s" % command
assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command)
# export target_host for use in __remote_{exec,copy} scripts
os_environ = os.environ.copy()
@ -167,12 +171,13 @@ class Remote(object):
self.log.debug("Remote run: %s", command)
try:
output, errout = exec_util.call_get_output(command, env=os_environ)
self.log.debug("Remote stdout: {}".format(output))
self.log.debug("Remote stderr: {}".format(errout))
if return_output:
return subprocess.check_output(command, env=os_environ).decode()
else:
subprocess.check_call(command, env=os_environ)
except subprocess.CalledProcessError:
raise cdist.Error("Command failed: " + " ".join(command))
return output.decode()
except subprocess.CalledProcessError as e:
exec_util.handle_called_process_error(e, command)
except OSError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1])
except UnicodeDecodeError:

84
cdist/exec/util.py Normal file
View file

@ -0,0 +1,84 @@
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import subprocess
import sys
from tempfile import TemporaryFile
import cdist
STDERR_UNSUPPORTED = '<Not supported in this python version>'
def call_get_output(command, env=None):
"""Run the given command with the given environment.
Return the tuple of stdout and stderr output as a byte strings.
"""
assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: {}".format(command))
if sys.version_info >= (3, 5):
return call_get_out_err(command, env)
else:
return (call_get_stdout(command, env), STDERR_UNSUPPORTED)
def handle_called_process_error(err, command):
if sys.version_info >= (3, 5):
errout = err.stderr
else:
errout = STDERR_UNSUPPORTED
raise cdist.Error("Command failed: " + " ".join(command) +
" with returncode: {} and stdout: {}, stderr: {}".format(
err.returncode, err.output, errout))
def call_get_stdout(command, env=None):
"""Run the given command with the given environment.
Return the stdout output as a byte string, stderr is ignored.
"""
assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: {}".format(command))
with TemporaryFile() as fout:
subprocess.check_call(command, env=env, stdout=fout)
fout.seek(0)
output = fout.read()
return output
def call_get_out_err(command, env=None):
"""Run the given command with the given environment.
Return the tuple of stdout and stderr output as a byte strings.
"""
assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: {}".format(command))
with TemporaryFile() as fout, TemporaryFile() as ferr:
subprocess.check_call(command, env=env,
stdout=fout, stderr=ferr)
fout.seek(0)
ferr.seek(0)
output = (fout.read(), ferr.read())
return output