cdist/cdist/exec/local.py
2020-06-21 16:53:47 +02:00

372 lines
14 KiB
Python

# -*- coding: utf-8 -*-
#
# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import sys
import re
import subprocess
import shutil
import logging
import tempfile
import time
import datetime
import cdist
import cdist.message
from cdist import core
import cdist.exec.util as util
CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
class Local:
"""Execute commands locally.
All interaction with the local side should be done through this class.
Directly accessing the local side from python code is a bug.
"""
def __init__(self,
target_host,
target_host_tags,
base_root_path,
host_dir_name,
exec_path=sys.argv[0],
initial_manifest=None,
add_conf_dirs=None,
cache_path_pattern=None,
quiet_mode=False,
configuration=None,
save_output_streams=True):
self.target_host = target_host
if target_host_tags is None:
self.target_host_tags = ""
else:
self.target_host_tags = ",".join(target_host_tags)
self.hostdir = host_dir_name
self.base_path = os.path.join(base_root_path, "data")
self.exec_path = exec_path
self.custom_initial_manifest = initial_manifest
self.cache_path_pattern = cache_path_pattern
self.quiet_mode = quiet_mode
if configuration:
self.configuration = configuration
else:
self.configuration = {}
self.save_output_streams = save_output_streams
self._init_log()
self._init_permissions()
self.mkdir(self.base_path)
self._init_cache_dir(None)
self._init_paths()
self._init_object_marker()
self._init_conf_dirs(add_conf_dirs)
def _init_log(self):
self.log = logging.getLogger(self.target_host[0])
# logger is not pickable, so remove it when we pickle
def __getstate__(self):
state = self.__dict__.copy()
if 'log' in state:
del state['log']
return state
# recreate logger when we unpickle
def __setstate__(self, state):
self.__dict__.update(state)
self._init_log()
def _init_permissions(self):
# Setup file permissions using umask
os.umask(0o077)
def _init_paths(self):
# Depending on out_path
self.bin_path = os.path.join(self.base_path, "bin")
self.conf_path = os.path.join(self.base_path, "conf")
self.global_explorer_out_path = os.path.join(self.base_path,
"explorer")
self.object_path = os.path.join(self.base_path, "object")
self.messages_path = os.path.join(self.base_path, "messages")
self.stdout_base_path = os.path.join(self.base_path, "stdout")
self.stderr_base_path = os.path.join(self.base_path, "stderr")
# Depending on conf_path
self.files_path = os.path.join(self.conf_path, "files")
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.manifest_path = os.path.join(self.conf_path, "manifest")
self.initial_manifest = (self.custom_initial_manifest or
os.path.join(self.manifest_path, "init"))
self.type_path = os.path.join(self.conf_path, "type")
def _init_object_marker(self):
self.object_marker_file = os.path.join(self.base_path, "object_marker")
# Does not need to be secure - just randomly different from .cdist
self.object_marker_name = tempfile.mktemp(prefix='.cdist-', dir='')
def _init_conf_dirs(self, add_conf_dirs):
self.conf_dirs = util.resolve_conf_dirs(
self.configuration, add_conf_dirs=add_conf_dirs)
def _init_directories(self):
self.mkdir(self.conf_path)
self.mkdir(self.global_explorer_out_path)
self.mkdir(self.object_path)
self.mkdir(self.bin_path)
self.mkdir(self.cache_path)
self.mkdir(self.stdout_base_path)
self.mkdir(self.stderr_base_path)
def create_files_dirs(self):
self._init_directories()
self._create_conf_path_and_link_conf_dirs()
self._create_messages()
self._link_types_for_emulator()
self._setup_object_marker_file()
def _setup_object_marker_file(self):
with open(self.object_marker_file, 'w') as fd:
fd.write("%s\n" % self.object_marker_name)
self.log.trace("Object marker %s saved in %s" % (
self.object_marker_name, self.object_marker_file))
def _init_cache_dir(self, cache_dir):
home_dir = cdist.home_dir()
if cache_dir:
self.cache_path = cache_dir
elif home_dir:
self.cache_path = os.path.join(home_dir, "cache")
else:
raise cdist.Error(
"No homedir setup and no cache dir location given")
def rmdir(self, path):
"""Remove directory on the local side."""
self.log.trace("Local rmdir: %s", path)
shutil.rmtree(path)
def mkdir(self, path):
"""Create directory on the local side."""
self.log.trace("Local mkdir: %s", path)
os.makedirs(path, exist_ok=True)
def run(self, command, env=None, return_output=False, message_prefix=None,
stdout=None, stderr=None, save_output=True, quiet_mode=False):
"""Run the given command with the given environment.
Return the output as a string.
"""
assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command)
quiet = self.quiet_mode or quiet_mode
do_save_output = save_output and not quiet and self.save_output_streams
close_stdout = False
close_stderr = False
if quiet:
stderr = subprocess.DEVNULL
stdout = subprocess.DEVNULL
elif do_save_output:
if not return_output and stdout is None:
stdout = util.get_std_fd(self.stdout_base_path, 'local')
close_stdout = True
if stderr is None:
stderr = util.get_std_fd(self.stderr_base_path, 'local')
close_stderr = True
if env is None:
env = os.environ.copy()
# Export __target_host, __target_hostname, __target_fqdn
# for use in __remote_{copy,exec} scripts
env['__target_host'] = self.target_host[0]
env['__target_hostname'] = self.target_host[1]
env['__target_fqdn'] = self.target_host[2]
# Export for emulator
env['__cdist_object_marker'] = self.object_marker_name
if message_prefix:
message = cdist.message.Message(message_prefix, self.messages_path)
env.update(message.env)
self.log.trace("Local run: %s", command)
try:
if return_output:
output = subprocess.check_output(
command, env=env, stderr=stderr).decode()
else:
subprocess.check_call(command, env=env, stderr=stderr,
stdout=stdout)
output = None
if do_save_output:
util.log_std_fd(self.log, command, stderr, 'Local stderr')
util.log_std_fd(self.log, command, stdout, 'Local stdout')
return output
except (OSError, subprocess.CalledProcessError) as error:
raise cdist.Error(" ".join(command) + ": " + str(error.args[1]))
finally:
if message_prefix:
message.merge_messages()
if close_stdout:
stdout.close()
if close_stderr:
stderr.close()
def run_script(self, script, env=None, return_output=False,
message_prefix=None, stdout=None, stderr=None):
"""Run the given script with the given environment.
Return the output as a string.
"""
if os.access(script, os.X_OK):
self.log.debug('%s is executable, running it', script)
command = [os.path.realpath(script)]
else:
command = [self.configuration.get('local_shell', "/bin/sh"), "-e"]
self.log.debug('%s is NOT executable, running it with %s',
script, " ".join(command))
command.append(script)
return self.run(command, env=env, return_output=return_output,
message_prefix=message_prefix, stdout=stdout,
stderr=stderr)
def _cache_subpath_repl(self, matchobj):
if matchobj.group(2) == '%P':
repl = str(os.getpid())
elif matchobj.group(2) == '%h':
repl = self.hostdir
elif matchobj.group(2) == '%N':
repl = self.target_host[0]
return matchobj.group(1) + repl
def _cache_subpath(self, start_time=time.time(), path_format=None):
if path_format:
repl_func = self._cache_subpath_repl
cache_subpath = re.sub(r'([^%]|^)(%h|%P|%N)', repl_func,
path_format)
dt = datetime.datetime.fromtimestamp(start_time)
cache_subpath = dt.strftime(cache_subpath)
else:
cache_subpath = self.hostdir
i = 0
while i < len(cache_subpath) and cache_subpath[i] == os.sep:
i += 1
cache_subpath = cache_subpath[i:]
if not cache_subpath:
cache_subpath = self.hostdir
return cache_subpath
def save_cache(self, start_time=time.time()):
self.log.trace("cache subpath pattern: {}".format(
self.cache_path_pattern))
cache_subpath = self._cache_subpath(start_time,
self.cache_path_pattern)
self.log.debug("cache subpath: {}".format(cache_subpath))
destination = os.path.join(self.cache_path, cache_subpath)
self.log.trace(("Saving cache: " + self.base_path + " to " +
destination))
if not os.path.exists(destination):
shutil.move(self.base_path, destination)
else:
for direntry in os.listdir(self.base_path):
srcentry = os.path.join(self.base_path, direntry)
destentry = os.path.join(destination, direntry)
try:
if os.path.isdir(destentry):
shutil.rmtree(destentry)
elif os.path.exists(destentry):
os.remove(destentry)
except (PermissionError, OSError) as e:
raise cdist.Error(
"Cannot delete old cache entry {}: {}".format(
destentry, e))
shutil.move(srcentry, destentry)
# add target_host since cache dir can be hash-ed target_host
host_cache_path = os.path.join(destination, "target_host")
with open(host_cache_path, 'w') as hostf:
print(self.target_host[0], file=hostf)
def _create_messages(self):
"""Create empty messages"""
with open(self.messages_path, "w"):
pass
def _create_conf_path_and_link_conf_dirs(self):
# Create destination directories
for sub_dir in CONF_SUBDIRS_LINKED:
self.mkdir(os.path.join(self.conf_path, sub_dir))
# Iterate over all directories and link the to the output dir
for conf_dir in self.conf_dirs:
self.log.debug("Checking conf_dir %s ..." % (conf_dir))
for sub_dir in CONF_SUBDIRS_LINKED:
current_dir = os.path.join(conf_dir, sub_dir)
# Allow conf dirs to contain only partial content
if not os.path.exists(current_dir):
continue
for entry in os.listdir(current_dir):
src = os.path.abspath(os.path.join(conf_dir,
sub_dir,
entry))
dst = os.path.join(self.conf_path, sub_dir, entry)
# Already exists? remove and link
if os.path.exists(dst):
os.unlink(dst)
self.log.trace("Linking %s to %s ..." % (src, dst))
try:
os.symlink(src, dst)
except OSError as e:
raise cdist.Error("Linking %s %s to %s failed: %s" % (
sub_dir, src, dst, e.__str__()))
def _link_types_for_emulator(self):
"""Link emulator to types"""
src = os.path.abspath(self.exec_path)
for cdist_type in core.CdistType.list_types(self.type_path):
dst = os.path.join(self.bin_path, cdist_type.name)
self.log.trace("Linking emulator: %s to %s", src, dst)
try:
os.symlink(src, dst)
except OSError as e:
raise cdist.Error(
"Linking emulator from %s to %s failed: %s" % (
src, dst, e.__str__()))