diff --git a/.gitignore b/.gitignore index d606aec7..69a8ea98 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,8 @@ doc/man/man7/cdist-type__*.text doc/man/man7/cdist-reference.text doc/man/man*/docbook-xsl.css -# Ignore cache for version control -cache/ +# Ignore cdist cache for version control +/cache/ -# Python -bin/__pycache__/ +# Python / cache +__pycache__/ diff --git a/README b/README index 5e60a093..aab4c3a5 100644 --- a/README +++ b/README @@ -15,7 +15,7 @@ "P' "" "" -[[!toc levels=2]] +[[!toc levels=3]] ## Introduction @@ -78,6 +78,7 @@ cdist was tested or is know to run on at least * A posix like shell * Python (>= 3.2 required) * SSH-Client + * Asciidoc (for building the manpages) ### Client ("target host") @@ -85,20 +86,46 @@ cdist was tested or is know to run on at least * SSH-Server -## Getting cdist +## Installation + +### Preperation + +Ensure you have Python 3.x and the **argparse** module installed on +the machine you use to **deploy to the targets**. + +#### Archlinux + +Archlinux already has python >= 3.2, so you only need to do: + + pacman -S python + +#### Debian + + aptitude install python3 python3-setuptools + easy_install3 argparse + + +#### Gentoo + +Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3). +If you want to ensure nothing breaks you must set back the python version to what was default before. + + emerge -av =python-3.2.2 --autounmask-write + emerge -av =python-3.2.2 + eselect python list + eselect python list set python3.2 + +### Get cdist You can clone cdist from git, which gives you the advantage of having a version control in place for development of your own stuff as well. - -### Installation - To install cdist, execute the following commands: git clone git://git.schottelius.org/cdist cd cdist export PATH=$PATH:$(pwd -P)/bin - # If you want the manpages (requires gmake and asciidoc to be installed) + # If you want the manpages ./build.sh man export MANPATH=$MANPATH:$(pwd -P)/doc/man diff --git a/bin/cdist b/bin/cdist index 5ce947ef..0bf3ed9c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -21,764 +21,104 @@ # import argparse -import datetime import logging -import multiprocessing import os import re -import subprocess -import shutil -import stat import sys -import tempfile -BANNER = """ - .. . .x+=:. s - dF @88> z` ^% :8 - '88bu. %8P . is an object - if content == DOT_CDIST: - object_paths.append(starting_point) - - return object_paths - - def get_type_from_object(self, cdist_object): - """Returns the first part (i.e. type) of an object""" - return cdist_object.split(os.sep)[0] - - def get_object_id_from_object(self, cdist_object): - """Returns everything but the first part (i.e. object_id) of an object""" - return os.sep.join(cdist_object.split(os.sep)[1:]) - - def object_dir(self, cdist_object): - """Returns the full path to the object (including .cdist)""" - return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) - - def remote_object_dir(self, cdist_object): - """Returns the remote full path to the object (including .cdist)""" - return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) - - def object_parameter_dir(self, cdist_object): - """Returns the dir to the object parameter""" - return os.path.join(self.object_dir(cdist_object), "parameter") - - def remote_object_parameter_dir(self, cdist_object): - """Returns the remote dir to the object parameter""" - return os.path.join(self.remote_object_dir(cdist_object), "parameter") - - def object_code_paths(self, cdist_object): - """Return paths to code scripts of object""" - return [os.path.join(self.object_dir(cdist_object), "code-local"), - os.path.join(self.object_dir(cdist_object), "code-remote")] - - def list_objects(self): - """Return list of existing objects""" - - objects = [] - if os.path.isdir(self.object_base_dir): - object_paths = self.list_object_paths(self.object_base_dir) - - for path in object_paths: - objects.append(os.path.relpath(path, self.object_base_dir)) - - return objects - - def type_dir(self, type, *args): - """Return directory the type""" - return os.path.join(self.type_base_dir, type, *args) - - def remote_type_explorer_dir(self, type): - """Return remote directory that holds the explorers of a type""" - return os.path.join(REMOTE_TYPE_DIR, type, "explorer") - - def transfer_object_parameter(self, cdist_object): - """Transfer the object parameter to the remote destination""" - # Create base path before using mkdir -p - self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) - - # Synchronise parameter dir afterwards - self.transfer_dir(self.object_parameter_dir(cdist_object), - self.remote_object_parameter_dir(cdist_object)) - - def transfer_global_explorers(self): - """Transfer the global explorers""" - self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) - self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) - - def transfer_type_explorers(self, type): - """Transfer explorers of a type, but only once""" - if type in self.type_explorers_transferred: - log.debug("Skipping retransfer for explorers of %s", type) - return - else: - # Do not retransfer - self.type_explorers_transferred[type] = 1 - - src = self.type_dir(type, "explorer") - remote_base = os.path.join(REMOTE_TYPE_DIR, type) - dst = self.remote_type_explorer_dir(type) - - # Only continue, if there is at least the directory - if os.path.isdir(src): - # Ensure that the path exists - self.remote_mkdir(remote_base) - self.transfer_dir(src, dst) - - - def link_type_to_emulator(self): - """Link type names to cdist-type-emulator""" - source = os.path.abspath(sys.argv[0]) - for type in self.list_types(): - destination = os.path.join(self.bin_dir, type) - log.debug("Linking %s to %s", source, destination) - os.symlink(source, destination) - - def run_global_explores(self): - """Run global explorers""" - explorers = self.list_global_explorers() - if(len(explorers) == 0): - exit_error("No explorers found in", self.global_explorer_dir) - - self.transfer_global_explorers() - for explorer in explorers: - output = self.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.remote_global_explorer_path(explorer)) - - self.run_or_fail(cmd, stdout=output_fd, remote=True) - output_fd.close() - - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" - # Based on bin/cdist-object-explorer-run - - # Transfering explorers for this type - type = self.get_type_from_object(cdist_object) - self.transfer_type_explorers(type) - - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) - - # Need to transfer at least the parameters for objects to be useful - self.transfer_object_parameter(cdist_object) - - explorers = self.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) - output_fd.close() - - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.remove_remote_dir(REMOTE_BASE_DIR) - self.remote_mkdir(REMOTE_BASE_DIR) - - def run_initial_manifest(self): - """Run the initial manifest""" - env = { "__manifest" : self.manifest_dir } - self.run_manifest(self.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.get_type_from_object(cdist_object) - manifest = self.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.object_dir(cdist_object), - "__object_id": self.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) - - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.bin_dir + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - - # Legacy stuff to make cdist-type-emulator work - env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") - env['__cdist_local_base_dir'] = self.temp_dir - - # Submit information to new type emulator - env['__cdist_manifest'] = manifest - env['__cdist_type_base_dir'] = self.type_base_dir - - # Other environment stuff - if extra_env: - env.update(extra_env) - - self.shell_run_or_debug_fail(manifest, [manifest], env=env) - - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.object_dir(cdist_object), "require") - requirements = file_to_list(file) - type = self.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - env["__object"] = self.object_dir(cdist_object) - env["__object_id"] = self.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.type_dir(type) - - if mode == "gencode": - paths = [ - self.type_dir(type, "gencode-local"), - self.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and - outfile=os.path.join(self.object_dir(cdist_object), - os.path.basename(bin)[3:]) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - if mode == "code": - local_dir = self.object_dir(cdist_object) - remote_dir = self.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - self.run_or_fail([bin], remote=False) - - - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.transfer_file(local_remote_code, remote_remote_code) - self.run_or_fail([remote_remote_code], remote=True) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - old_objects = [] - objects = self.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - log.debug("Prepare stage") - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) - - objects = self.list_objects() - - def stage_run(self): - """The final (and real) step of deployment""" - log.debug("Actual run objects") - # Now do the final steps over the existing objects - for cdist_object in self.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() - - self.stage_prepare() - self.stage_run() - - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() - -def banner(args): - """Guess what :-)""" - print(BANNER) - sys.exit(0) - -def config(args): - """Configure remote system""" - process = {} - - time_start = datetime.datetime.now() - - for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) - if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) - process[host].start() - else: - c.deploy_and_cleanup() - - if args.parallel: - for p in process.keys(): - log.debug("Joining %s", p) - process[p].join() - - time_end = datetime.datetime.now() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start).total_seconds()) - -def install(args): - """Install remote system""" - process = {} - -def emulator(): - """Emulate type commands (i.e. __file and co)""" - type = os.path.basename(sys.argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) - param_dir = os.path.join(type_dir, "parameter") - global_dir = os.environ['__global'] - object_source = os.environ['__cdist_manifest'] - - parser = argparse.ArgumentParser(add_help=False) - - # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=False) - - # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=True) - - # Setup positional parameter, if not singleton - - if not os.path.isfile(os.path.join(type_dir, "singleton")): - parser.add_argument("object_id", nargs=1) - - # And finally verify parameter - args = parser.parse_args(sys.argv[1:]) - - # Setup object_id - if os.path.isfile(os.path.join(type_dir, "singleton")): - object_id = "singleton" - else: - object_id = args.object_id[0] - del args.object_id - - # FIXME: / hardcoded - better portable solution available? - if object_id[0] == '/': - object_id = object_id[1:] - - # FIXME: verify object id - log.debug(args) - - object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) - param_out_dir = os.path.join(object_dir, "parameter") - - object_source_file = os.path.join(object_dir, "source") - - if os.path.exists(param_out_dir): - object_exists = True - old_object_source_fd = open(object_source_file, "r") - old_object_source = old_object_source_fd.readlines() - old_object_source_fd.close() - - else: - object_exists = False - try: - os.makedirs(param_out_dir, exist_ok=True) - except OSError as error: - exit_error(param_out_dir + ": " + error.args[1]) - - # Record parameter - params = vars(args) - for param in params: - value = getattr(args, param) - if value: - file = os.path.join(param_out_dir, param) - log.debug(file + "<-" + param + " = " + value) - - # Already exists, verify all parameter are the same - if object_exists: - if not os.path.isfile(file): - print("New parameter + " + param + "specified, aborting") - print("Source = " + old_object_source + "new =" + object_source) - sys.exit(1) - else: - param_fd = open(file, "r") - param_old = param_fd.realines() - param_fd.close() - - if(param_old != param): - print("Parameter differs: " + param_old + "vs," + param) - print("Source = " + old_object_source + "new =" + object_source) - sys.exit(1) - else: - param_fd = open(file, "w") - param_fd.writelines(value) - param_fd.close() - - # Record requirements - if "__require" in os.environ: - requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) - require_fd = open(os.path.join(object_dir, "require"), "a") - require_fd.writelines(requirements.split(" ")) - require_fd.close() - - # Record / Append source - source_fd = open(os.path.join(object_dir, "source"), "a") - source_fd.writelines(object_source) - source_fd.close() - - # sys.exit(1) - print("Finished " + type + "/" + object_id + repr(params)) +# Ensure our /lib/ is included into PYTHON_PATH +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) +TYPE_PREFIX = "__" def commandline(): - """Parse command line""" - # Construct parser others can reuse - parser = {} - # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') + """Parse command line""" + # Construct parser others can reuse + parser = {} + # Options _all_ parsers have in common + parser['loglevel'] = argparse.ArgumentParser(add_help=False) + parser['loglevel'].add_argument('-d', '--debug', + help='Set log level to debug', action='store_true', + default=False) + parser['loglevel'].add_argument('-v', '--verbose', + help='Set log level to info, be more verbose', + action='store_true', default=False) - # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) - parser['main'].add_argument('-V', '--version', - help='Show version', action='version', - version='%(prog)s ' + VERSION) - parser['sub'] = parser['main'].add_subparsers(title="Commands") + # Main subcommand parser + parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION, + parents=[parser['loglevel']]) + parser['main'].add_argument('-V', '--version', + help='Show version', action='version', + version='%(prog)s ' + cdist.VERSION) + parser['sub'] = parser['main'].add_subparsers(title="Commands") - # Banner - parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) - parser['banner'].set_defaults(func=banner) + # Banner + parser['banner'] = parser['sub'].add_parser('banner', + parents=[parser['loglevel']]) + parser['banner'].set_defaults(func=cdist.banner.banner) - # Config and install (common stuff) - parser['configinstall'] = argparse.ArgumentParser(add_help=False) - parser['configinstall'].add_argument('host', nargs='+', - help='one or more hosts to operate on') - parser['configinstall'].add_argument('-c', '--cdist-home', - help='Change cdist home (default: .. from bin directory)', - action='store') - parser['configinstall'].add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest', - dest='manifest', required=False) - parser['configinstall'].add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['configinstall'].add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') + # Config and install (common stuff) + parser['configinstall'] = argparse.ArgumentParser(add_help=False) + parser['configinstall'].add_argument('host', nargs='+', + help='one or more hosts to operate on') + parser['configinstall'].add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store') + parser['configinstall'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest', + dest='manifest', required=False) + parser['configinstall'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['configinstall'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') - # Config - parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) - parser['config'].set_defaults(func=config) + # Config + parser['config'] = parser['sub'].add_parser('config', + parents=[parser['loglevel'], parser['configinstall']]) + parser['config'].set_defaults(func=cdist.config.config) - # Install - parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) - parser['install'].set_defaults(func=install) + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['loglevel'], parser['configinstall']]) + parser['install'].set_defaults(func=cdist.install.install) - for p in parser: - parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" + for p in parser: + parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" - args = parser['main'].parse_args(sys.argv[1:]) + args = parser['main'].parse_args(sys.argv[1:]) - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) + # Loglevels are handled globally in here and debug wins over verbose + if args.verbose: + logging.root.setLevel(logging.INFO) + if args.debug: + logging.root.setLevel(logging.DEBUG) - args.func(args) + log.debug(args) + args.func(args) if __name__ == "__main__": - try: - if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - emulator() - else: - commandline() - except KeyboardInterrupt: - sys.exit(0) + try: + logging.basicConfig(format='%(levelname)s: %(message)s') + + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): + import cdist.emulator + cdist.emulator.run(sys.argv) + else: + import cdist + import cdist.banner + import cdist.config + import cdist.exec + import cdist.install + import cdist.path + + commandline() + except KeyboardInterrupt: + sys.exit(0) + except cdist.Error as e: + log.error(e) + sys.exit(1) diff --git a/bin/cdist.py b/bin/cdist.py deleted file mode 120000 index 9a039b33..00000000 --- a/bin/cdist.py +++ /dev/null @@ -1 +0,0 @@ -cdist \ No newline at end of file diff --git a/build.sh b/build.sh index bef4393b..d20e0211 100755 --- a/build.sh +++ b/build.sh @@ -27,8 +27,8 @@ #set -e # Manpage and HTML -A2XM="a2x -f manpage --no-xmllint" -A2XH="a2x -f xhtml --no-xmllint" +A2XM="a2x -f manpage --no-xmllint -a encoding=UTF-8" +A2XH="a2x -f xhtml --no-xmllint -a encoding=UTF-8" # Developer webbase WEBDIR=$HOME/niconetz diff --git a/conf/explorer/os b/conf/explorer/os index e922c067..e59301e7 100755 --- a/conf/explorer/os +++ b/conf/explorer/os @@ -65,8 +65,13 @@ if [ -f /etc/SuSE-release ]; then exit 0 fi +if [ -f /etc/owl-release ]; then + echo owl + exit 0 +fi + if [ -f /etc/cdist-preos ]; then - echo preos + echo cdist-preos exit 0 fi diff --git a/conf/explorer/os_version b/conf/explorer/os_version index 08fda60b..ef80e8fc 100755 --- a/conf/explorer/os_version +++ b/conf/explorer/os_version @@ -42,6 +42,9 @@ case "$($__explorer/os)" in *bsd|solaris) uname -r ;; + owl) + cat /etc/owl-release + ;; redhat|centos) cat /etc/redhat-release ;; diff --git a/conf/type/__partition_msdos_apply/man.text b/conf/type/__partition_msdos_apply/man.text index 4d4f127c..6cc53b77 100644 --- a/conf/type/__partition_msdos_apply/man.text +++ b/conf/type/__partition_msdos_apply/man.text @@ -5,7 +5,7 @@ Steven Armstrong NAME ---- -cdist-type__partition_msdos_apply +cdist-type__partition_msdos_apply - Apply dos partition settings DESCRIPTION diff --git a/doc/changelog b/doc/changelog index cc6aa78b..9fb11307 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,5 +1,15 @@ -2.0.1: - * Bugfix cdist: Always print source of error in case of exec errors +2.0.3: + * Improved logging, added --verbose, by more quiet by default + +2.0.2: 2011-09-27 + * Add support for detection of OpenWall Linux (Matthias Teege) + * Add support for __debug variable in manifests + * Bugfix core: Various issues with type emulator + +2.0.1: 2011-09-23 + * Bugfix core: Always print source of error in case of exec errors + * Bugfix core: Various smaller bugs in string concatenation + * Feature: Add marker "changed" to changed objects 2.0.0: 2011-09-16 * New Type: __package_rubygem (Chase Allen James) diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index 5439a1b9..4cdb6fcf 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -9,6 +9,12 @@ CORE - allow cdist to run without $PATH setup: ./bin/cdist-deploy-to - support non-ssh access? +TESTS +----- +- multiple defines of object: + - fail if different parameters + - succeed if same parameters + USER INTERFACE -------------- - add support $__tmp? @@ -28,6 +34,8 @@ USER INTERFACE -> given after manifest run already! - use absent/present for state by default? +- buggy output with packages that don't exist in archlinux and fedora: + python3 vs. python TYPES ------ diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 84745512..11c734f9 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,18 +1,296 @@ -2.0.1: - -- Rewrite cdist-type-emulator - - Remove legacy code in cdist - - Remove cdist-config - - Remove man1/cdist-type-emulator.text - - Remove the PATH=... part from the README - - - how to access output dir? - - Test: - __cdist_type_base_dir=$(pwd -P)/conf/type __file - - Fix / rewrite cdist-quickstart +- write tutorial!!!!!!!!! + - like ccollect! + - include ssh control master! + - add local/ hint (and add to git) + - add hint for ssh StrictHostKeyChecking no + - and that ssh will wait for answer of prompt + - nasty if used in parallel mode (scroll up!) + +- rewrite cdist-stages, remove +- update man7! +- exec flag is not true for manifest anymore + +SSH HINTS +--------- +Control master, ssh agent + +Everything you specify in manifests + + +# Intro of quickstart +# +cat << eof +$banner cdist version $__cdist_version + +Welcome to the interactive guide to cdist! +This is the interactive tutorial and beginners help for cdist and here's +our schedule: + + - Stages: How cdist operates + - Explorer: Explore facts of the target host + - Manifest: Map configurations to hosts + - Types: Bundled functionality + - Deploy a configuration to the local host! + +eof +__prompt "$continue" + +################################################################################ +# Stages +# +cat << eof + +To deploy configurations to a host, you call + + cdist-deploy-to + +which makes calls to other scripts, which realise the so called "stages". +Usually you'll not notice this, but in case you want to debug or hack cdist, +you can run each stage on its own. Besides that, you just need to remember +that the command cdist-deploy-to is the main cdist command. + +See also: + + Source of cdist-deploy-to(1), cdist-stages(7) + +eof +__prompt "$continue" + +################################################################################ +# Explorer +# +cat << eof + +The first thing cdist always does is running different explorers on the +target host. The explorers can be found in the directory + + ${__cdist_explorer_dir} + +An explorer is executed on the target host and its output is saved to a file. +You can use these files later to decide what or how to configure the host. + +For a demonstration, we'll call the OS explorer locally now, but remember: +This is only for demonstration, normally it is run on the target host. +The os explorer will which either displays the detected operating system or +nothing if it does not know your OS. + +See also: + + cdist-explorer(7) + +eof +explorer="${__cdist_explorer_dir}/os" + +__prompt "Press enter to execute $explorer" + +set -x +"$explorer" +set +x + +################################################################################ +# Manifest +# +cat << eof + +The initial manifest is the entry point for cdist to find out, what you would +like to have configured. It is located at + + ${__cdist_manifest_init} + +And can be as simple as + +-------------------------------------------------------------------------------- +__file /etc/cdist-configured --type file +-------------------------------------------------------------------------------- + +See also: + + cdist-manifest(7) + +eof +__prompt "$continue" + +cat << eof + +Let's take a deeper look at the initial manifest to understand what it means: + + __file /etc/cdist-configured --type file + | | | \\ + | | The parameter type \\ With the value file + | | + | | + | | This is the object id + | + __file is a so called "type" + + +This essentially looks like a standard command executed in the shell. +eof +__prompt "$continue" + +cat << eof + +And that's exactly true. Manifests are shell snippets that can use +types as commands with arguments. cdist prepends a special path +that contain links to the cdist-type-emulator, to \$PATH, so you +can use your types as a command. + +This is also the reason why types should always be prefixed with +"__", to prevent collisions with existing binaries. + +The object id is unique per type and used to prevent you from creating +the same object twice. + +Parameters are type specific and are always specified as --parameter . + +See also: + + cdist-type-build-emulation(1), cdist-type-emulator(1) + +eof +__prompt "$continue" + +################################################################################ +# Types +# +cat << eof + +Types are bundled functionality and are the main component of cdist. +If you want to have a feature x, you write the type __x. Types are stored in + + ${__cdist_type_dir} + +And cdist ships with some types already! + +See also: + + cdist-type(7) + +eof +__prompt "Press enter to see available types" + +set -x +ls ${__cdist_type_dir} +set +x + +cat << eof + +Types consist of the following parts: + + - ${__cdist_name_parameter} (${__cdist_name_parameter_required}/${__cdist_name_parameter_optional} + - ${__cdist_name_manifest} + - ${__cdist_name_explorer} + - ${__cdist_name_gencode} + +eof +__prompt "$continue" + + +cat << eof + +Every type must have a directory named ${__cdist_name_parameter}, which +contains required or optional parameters (in newline seperated files). + +If an object of a specific type was created in the initial manifest, +the manifest of the type is run and may create other objects. + +A type may have ${__cdist_name_explorer}, which are very similar to the +${__cdist_name_explorer} seen above, but with a different purpose: +They are specific to the type and are not relevant for other types. + +You may use them for instance to find out details on the target host, +so you can decide what to do on the target host eventually. + +After the ${__cdist_name_manifest} and the ${__cdist_name_explorer} of +a type have been run, ${__cdist_name_gencode} is executed, which creates +code to be executed on the target on stdout. + +eof +__prompt "$continue" + +################################################################################ +# Deployment +# + +cat << eof + +Now you've got some basic knowledge about cdist, let's configure your a host! + +Ensure that you have a ssh server running on the host and that you can login as root. + +eof + +__prompt "Enter hostname or press enter for localhost: " + +if [ "$answer" ]; then + host="$answer" +else + host="localhost" +fi + +manifestinit="conf/manifest/init" +cat << eof + +I'll now setup $manifestinit, containing the following code: + +-------------------------------------------------------------------------------- +# Every machine becomes a marker, so sysadmins know that automatic +# configurations are happening +__file /etc/cdist-configured + +case "\$__target_host" in + $host) + __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic + __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" + ;; +esac +-------------------------------------------------------------------------------- + +WARNING: This will overwrite ${manifestinit}. + +eof + +cat > "$__cdist_abs_mydir/../$manifestinit" << eof + +# Every machine becomes a marker, so sysadmins know that automatic +# configurations are happening +__file /etc/cdist-configured + +case "\$__target_host" in + $host) + __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic + __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" + ;; +esac + +eof + +chmod u+x "$__cdist_abs_mydir/../$manifestinit" + +cmd="cdist-deploy-to $host" + +__prompt "Press enter to run \"$cmd\"" + +# No quotes, we need field splitting +$cmd + +################################################################################ +# End +# + +cat << eof + + +-------------------------------------------------------------------------------- +That's it, this is the end of the cdist-quickstart. + +I hope you've got some impression on how cdist works, here are again some +pointers on where to continue to read: + + +eof -------------------------------------------------------------------------------- - Initial install support @@ -33,11 +311,13 @@ via __global/ - Support parallel execution - - and maximum number of parallel runs (-p X) - error handling / report failed hosts -- Allow manifest to be read from stdin - Create new video for cdist 2.0.0 http://www.youtube.com/watch?v=PRMjzy48eTI - Setup __debug, if -d is given, so other tools can reuse it + - implement everywhere to external! + +- remote_prefix: + scp vs. ssh issue diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index e38f157d..7196c3b3 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -154,8 +154,23 @@ done cat << eof +OBJECTS +------- +For object to object communication and tests, the following paths are +usable within a object directory: + +changed:: + This empty file exists in an object directory, if the object has + code to be excuted (either remote or local) + + ENVIRONMENT VARIABLES --------------------- +__debug:: + If this variable is setup, cdist runs in debug mode. + You can use this information, to only output stuff in debug + mode as well. + Available for: initial manifest, type manifest __explorer:: Directory that contains all global explorers. Available for: explorer diff --git a/doc/man/man1/cdist-deploy-stdin-to.text b/doc/man/man1/cdist-deploy-stdin-to.text deleted file mode 100644 index 14f19478..00000000 --- a/doc/man/man1/cdist-deploy-stdin-to.text +++ /dev/null @@ -1,30 +0,0 @@ -cdist-deploy-stdin-to(1) -======================== -Steven Armstrong - - -NAME ----- -cdist-deploy-stdin-to - Deploy the configuration given on stdin to host - - -SYNOPSIS --------- -echo "__file /tmp/whatever" | cdist-deploy-stdin-to HOSTNAME - - -DESCRIPTION ------------ -Use stdin as the manifest for cdist-deploy-to. - - -SEE ALSO --------- -- cdist(7) -- cdist-deploy-to(1) - - -COPYING -------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man7/cdist-explorer.text b/doc/man/man7/cdist-explorer.text index 63c7a5c9..e1909ab5 100644 --- a/doc/man/man7/cdist-explorer.text +++ b/doc/man/man7/cdist-explorer.text @@ -26,7 +26,7 @@ $__explorer/ (general and type explorer) or $__type_explorer/ (type explorer). In case of significant errors, the explorer may exit non-zero and return an -error message on stderr, which will cause the cdist run to abort. +error message on stderr, which will cause cdist to abort. You can also use stderr for debugging purposes while developing a new explorer. diff --git a/doc/man/man7/cdist-hacker.text b/doc/man/man7/cdist-hacker.text index efd6ef7d..b9f79d01 100644 --- a/doc/man/man7/cdist-hacker.text +++ b/doc/man/man7/cdist-hacker.text @@ -26,26 +26,12 @@ in the latest version, drop a mail to the cdist mailing list, subject prefixed with "[BUG] ". -UNDERSTANDING CDIST INTERNALS ------------------------------ -IF you are interested in how cdist internally works, you can open -bin/cdist-config and bin/cdist-deploy-to in your favorite editor and -read the scripts bin/cdist-deploy-to calls. The magnificent HACKERS_README -may be of great help as well. - - CODING CONVENTIONS (EVERYWHERE) ------------------------------- If something should be better done or needs to fixed, add the word FIXME nearby, so grepping for FIXME gives all positions that need to be fixed. -CODING CONVENTIONS (CORE) -------------------------- -- All variables exported by cdist are prefixed with a double underscore (__) -- All cdist-internal variables are prefixed with __cdist_ and are generally not exported. - - HOW TO SUBMIT STUFF FOR INCLUSION INTO UPSTREAM CDIST ----------------------------------------------------- If you did some cool changes to cdist, which you value as a benefit for diff --git a/doc/man/man7/cdist-quickstart.text b/doc/man/man7/cdist-quickstart.text deleted file mode 100755 index e146f1a8..00000000 --- a/doc/man/man7/cdist-quickstart.text +++ /dev/null @@ -1,310 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Give the user an introduction into cdist -# - -. cdist-config -set -eu - -banner="cdist-quickstart>" -continue="Press enter to continue or ctrl-c to abort." -create_continue="Press enter to create the described files/directories" - -__prompt() -{ - echo -n "$banner" "$@" - read answer -} - -################################################################################ -# Intro of quickstart -# -cat << eof -$banner cdist version $__cdist_version - -Welcome to the interactive guide to cdist! -This is the interactive tutorial and beginners help for cdist and here's -our schedule: - - - Stages: How cdist operates - - Explorer: Explore facts of the target host - - Manifest: Map configurations to hosts - - Types: Bundled functionality - - Deploy a configuration to the local host! - -eof -__prompt "$continue" - -################################################################################ -# Stages -# -cat << eof - -To deploy configurations to a host, you call - - cdist-deploy-to - -which makes calls to other scripts, which realise the so called "stages". -Usually you'll not notice this, but in case you want to debug or hack cdist, -you can run each stage on its own. Besides that, you just need to remember -that the command cdist-deploy-to is the main cdist command. - -See also: - - Source of cdist-deploy-to(1), cdist-stages(7) - -eof -__prompt "$continue" - -################################################################################ -# Explorer -# -cat << eof - -The first thing cdist always does is running different explorers on the -target host. The explorers can be found in the directory - - ${__cdist_explorer_dir} - -An explorer is executed on the target host and its output is saved to a file. -You can use these files later to decide what or how to configure the host. - -For a demonstration, we'll call the OS explorer locally now, but remember: -This is only for demonstration, normally it is run on the target host. -The os explorer will which either displays the detected operating system or -nothing if it does not know your OS. - -See also: - - cdist-explorer(7) - -eof -explorer="${__cdist_explorer_dir}/os" - -__prompt "Press enter to execute $explorer" - -set -x -"$explorer" -set +x - -################################################################################ -# Manifest -# -cat << eof - -The initial manifest is the entry point for cdist to find out, what you would -like to have configured. It is located at - - ${__cdist_manifest_init} - -And can be as simple as - --------------------------------------------------------------------------------- -__file /etc/cdist-configured --type file --------------------------------------------------------------------------------- - -See also: - - cdist-manifest(7) - -eof -__prompt "$continue" - -cat << eof - -Let's take a deeper look at the initial manifest to understand what it means: - - __file /etc/cdist-configured --type file - | | | \\ - | | The parameter type \\ With the value file - | | - | | - | | This is the object id - | - __file is a so called "type" - - -This essentially looks like a standard command executed in the shell. -eof -__prompt "$continue" - -cat << eof - -And that's exactly true. Manifests are shell snippets that can use -types as commands with arguments. cdist prepends a special path -that contain links to the cdist-type-emulator, to \$PATH, so you -can use your types as a command. - -This is also the reason why types should always be prefixed with -"__", to prevent collisions with existing binaries. - -The object id is unique per type and used to prevent you from creating -the same object twice. - -Parameters are type specific and are always specified as --parameter . - -See also: - - cdist-type-build-emulation(1), cdist-type-emulator(1) - -eof -__prompt "$continue" - -################################################################################ -# Types -# -cat << eof - -Types are bundled functionality and are the main component of cdist. -If you want to have a feature x, you write the type __x. Types are stored in - - ${__cdist_type_dir} - -And cdist ships with some types already! - -See also: - - cdist-type(7) - -eof -__prompt "Press enter to see available types" - -set -x -ls ${__cdist_type_dir} -set +x - -cat << eof - -Types consist of the following parts: - - - ${__cdist_name_parameter} (${__cdist_name_parameter_required}/${__cdist_name_parameter_optional} - - ${__cdist_name_manifest} - - ${__cdist_name_explorer} - - ${__cdist_name_gencode} - -eof -__prompt "$continue" - - -cat << eof - -Every type must have a directory named ${__cdist_name_parameter}, which -contains required or optional parameters (in newline seperated files). - -If an object of a specific type was created in the initial manifest, -the manifest of the type is run and may create other objects. - -A type may have ${__cdist_name_explorer}, which are very similar to the -${__cdist_name_explorer} seen above, but with a different purpose: -They are specific to the type and are not relevant for other types. - -You may use them for instance to find out details on the target host, -so you can decide what to do on the target host eventually. - -After the ${__cdist_name_manifest} and the ${__cdist_name_explorer} of -a type have been run, ${__cdist_name_gencode} is executed, which creates -code to be executed on the target on stdout. - -eof -__prompt "$continue" - -################################################################################ -# Deployment -# - -cat << eof - -Now you've got some basic knowledge about cdist, let's configure your a host! - -Ensure that you have a ssh server running on the host and that you can login as root. - -eof - -__prompt "Enter hostname or press enter for localhost: " - -if [ "$answer" ]; then - host="$answer" -else - host="localhost" -fi - -manifestinit="conf/manifest/init" -cat << eof - -I'll now setup $manifestinit, containing the following code: - --------------------------------------------------------------------------------- -# Every machine becomes a marker, so sysadmins know that automatic -# configurations are happening -__file /etc/cdist-configured - -case "\$__target_host" in - $host) - __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic - __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" - ;; -esac --------------------------------------------------------------------------------- - -WARNING: This will overwrite ${manifestinit}. - -eof - -cat > "$__cdist_abs_mydir/../$manifestinit" << eof - -# Every machine becomes a marker, so sysadmins know that automatic -# configurations are happening -__file /etc/cdist-configured - -case "\$__target_host" in - $host) - __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic - __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" - ;; -esac - -eof - -chmod u+x "$__cdist_abs_mydir/../$manifestinit" - -cmd="cdist-deploy-to $host" - -__prompt "Press enter to run \"$cmd\"" - -# No quotes, we need field splitting -$cmd - -################################################################################ -# End -# - -cat << eof - - --------------------------------------------------------------------------------- -That's it, this is the end of the cdist-quickstart. - -I hope you've got some impression on how cdist works, here are again some -pointers on where to continue to read: - -cdist(7), cdist-deploy-to(1), cdist-type(7), cdist-stages(7) - -eof diff --git a/doc/man/man7/cdist-stages.text b/doc/man/man7/cdist-stages.text index 294dffc7..8ac30015 100644 --- a/doc/man/man7/cdist-stages.text +++ b/doc/man/man7/cdist-stages.text @@ -32,11 +32,6 @@ explorers. Every existing explorer is run on the target and the output of all explorers are copied back into the local cache. The results can be used by manifests and types. -Related documentation: - - cdist-explorer-run-global(1) - - cdist-remote-explorer-run(1) - - cdist-explorer(7) - STAGE 2: RUN THE INITIAL MANIFEST --------------------------------- @@ -46,11 +41,6 @@ the objects as defined in the manifest for the specific host. In this stage, no conflicts may occur, i.e. no object of the same type with the same id may be created. -Related documentation: - - cdist-manifest-run-init(1) - - cdist-manifest-run(1) - - cdist-manifest(7) - STAGE 3: OBJECT INFORMATION RETRIEVAL ------------------------------------- @@ -59,12 +49,6 @@ transfered to the target host and executed. The results are transfered back and can be used in the following stages to decide what changes need to be made on the target to implement the desired state. -Related documentation: - - cdist-object-explorer-run(1) - - cdist-remote-explorer-run(1) - - cdist-type(7) - - cdist-explorer(7) - STAGE 4: RUN THE OBJECT MANIFEST -------------------------------- @@ -79,11 +63,6 @@ The newly created objects are merged back into the existing tree. No conflicts may occur during the merge. A conflict would mean that two different objects try to create the same object, which indicates a broken configuration. -Related documentation: - - cdist-object-manifest-run(1) - - cdist-manifest-run(1) - - cdist-type(7) - STAGE 5: CODE GENERATION ------------------------ @@ -92,29 +71,17 @@ gencode scripts. The gencode scripts generate the code to be executed on the target on stdout. If the gencode executables fail, they must print diagnostic messages on stderr and exit non-zero. -Related documentation: - - cdist-object-gencode-run(1) - - cdist-object-gencode(1) - - cdist-type(7) - STAGE 6: CODE EXECUTION ----------------------- For every object the resulting code from the previous stage is transferred to the target host and executed there to apply the configuration changes. -Related documentation: - - cdist-object-code-run(1) - - cdist-code-run(1) - STAGE 7: CACHE -------------- The cache stores the information from the current run for later use. -Related documentation: - - cdist-cache(1) - SUMMARY ------- @@ -126,8 +93,8 @@ in correct order. SEE ALSO -------- +- cdist(1) - cdist(7) -- cdist-deploy-to(1) - cdist-reference(7) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text new file mode 100644 index 00000000..80135da9 --- /dev/null +++ b/doc/man/man7/cdist-tutorial.text @@ -0,0 +1,83 @@ +cdist-tutorial(7) +================= +Nico Schottelius + + +NAME +---- +cdist-tutorial - a guided introduction into cdist + + +INTRODUCTION +------------ +This tutorial is aimed at people learning cdist and shows +typical approaches as well as gives an easy start into +the world of configuration management. + +This tutorial assumes you are configuring **localhost**, because +it is always available. Just repace **localhost** with your target +host for real life usage. + + + +QUICK START +----------- +For those who just want to configure a system with the +cdist configuration management and do not need (or want) +to understand everything. + +Cdist uses **ssh** for communication and transportation +and usually logs into the **target host** as the +**root** user. So you need to configure the **ssh server** +of the target host to allow root logins: Edit +the file **/etc/ssh/sshd_config** and add one of the following +lines: + +-------------------------------------------------------------------------------- +# Allow login only via public key +PermitRootLogin without-password + +# Allow login via password and public key +PermitRootLogin yes +-------------------------------------------------------------------------------- + +As cdist uses ssh intensively, it is recommended to setup authentication +with public keys: + +-------------------------------------------------------------------------------- +# Generate pubkey pair as a normal user +ssh-keygen + +# Copy pubkey over to target host +ssh-copy-id root@localhost +-------------------------------------------------------------------------------- + +As soon as you are able to login without passwort to the target host, +we can use cdist, to configure it. You can copy and paste the following +code into your shell to get started and configure localhost: + +-------------------------------------------------------------------------------- +# Get cdist +git clone git://git.schottelius.org/cdist + +# Create manifest (maps configuration to host(s) +cd cdist +echo '__file /etc/cdist-configured' > conf/manifest/init +chmod 0700 conf/manifest/init + +# Configure localhost +./bin/cdist config localhost + +# Find out that cdist created /etc/cdist-configured +ls -l /etc/cdist-configured +-------------------------------------------------------------------------------- + +The file 'conf/manifest/init' is usually the entry point for cdist, +to find out what to configure on which host. All manifests are +essentially shell scripts. Every manifest can use the types known to +cdist, which are usually underline prefixed (__). + + +SEE ALSO +-------- +cdist(1), cdist-type(7), cdist-stages(7) diff --git a/doc/man/man7/cdist-type.text b/doc/man/man7/cdist-type.text index 1af386fb..2439876c 100644 --- a/doc/man/man7/cdist-type.text +++ b/doc/man/man7/cdist-type.text @@ -35,10 +35,6 @@ __file /etc/cdist-configured --type file __package tree --state installed -------------------------------------------------------------------------------- -Internally cdist-type-emulator(1) will be called from cdist-manifest-run(1) to -save the given parameters into a cconfig database, so they can be accessed by -the manifest and gencode scripts of the type (see below). - A list of supported types can be found in the cdist-reference(7) manpage. SINGLETON TYPES @@ -111,7 +107,7 @@ __package_$type "$@" -------------------------------------------------------------------------------- As you can see, the type can reference different environment variables, -which are documented in cdist-environment-variables(7). +which are documented in cdist-reference(7). Always ensure the manifest is executable, otherwise cdist will not be able to execute it. diff --git a/doc/man/man7/cdist.text b/doc/man/man7/cdist.text index 9f7dbbab..2a5d1fe5 100644 --- a/doc/man/man7/cdist.text +++ b/doc/man/man7/cdist.text @@ -35,12 +35,11 @@ pull mechanism (client requests configuration). SEE ALSO -------- - Website: http://www.nico.schottelius.org/software/cdist/[] -- cdist-best-practise(7) -- cdist-deploy-to(1) - cdist-hacker(7) - cdist-manifest(7) -- cdist-quickstart(1) - cdist-type(7) +- cdist(1) +- cdist(7) COPYING diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py new file mode 100644 index 00000000..a0ca2ba2 --- /dev/null +++ b/lib/cdist/__init__.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +VERSION = "2.0.3" + +class Error(Exception): + """Base exception class for this project""" + pass diff --git a/lib/cdist/banner.py b/lib/cdist/banner.py new file mode 100644 index 00000000..a07deea9 --- /dev/null +++ b/lib/cdist/banner.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import sys + +log = logging.getLogger(__name__) + +BANNER = """ + .. . .x+=:. s + dF @88> z` ^% :8 + '88bu. %8P . . +# +# + +import datetime +import logging +import os +import stat +import sys + +log = logging.getLogger(__name__) + +import cdist.emulator +import cdist.path + +CODE_HEADER = "#!/bin/sh -e\n" + +class Config: + """Cdist main class to hold arbitrary data""" + + def __init__(self, target_host, + initial_manifest=False, + remote_user="root", + home=None, + exec_path=sys.argv[0], + debug=False): + + self.target_host = target_host + self.debug = debug + self.remote_user = remote_user + self.exec_path = exec_path + + # FIXME: broken - construct elsewhere! + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + + self.path = cdist.path.Path(self.target_host, + initial_manifest=initial_manifest, + remote_user=self.remote_user, + remote_prefix=self.remote_prefix, + base_dir=home, + debug=debug) + + self.objects_prepared = [] + + def cleanup(self): + self.path.cleanup() + + def run_global_explores(self): + """Run global explorers""" + log.info("Running global explorers") + explorers = self.path.list_global_explorers() + if(len(explorers) == 0): + raise CdistError("No explorers found in", self.path.global_explorer_dir) + + self.path.transfer_global_explorers() + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.path.remote_global_explorer_path(explorer)) + + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + + type = self.path.get_type_from_object(cdist_object) + self.path.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.path.transfer_object_parameter(cdist_object) + + explorers = self.path.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + self.link_emulator() + + def run_initial_manifest(self): + """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) + env = { "__manifest" : self.path.manifest_dir } + self.run_manifest(self.path.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.path.get_type_from_object(cdist_object) + manifest = self.path.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.path.object_dir(cdist_object), + "__object_id": self.path.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.path.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.path.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + + # Submit debug flag to manifest, can be used by emulator and types + if self.debug: + env['__debug'] = "yes" + + # Required for recording source + env['__cdist_manifest'] = manifest + + # Required to find types + env['__cdist_type_base_dir'] = self.path.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.path.object_dir(cdist_object), "require") + requirements = cdist.path.file_to_list(file) + type = self.path.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + env["__object"] = self.path.object_dir(cdist_object) + env["__object_id"] = self.path.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.path.type_dir(type) + + if mode == "gencode": + paths = [ + self.path.type_dir(type, "gencode-local"), + self.path.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.path.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.path.object_dir(cdist_object) + remote_dir = self.path.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + cdist.exec.run_or_fail([bin]) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.path.transfer_file(local_remote_code, remote_remote_code) + # FIXME: remote_prefix + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + log.info("Running object manifests and type explorers") + + old_objects = [] + objects = self.path.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.path.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.info("Generating and executing code") + # Now do the final steps over the existing objects + for cdist_object in self.path.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() + +def config(args): + """Configure remote system""" + process = {} + + time_start = datetime.datetime.now() + + for host in args.host: + c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) + if args.parallel: + log.debug("Creating child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + else: + c.deploy_and_cleanup() + + if args.parallel: + for p in process.keys(): + log.debug("Joining process %s", p) + process[p].join() + + time_end = datetime.datetime.now() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py new file mode 100644 index 00000000..38a58f8c --- /dev/null +++ b/lib/cdist/emulator.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import argparse +import logging +import os + +import cdist +import cdist.path + +log = logging.getLogger(__name__) + +def run(argv): + """Emulate type commands (i.e. __file and co)""" + type = os.path.basename(argv[0]) + type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) + param_dir = os.path.join(type_dir, "parameter") + global_dir = os.environ['__global'] + object_source = os.environ['__cdist_manifest'] + + if '__debug' in os.environ: + logging.root.setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser(add_help=False) + + for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=False) + for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=True) + + # If not singleton support one positional parameter + if not os.path.isfile(os.path.join(type_dir, "singleton")): + parser.add_argument("object_id", nargs=1) + + # And finally verify parameter + args = parser.parse_args(argv[1:]) + + # Setup object_id + if os.path.isfile(os.path.join(type_dir, "singleton")): + object_id = "singleton" + else: + object_id = args.object_id[0] + del args.object_id + + # FIXME: / hardcoded - better portable solution available? + if object_id[0] == '/': + object_id = object_id[1:] + + # Prefix output by object_self + logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s' + logging.basicConfig(format=logformat) + + # FIXME: verify object id + log.debug(args) + + object_dir = os.path.join(global_dir, "object", type, + object_id, cdist.path.DOT_CDIST) + param_out_dir = os.path.join(object_dir, "parameter") + + object_source_file = os.path.join(object_dir, "source") + + if os.path.exists(param_out_dir): + object_exists = True + old_object_source_fd = open(object_source_file, "r") + old_object_source = old_object_source_fd.readlines() + old_object_source_fd.close() + + else: + object_exists = False + try: + os.makedirs(param_out_dir, exist_ok=True) + except OSError as error: + raise cdist.Error(param_out_dir + ": " + error.args[1]) + + # Record parameter + params = vars(args) + for param in params: + value = getattr(args, param) + if value: + file = os.path.join(param_out_dir, param) + log.debug(file + "<-" + param + " = " + value) + + # Already exists, verify all parameter are the same + if object_exists: + if not os.path.isfile(file): + raise cdist.Error("New parameter \"" + + param + "\" specified, aborting\n" + + "Source = " + + " ".join(old_object_source) + + " new =" + object_source) + else: + param_fd = open(file, "r") + value_old = param_fd.readlines() + param_fd.close() + + if(value_old[0] != value): + raise cdist.Error("Parameter\"" + param + + "\" differs: " + " ".join(value_old) + " vs. " + + value + + "\nSource = " + " ".join(old_object_source) + + " new = " + object_source) + else: + param_fd = open(file, "w") + param_fd.writelines(value) + param_fd.close() + + # Record requirements + if "__require" in os.environ: + requirements = os.environ['__require'] + log.debug(object_id + ":Writing requirements: " + requirements) + require_fd = open(os.path.join(object_dir, "require"), "a") + require_fd.writelines(requirements.split(" ")) + require_fd.close() + + # Record / Append source + source_fd = open(os.path.join(object_dir, "source"), "a") + source_fd.writelines(object_source) + source_fd.close() + + log.debug("Finished " + type + "/" + object_id + repr(params)) + + +def link(exec_path, bin_dir, type_list): + """Link type names to cdist-type-emulator""" + source = os.path.abspath(exec_path) + for type in type_list: + destination = os.path.join(bin_dir, type) + log.debug("Linking %s to %s", source, destination) + os.symlink(source, destination) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py new file mode 100644 index 00000000..9cedefcc --- /dev/null +++ b/lib/cdist/exec.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import subprocess + +log = logging.getLogger(__name__) + +import cdist + +def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0][:0] = [ "/bin/sh", "-e" ] + + if remote_prefix: + args[0][:0] = remote_prefix + + log.debug("Shell exec cmd: %s", args) + + if 'env' in kargs: + log.debug("Shell exec env: %s", kargs['env']) + + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + log.error("Code that raised the error:\n") + if remote_prefix: + run_or_fail(["cat", script], remote_prefix=remote_prefix) + + else: + try: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + except IOError as error: + raise cdist.Error(str(error)) + + raise cdist.Error("Command failed (shell): " + " ".join(*args)) + except OSError as error: + raise cdist.Error(" ".join(*args) + ": " + error.args[1]) + +def run_or_fail(*args, remote_prefix=False, **kargs): + if remote_prefix: + args[0][:0] = remote_prefix + + log.debug("Exec: " + " ".join(*args)) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + raise cdist.Error("Command failed: " + " ".join(*args)) + except OSError as error: + raise cdist.Error(" ".join(*args) + ": " + error.args[1]) diff --git a/bin/cdist-deploy-stdin-to b/lib/cdist/install.py old mode 100755 new mode 100644 similarity index 64% rename from bin/cdist-deploy-stdin-to rename to lib/cdist/install.py index 391dd431..98b388ec --- a/bin/cdist-deploy-stdin-to +++ b/lib/cdist/install.py @@ -1,6 +1,7 @@ -#!/bin/sh +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -18,19 +19,12 @@ # along with cdist. If not, see . # # -# Use stdin as the manifest to deploy on the given host. -# -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -eu +import logging -__cdist_target_host="$1" -shift +log = logging.getLogger(__name__) -cat >> "$__cdist_tmp_file" +def install(args): + """Install remote system""" + process = {} -chmod +x "$__cdist_tmp_file" - -export __cdist_manifest_init="$__cdist_tmp_file" -cdist-deploy-to "$__cdist_target_host" diff --git a/lib/cdist/path.py b/lib/cdist/path.py new file mode 100644 index 00000000..e416c42d --- /dev/null +++ b/lib/cdist/path.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import os +import shutil +import sys +import tempfile + +# Hardcoded paths usually not changable +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") +REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") + +DOT_CDIST = ".cdist" + +log = logging.getLogger(__name__) + +import cdist.exec + +def file_to_list(filename): + """Return list from \n seperated file""" + if os.path.isfile(filename): + file_fd = open(filename, "r") + lines = file_fd.readlines() + file_fd.close() + + # Remove \n from all lines + lines = map(lambda s: s.strip(), lines) + else: + lines = [] + + return lines + +class Path: + """Class that handles path related configurations""" + + def __init__(self, + target_host, + remote_user, + remote_prefix, + initial_manifest=False, + base_dir=None, + debug=False): + + # Base and Temp Base + if base_dir: + self.base_dir = base_dir + else: + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) + + self.temp_dir = tempfile.mkdtemp() + self.target_host = target_host + + self.remote_user = remote_user + self.remote_prefix = remote_prefix + + self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, target_host) + self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") + + self.out_dir = os.path.join(self.temp_dir, "out") + os.mkdir(self.out_dir) + + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + os.mkdir(self.global_explorer_out_dir) + + self.object_base_dir = os.path.join(self.out_dir, "object") + + # Setup binary directory + contents + self.bin_dir = os.path.join(self.out_dir, "bin") + os.mkdir(self.bin_dir) + + # List of type explorers transferred + self.type_explorers_transferred = {} + + # objects + self.objects_prepared = [] + + # Mostly static, but can be overwritten on user demand + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(self.manifest_dir, "init") + + def cleanup(self): + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # + log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) + # Remove previous cache + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + shutil.move(self.temp_dir, self.cache_dir) + + + def remote_mkdir(self, directory): + """Create directory on remote side""" + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix) + + def remove_remote_dir(self, destination): + cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix) + + def transfer_dir(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_dir(destination) + cdist.exec.run_or_fail(["scp", "-qr", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def transfer_file(self, source, destination): + """Transfer file""" + cdist.exec.run_or_fail(["scp", "-q", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def global_explorer_output_path(self, explorer): + """Returns path of the output for a global explorer""" + return os.path.join(self.global_explorer_out_dir, explorer) + + def type_explorer_output_dir(self, cdist_object): + """Returns and creates dir of the output for a type explorer""" + dir = os.path.join(self.object_dir(cdist_object), "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + + return dir + + def remote_global_explorer_path(self, explorer): + """Returns path to the remote explorer""" + return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + + def list_global_explorers(self): + """Return list of available explorers""" + return os.listdir(self.global_explorer_dir) + + def list_type_explorers(self, type): + """Return list of available explorers for a specific type""" + dir = self.type_dir(type, "explorer") + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + log.debug("Explorers for %s in %s: %s", type, dir, list) + + return list + + def list_types(self): + return os.listdir(self.type_base_dir) + + def list_object_paths(self, starting_point): + """Return list of paths of existing objects""" + object_paths = [] + + for content in os.listdir(starting_point): + full_path = os.path.join(starting_point, content) + if os.path.isdir(full_path): + object_paths.extend(self.list_object_paths(starting_point = full_path)) + + # Directory contains .cdist -> is an object + if content == DOT_CDIST: + object_paths.append(starting_point) + + return object_paths + + # FIXME + def get_type_from_object(self, cdist_object): + """Returns the first part (i.e. type) of an object""" + return cdist_object.split(os.sep)[0] + + def get_object_id_from_object(self, cdist_object): + """Returns everything but the first part (i.e. object_id) of an object""" + return os.sep.join(cdist_object.split(os.sep)[1:]) + + def object_dir(self, cdist_object): + """Returns the full path to the object (including .cdist)""" + return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + + def remote_object_dir(self, cdist_object): + """Returns the remote full path to the object (including .cdist)""" + return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + + def object_parameter_dir(self, cdist_object): + """Returns the dir to the object parameter""" + return os.path.join(self.object_dir(cdist_object), "parameter") + + def remote_object_parameter_dir(self, cdist_object): + """Returns the remote dir to the object parameter""" + return os.path.join(self.remote_object_dir(cdist_object), "parameter") + + def object_code_paths(self, cdist_object): + """Return paths to code scripts of object""" + return [os.path.join(self.object_dir(cdist_object), "code-local"), + os.path.join(self.object_dir(cdist_object), "code-remote")] + + def list_objects(self): + """Return list of existing objects""" + + objects = [] + if os.path.isdir(self.object_base_dir): + object_paths = self.list_object_paths(self.object_base_dir) + + for path in object_paths: + objects.append(os.path.relpath(path, self.object_base_dir)) + + return objects + + def type_dir(self, type, *args): + """Return directory the type""" + return os.path.join(self.type_base_dir, type, *args) + + def remote_type_explorer_dir(self, type): + """Return remote directory that holds the explorers of a type""" + return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + + def transfer_object_parameter(self, cdist_object): + """Transfer the object parameter to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) + + def transfer_global_explorers(self): + """Transfer the global explorers""" + self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) + self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + + def transfer_type_explorers(self, type): + """Transfer explorers of a type, but only once""" + if type in self.type_explorers_transferred: + log.debug("Skipping retransfer for explorers of %s", type) + return + else: + # Do not retransfer + self.type_explorers_transferred[type] = 1 + + src = self.type_dir(type, "explorer") + remote_base = os.path.join(REMOTE_TYPE_DIR, type) + dst = self.remote_type_explorer_dir(type) + + # Only continue, if there is at least the directory + if os.path.isdir(src): + # Ensure that the path exists + self.remote_mkdir(remote_base) + self.transfer_dir(src, dst) diff --git a/test.py b/test.py new file mode 100755 index 00000000..5d57fae4 --- /dev/null +++ b/test.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import os +import sys +import shutil +import subprocess +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib'))) + +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) + +cdist_commands=["banner", "config", "install"] + +import cdist +import cdist.config +import cdist.exec + +class Exec(unittest.TestCase): + def setUp(self): + """Create shell code and co.""" + + self.temp_dir = tempfile.mkdtemp() + self.shell_false = os.path.join(self.temp_dir, "shell_false") + self.shell_true = os.path.join(self.temp_dir, "shell_true") + + true_fd = open(self.shell_true, "w") + true_fd.writelines(["#!/bin/sh\n", "/bin/true"]) + true_fd.close() + + false_fd = open(self.shell_false, "w") + false_fd.writelines(["#!/bin/sh\n", "/bin/false"]) + false_fd.close() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_local_success_shell(self): + try: + cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail_shell(self): + self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail, + self.shell_false, [self.shell_false]) + + def test_local_success(self): + try: + cdist.exec.run_or_fail(["/bin/true"]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail(self): + self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) + +class Config(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.init_manifest = os.path.join(self.temp_dir, "manifest") + self.config = cdist.config.Config("localhost", + initial_manifest=self.init_manifest, + exec_path=cdist_exec_path) + self.config.link_emulator() + + def test_initial_manifest_different_parameter(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0700\n", + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_added(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + '\n', + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_removed(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + "\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_non_existent_command(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "thereisdefinitelynosuchcommend"]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_twice(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + try: + print("a") + self.config.run_initial_manifest() + print("b") + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + +class UI(unittest.TestCase): + def test_banner(self): + self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) + + def test_help(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) + + # FIXME: mockup needed + def test_config_localhost(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) + + +if __name__ == '__main__': + unittest.main()