diff --git a/build.sh b/build.sh index 021fb480..6a91ff3d 100755 --- a/build.sh +++ b/build.sh @@ -127,11 +127,18 @@ case "$1" in ;; test) - python3 -m unittest discover test 'test_*.py' + PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ + python3 -m cdist.test + ;; + + test-install) + PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ + python3 -m unittest cdist.test.test_install ;; test-all) - python3 -m unittest discover test '*.py' + PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ + python3 -m unittest discover lib/cdist/test '*.py' ;; *) diff --git a/conf/type/__mkfs/gencode-remote b/conf/type/__mkfs/gencode-remote new file mode 100755 index 00000000..b3561bad --- /dev/null +++ b/conf/type/__mkfs/gencode-remote @@ -0,0 +1,38 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +device="$(cat "$__object/parameter/device")" +type="$(cat "$__object/parameter/type")" + +if [ "$type" = "swap" ]; then + echo "mkswap $device" +else + command="mkfs -t $type -q" + if [ -f "$__object/parameter/options" ]; then + options="$(cat "$__object/parameter/options")" + command="$command $options" + fi + command="$command $device" + if [ -f "$__object/parameter/blocks" ]; then + blocks="$(cat "$__object/parameter/blocks")" + command="$command $blocks" + fi + echo "$command" +fi diff --git a/conf/type/__mkfs/install b/conf/type/__mkfs/install new file mode 100644 index 00000000..e69de29b diff --git a/conf/type/__mkfs/man.text b/conf/type/__mkfs/man.text new file mode 100644 index 00000000..4320c639 --- /dev/null +++ b/conf/type/__mkfs/man.text @@ -0,0 +1,57 @@ +cdist-type__mkfs(7) +=================== +Steven Armstrong + + +NAME +---- +cdist-type__mkfs - build a linux file system + + +DESCRIPTION +----------- +This cdist type is a wrapper for the mkfs command. + + +REQUIRED PARAMETERS +------------------- +type:: + The filesystem type to use. Same as mkfs -t. + + +OPTIONAL PARAMETERS +------------------- +device:: + defaults to object_id + +options:: + file system-specific options to be passed to the mkfs command + +blocks:: + the number of blocks to be used for the file system + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# reiserfs /dev/sda5 +__mkfs /dev/sda5 --type reiserfs +# same thing with explicit device +__mkfs whatever --device /dev/sda5 --type reiserfs + +# jfs with journal on /dev/sda2 +__mkfs /dev/sda1 --type jfs --options "-j /dev/sda2" +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- mkfs(8) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/conf/type/__mkfs/manifest b/conf/type/__mkfs/manifest new file mode 100755 index 00000000..e9d275a4 --- /dev/null +++ b/conf/type/__mkfs/manifest @@ -0,0 +1,31 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +if [ -f "$__object/parameter/device" ]; then + device="(cat "$__object/parameter/device")" +else + device="/$__object_id" + echo "$device" > "$__object/parameter/device" +fi + +type="(cat "$__object/parameter/type")" + +options="(cat "$__object/parameter/options")" diff --git a/conf/type/__mkfs/parameter/optional b/conf/type/__mkfs/parameter/optional new file mode 100644 index 00000000..86aeae30 --- /dev/null +++ b/conf/type/__mkfs/parameter/optional @@ -0,0 +1,3 @@ +device +options +blocks diff --git a/conf/type/__mkfs/parameter/required b/conf/type/__mkfs/parameter/required new file mode 100644 index 00000000..aa80e646 --- /dev/null +++ b/conf/type/__mkfs/parameter/required @@ -0,0 +1 @@ +type diff --git a/conf/type/__partition_msdos/install b/conf/type/__partition_msdos/install new file mode 100644 index 00000000..e69de29b diff --git a/conf/type/__partition_msdos_apply/files/lib.sh b/conf/type/__partition_msdos_apply/files/lib.sh index f0859aab..5767ea43 100644 --- a/conf/type/__partition_msdos_apply/files/lib.sh +++ b/conf/type/__partition_msdos_apply/files/lib.sh @@ -3,7 +3,8 @@ die() { exit 1 } debug() { - echo "[__partition_msdos_apply] $@" >&2 + #echo "[__partition_msdos_apply] $@" >&2 + : } fdisk_command() { @@ -51,7 +52,7 @@ create_partition() { first_minor="${minor}\n" type_minor="${minor}\n" primary_extended="l\n" - [ "$primary_count" > "3" ] && primary_extended="" + [ "$primary_count" -gt "3" ] && primary_extended="" fi [ -n "${size}" ] && size="+${size}M" fdisk_command ${device} "n\n${primary_extended}${first_minor}\n${size}\nt\n${type_minor}${type}\n" diff --git a/conf/type/__partition_msdos_apply/install b/conf/type/__partition_msdos_apply/install new file mode 100644 index 00000000..e69de29b diff --git a/doc/dev/logs/2011-10-04 b/doc/dev/logs/2011-10-04 new file mode 100644 index 00000000..f3bb852d --- /dev/null +++ b/doc/dev/logs/2011-10-04 @@ -0,0 +1,3 @@ +Testing for single tests: + PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib python3 -m unittest cdist.test.test_install.Install.test_explorer_ran + diff --git a/doc/dev/logs/2011-10-05 b/doc/dev/logs/2011-10-05 new file mode 100644 index 00000000..39fc48a2 --- /dev/null +++ b/doc/dev/logs/2011-10-05 @@ -0,0 +1,116 @@ +Config/Install/Deploy/Run: + target host + remote_cmd_prefix - ssh user@bla sudo foo????? + remote_cp_prefix - cp statt scp oder so + + debug -> env für alles += __debug + + +Storage/Metaobject/Tree? == Path? + base_dir? + nimmt objekte + + Sammelt Objekte + + Ist prepared hier? + +Object + "Infos" / Datenhalde + + Base_Dir-Abhängigkeit? - wo + + out_dir - wo speichern + + nur eigenes verzeichnis interessant? + -> nicht für shell code / aka gencode! + -> __global abhängigkeit + + object.gencode()? + + hast du type-explorer? + ja? + führe JEDEN remote aus + speichere ausgabe in object + nein: + fertig + hast du gencode-{local,remote}? + ja? + führe local oder remote aus + speichere ausgabe in s/^gen// + nein: + fertig + + hast du code-{local,remote}? + ja? + führe local oder remote aus + nein: + fertig + + ich habe ... + object_id + type + type.singleton() == False -> require object_id + parameter gegeben + requirements / order + + type_explorer := methode zum ausführen? + + cdist.object.Object(type, id) + + methoden: + gen_code + code + run_manifest + manifest == ort + +Type + singleton: ja / nein + install: ja / nein + type_explorer := liste + + optional_parameter + required_parameter + + TypeExplorer + verwandt oder == explorer + Verwandschaft klären! + + sehr abhängig von base_dir! + - welche gibt es? + - was für optionen haben sie + + cdist.type.Type("/path/to/type") + Tree/Path vieh, das liste von $_ speichert + Einfach iterieren + + + +Explorer + execute(env) + env == __explorer -> nur im explorer + +z.B. BaseExplorer oder andersherum GlobalExplorer + +Manifest + +Exec + wrapper um auszuführen, + error handling, + output redirection (variable, file, beides, socat :-) + + +-------------------------------------------------------------------------------- + +- base_dir (conf/, type, ...) +- manifest (initiale) + $methode_mit_inhalt_von_manifest? + run_manifest(code) + ob sinnvoll? + geht auch mit stdin oder datei + + stdin -> muss in tmp-datei, für sh -e? +- +-------------------------------------------------------------------------------- + +save output of shell in buffer instead of displaying? + -> freedom to decide whether to display or not! diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index a0ca2ba2..864b4f37 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -24,3 +24,13 @@ VERSION = "2.0.3" class Error(Exception): """Base exception class for this project""" pass + + +class MissingEnvironmentVariableError(Error): + """Raised when a required environment variable is not set.""" + + def __init__(self, name) + self.name = name + + def __str__(self): + return 'Missing required environment variable: {0.name}'.format(o) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index c027da17..657714a4 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -22,10 +22,6 @@ import datetime import logging -import os -import stat -import sys - log = logging.getLogger(__name__) import cdist.emulator @@ -280,10 +276,8 @@ class Config: self.target_host, duration.total_seconds()) - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() +class Config(cdist.config_install.ConfigInstall): + pass def config(args): """Configure remote system""" diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py new file mode 100644 index 00000000..9a84c2cf --- /dev/null +++ b/lib/cdist/config_install.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import datetime +import logging +import os +import stat +import sys + +import cdist.emulator +import cdist.path + +log = logging.getLogger(__name__) + +CODE_HEADER = "#!/bin/sh -e\n" + +class ConfigInstall: + """Class to hold install and config methods""" + + def __init__(self, target_host, + initial_manifest=False, + remote_user="root", + home=None, + exec_path=sys.argv[0], + debug=False): + + self.target_host = target_host + self.debug = debug + self.remote_user = remote_user + self.exec_path = exec_path + + # FIXME: broken - construct elsewhere! + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + + self.path = cdist.path.Path(self.target_host, + initial_manifest=initial_manifest, + remote_user=self.remote_user, + remote_prefix=self.remote_prefix, + base_dir=home, + debug=debug) + + self.objects_prepared = [] + + def cleanup(self): + self.path.cleanup() + + def run_global_explorers(self): + """Run global explorers""" + log.info("Running global explorers") + explorers = self.path.list_global_explorers() + if(len(explorers) == 0): + raise CdistError("No explorers found in", self.path.global_explorer_dir) + + self.path.transfer_global_explorers() + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.path.remote_global_explorer_path(explorer)) + + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + +# FIXME: where to call this from? + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + + type = self.path.get_type_from_object(cdist_object) + self.path.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.path.transfer_object_parameter(cdist_object) + + # FIXME: Broken due to refactoring into type.py + explorers = self.path.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + self.link_emulator() + + def run_initial_manifest(self): + """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) + env = { "__manifest" : self.path.manifest_dir } + self.run_manifest(self.path.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.path.get_type_from_object(cdist_object) + manifest = self.path.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.path.object_dir(cdist_object), + "__object_id": self.path.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.path.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.path.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + + # Submit debug flag to manifest, can be used by emulator and types + if self.debug: + env['__debug'] = "yes" + + # Required for recording source + env['__cdist_manifest'] = manifest + + # Required to find types + env['__cdist_type_base_dir'] = self.path.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.path.object_dir(cdist_object), "require") + requirements = cdist.path.file_to_list(file) + type = self.path.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + env["__object"] = self.path.object_dir(cdist_object) + env["__object_id"] = self.path.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.path.type_dir(type) + + if mode == "gencode": + paths = [ + self.path.type_dir(type, "gencode-local"), + self.path.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.path.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + # FIXME: CODE_HEADER needed in our sh -e scenario? + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.path.object_dir(cdist_object) + remote_dir = self.path.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + cdist.exec.run_or_fail([bin]) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.path.transfer_file(local_remote_code, remote_remote_code) + # FIXME: remote_prefix + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explorers() + self.run_initial_manifest() + + log.info("Running object manifests and type explorers") + + old_objects = [] + objects = self.path.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + # FIXME: run_type_explorer: + # object can return type + # type has explorers + # path knows about where to save explorer output + # type = self.path.objects[object].type() + # self.path.types['type'].explorers() + # for explorer in explorers: + # output = cdist.exec.run_debug_or_fail_shell(explorer) + # if output: + # write_output_to(output, os.path.join(self.path.objects[object].explorer_dir(),explorer) ) + # + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.path.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.info("Generating and executing code") + # Now do the final steps over the existing objects + for cdist_object in self.path.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() diff --git a/lib/cdist/core/__init__.py b/lib/cdist/core/__init__.py new file mode 100644 index 00000000..80310ffc --- /dev/null +++ b/lib/cdist/core/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +__all__ = ['Type', 'Object'] + +from cdist.core.type import Type +from cdist.core.object import Object diff --git a/lib/cdist/core/object.py b/lib/cdist/core/object.py new file mode 100644 index 00000000..80c2c351 --- /dev/null +++ b/lib/cdist/core/object.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os + +import cdist +import cdist.path + + +class Object(object): + """Represents a cdist object. + + All interaction with objects in cdist should be done through this class. + Directly accessing an object through the file system from python code is + a bug. + + """ + + @staticmethod + def base_dir(): + """Return the absolute path to the top level directory where objects + are defined. + + Requires the environment variable '__cdist_out_dir' to be set. + + """ + try: + base_dir = os.path.join( + os.environ['__cdist_out_dir'], + 'object' + ) + except KeyError as e: + raise cdist.MissingEnvironmentVariableError(e.args[0]) + + # FIXME: should directory be created elsewhere? + if not os.path.isdir(base_dir): + os.mkdir(base_dir) + return base_dir + + @classmethod + def list_objects(cls): + """Return a list of object instances""" + for object_name in cls.list_object_names(): + type_name = object_name.split(os.sep)[0] + object_id = os.sep.join(object_name.split(os.sep)[1:]) + yield cls(Type(type_name), object_id=object_id) + + @classmethod + def list_type_names(cls): + """Return a list of type names""" + return os.listdir(cls.base_dir()) + + @classmethod + def list_object_names(cls): + """Return a list of object names""" + for path, dirs, files in os.walk(cls.base_dir()): + # FIXME: use constant instead of string + if cdist.path.DOT_CDIST in dirs: + yield os.path.relpath(path, cls.base_dir()) + + def __init__(self, type, object_id=None, parameter=None, requirements=None): + self.type = type # instance of Type + self.object_id = object_id + self.qualified_name = os.path.join(self.type.name, self.object_id) + self.parameter = parameter or {} + self.requirements = requirements or [] + + def __repr__(self): + return '' % self.qualified_name + + @property + def path(self): + return os.path.join( + self.base_dir(), + self.qualified_name, + cdist.path.DOT_CDIST + ) + + @property + def changed(self): + """Check whether the object has been changed.""" + return os.path.isfile(os.path.join(self.path, "changed")) + + @changed.setter + def changed(self, value): + """Change the objects changed status.""" + path = os.path.join(self.path, "changed") + if value: + open(path, "w").close() + else: + try: + os.remove(path) + except EnvironmentError: + # ignore + pass + + # FIXME: implement other properties/methods diff --git a/lib/cdist/core/type.py b/lib/cdist/core/type.py new file mode 100644 index 00000000..c35e0ad8 --- /dev/null +++ b/lib/cdist/core/type.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os + +import cdist + + +class Type(object): + + @staticmethod + def base_dir(): + """Return the absolute path to the top level directory where types + are defined. + + Requires the environment variable '__cdist_base_dir' to be set. + + """ + try: + return os.path.join( + os.environ['__cdist_base_dir'], + 'conf', + 'type' + ) + except KeyError as e: + raise cdist.MissingEnvironmentVariableError(e.args[0]) + + @classmethod + def list_types(cls): + """Return a list of type instances""" + for type_name in cls.list_type_names(): + yield cls(type_name) + + @classmethod + def list_type_names(cls): + """Return a list of type names""" + return os.listdir(cls.base_dir()) + + + def __init__(self, name): + self.name = name + self.__explorers = None + self.__required_parameters = None + self.__optional_parameters = None + + def __repr__(self): + return '' % self.name + + @property + def path(self): + return os.path.join( + self.base_dir(), + self.name + ) + + @property + def is_singleton(self): + """Check whether a type is a singleton.""" + return os.path.isfile(os.path.join(self.path, "singleton")) + + @property + def is_install(self): + """Check whether a type is used for installation (if not: for configuration)""" + return os.path.isfile(os.path.join(self.path, "install")) + + @property + def explorers(self): + """Return a list of available explorers""" + if not self.__explorers: + try: + self.__explorers = os.listdir(os.path.join(self.path, "explorer")) + except EnvironmentError as e: + # error ignored + self.__explorers = [] + return self.__explorers + + @property + def required_parameters(self): + """Return a list of required parameters""" + if not self.__required_parameters: + parameters = [] + try: + with open(os.path.join(self.path, "parameter", "required")) as fd: + for line in fd: + parameters.append(line.strip()) + except EnvironmentError as e: + # error ignored + pass + finally: + self.__required_parameters = parameters + return self.__required_parameters + + @property + def optional_parameters(self): + """Return a list of optional parameters""" + if not self.__optional_parameters: + parameters = [] + try: + with open(os.path.join(self.path, "parameter", "optional")) as fd: + for line in fd: + parameters.append(line.strip()) + except EnvironmentError as e: + # error ignored + pass + finally: + self.__optional_parameters = parameters + return self.__optional_parameters diff --git a/lib/cdist/install.py b/lib/cdist/install.py index 98b388ec..5a35626d 100644 --- a/lib/cdist/install.py +++ b/lib/cdist/install.py @@ -22,8 +22,14 @@ import logging +import cdist.config_install + log = logging.getLogger(__name__) + +Class Install(cdist.config_install.ConfigInstall): + pass + def install(args): """Install remote system""" process = {} diff --git a/lib/cdist/object.py b/lib/cdist/object.py new file mode 100644 index 00000000..0a282dc2 --- /dev/null +++ b/lib/cdist/object.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import logging +log = logging.getLogger(__name__) + + + +class Object(object): + + def __init__(self, path, remote_path, object_fq): + self.path = path + self.remote_path = remote_path + self.object_fq = object_fq + self.type = self.object_fq.split(os.sep)[0] + self.object_id = self.object_fq.split(os.sep)[1:] + self.parameter_dir = os.path.join(self.path, "parameter") + self.remote_object_parameter_dir = os.path.join(self.remote_path, "parameter") + self.object_code_paths = [ + os.path.join(self.path, "code-local"), + os.path.join(self.path, "code-remote")] + + @property + def type_explorer_output_dir(self): + """Returns and creates dir of the output for a type explorer""" + if not self.__type_explorer_output_dir: + dir = os.path.join(self.path, "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + self.__type_explorer_output_dir = dir + return self.__type_explorer_output_dir + diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 85d1a87f..8224d0c0 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -70,6 +70,7 @@ class Path: self.temp_dir = tempfile.mkdtemp() self.target_host = target_host + # Input directories self.conf_dir = os.path.join(self.base_dir, "conf") self.cache_base_dir = os.path.join(self.base_dir, "cache") self.cache_dir = os.path.join(self.cache_base_dir, target_host) @@ -78,30 +79,27 @@ class Path: self.manifest_dir = os.path.join(self.conf_dir, "manifest") self.type_base_dir = os.path.join(self.conf_dir, "type") - self.out_dir = os.path.join(self.temp_dir, "out") - os.mkdir(self.out_dir) - - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") - os.mkdir(self.global_explorer_out_dir) - - self.object_base_dir = os.path.join(self.out_dir, "object") - - # Setup binary directory + contents - self.bin_dir = os.path.join(self.out_dir, "bin") - os.mkdir(self.bin_dir) - - # List of type explorers transferred - self.type_explorers_transferred = {} - - # objects - self.objects_prepared = [] - # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest else: self.initial_manifest = os.path.join(self.manifest_dir, "init") + # Output directories + self.out_dir = os.path.join(self.temp_dir, "out") + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + self.object_base_dir = os.path.join(self.out_dir, "object") + self.bin_dir = os.path.join(self.out_dir, "bin") + + # List of type explorers transferred + self.type_explorers_transferred = {} + + # objects prepared + self.objects_prepared = [] + + # Create directories + self.__init_out_dirs() + def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization @@ -114,29 +112,49 @@ class Path: shutil.rmtree(self.cache_dir) shutil.move(self.temp_dir, self.cache_dir) + + def __init_out_dirs(self): + """Initialise output directory structure""" + os.mkdir(self.out_dir) + os.mkdir(self.global_explorer_out_dir) + os.mkdir(self.bin_dir) + + # Stays here + def list_types(self): + """Retuns list of types""" + return os.listdir(self.type_base_dir) + + ###################################################################### + + # FIXME: belongs to here - clearify remote* def remote_mkdir(self, directory): """Create directory on remote side""" cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) + # FIXME: belongs to here - clearify remote* def remove_remote_dir(self, destination): cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) + # FIXME: belongs to here - clearify remote* def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + ["-r", source, self.target_host + ":" + destination]) + # FIXME: belongs to here - clearify remote* def transfer_file(self, source, destination): """Transfer file""" cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + [source, self.target_host + ":" + destination]) + # FIXME: Explorer or stays def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) + # FIXME: object def type_explorer_output_dir(self, cdist_object): """Returns and creates dir of the output for a type explorer""" dir = os.path.join(self.object_dir(cdist_object), "explorer") @@ -145,29 +163,17 @@ class Path: return dir + # FIXME Stays here / Explorer? def remote_global_explorer_path(self, explorer): """Returns path to the remote explorer""" return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + # FIXME: stays here def list_global_explorers(self): """Return list of available explorers""" return os.listdir(self.global_explorer_dir) - def list_type_explorers(self, type): - """Return list of available explorers for a specific type""" - dir = self.type_dir(type, "explorer") - if os.path.isdir(dir): - list = os.listdir(dir) - else: - list = [] - - log.debug("Explorers for %s in %s: %s", type, dir, list) - - return list - - def list_types(self): - return os.listdir(self.type_base_dir) - + # Stays here def list_object_paths(self, starting_point): """Return list of paths of existing objects""" object_paths = [] @@ -183,36 +189,43 @@ class Path: return object_paths - # FIXME + # FIXME: Object def get_type_from_object(self, cdist_object): """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] + # FIXME: Object def get_object_id_from_object(self, cdist_object): """Returns everything but the first part (i.e. object_id) of an object""" return os.sep.join(cdist_object.split(os.sep)[1:]) + # FIXME: Object def object_dir(self, cdist_object): """Returns the full path to the object (including .cdist)""" return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + # FIXME: Object def remote_object_dir(self, cdist_object): """Returns the remote full path to the object (including .cdist)""" return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + # FIXME: Object def object_parameter_dir(self, cdist_object): """Returns the dir to the object parameter""" return os.path.join(self.object_dir(cdist_object), "parameter") + # FIXME: object def remote_object_parameter_dir(self, cdist_object): """Returns the remote dir to the object parameter""" return os.path.join(self.remote_object_dir(cdist_object), "parameter") + # FIXME: object def object_code_paths(self, cdist_object): """Return paths to code scripts of object""" return [os.path.join(self.object_dir(cdist_object), "code-local"), os.path.join(self.object_dir(cdist_object), "code-remote")] + # Stays here def list_objects(self): """Return list of existing objects""" @@ -225,14 +238,7 @@ class Path: return objects - def type_dir(self, type, *args): - """Return directory the type""" - return os.path.join(self.type_base_dir, type, *args) - - def remote_type_explorer_dir(self, type): - """Return remote directory that holds the explorers of a type""" - return os.path.join(REMOTE_TYPE_DIR, type, "explorer") - + # Stays here def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" # Create base path before using mkdir -p @@ -242,11 +248,13 @@ class Path: self.transfer_dir(self.object_parameter_dir(cdist_object), self.remote_object_parameter_dir(cdist_object)) + # Stays here def transfer_global_explorers(self): """Transfer the global explorers""" self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + # Stays here - FIXME: adjust to type code, loop over types! def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" if type in self.type_explorers_transferred: diff --git a/lib/cdist/test/__init__.py b/lib/cdist/test/__init__.py new file mode 100644 index 00000000..f614fa05 --- /dev/null +++ b/lib/cdist/test/__init__.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import os +import subprocess +import unittest + +cdist_commands=["banner", "config", "install"] + +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../bin/cdist")) + +def exec(): + print(cdist_exec_path) + +#class UI(unittest.TestCase): +# def test_banner(self): +# self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) +# +# def test_help(self): +# for cmd in cdist_commands: +# self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) +# +# # FIXME: mockup needed +# def test_config_localhost(self): +# for cmd in cdist_commands: +# self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) diff --git a/lib/cdist/test/__main__.py b/lib/cdist/test/__main__.py new file mode 100644 index 00000000..3b31a2cd --- /dev/null +++ b/lib/cdist/test/__main__.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import os +import sys +import cdist.test +import unittest + +#class UI(unittest.TestCase): +# def test_banner(self): +# self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) +# +# def test_help(self): +# for cmd in cdist_commands: +# self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) +# +# # FIXME: mockup needed +# def test_config_localhost(self): +# for cmd in cdist_commands: +# self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) + +print(cdist.test.cdist_exec_path) +print(sys.argv) + +suite = unittest.defaultTestLoader.discover(os.path.dirname(__file__)) +unittest.TextTestRunner(verbosity=1).run(suite) diff --git a/test/nico_ui.py b/lib/cdist/test/nico_ui.py old mode 100755 new mode 100644 similarity index 100% rename from test/nico_ui.py rename to lib/cdist/test/nico_ui.py diff --git a/test/test_config.py b/lib/cdist/test/test_config.py similarity index 100% rename from test/test_config.py rename to lib/cdist/test/test_config.py diff --git a/test/test_exec.py b/lib/cdist/test/test_exec.py old mode 100755 new mode 100644 similarity index 100% rename from test/test_exec.py rename to lib/cdist/test/test_exec.py diff --git a/lib/cdist/test/test_install.py b/lib/cdist/test/test_install.py new file mode 100644 index 00000000..9cfae066 --- /dev/null +++ b/lib/cdist/test/test_install.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) + +import cdist.config + +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) + + +class Install(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.init_manifest = os.path.join(self.temp_dir, "manifest") + self.config = cdist.config.Config("localhost", + initial_manifest=self.init_manifest, + exec_path=cdist_exec_path) + self.config.link_emulator() + +### NEW FOR INSTALL ############################################################ + + def test_explorer_ran(self): + """Check that all explorers returned a result""" + self.config.run_global_explores() + explorers = self.config.path.list_global_explorers() + + for explorer in explorers: + output = self.config.path.global_explorer_output_path(explorer) + self.assertTrue(os.path.isfile(output)) + + def test_manifest_uses_install_types_only(self): + """Check that objects created from manifest are only of install type""" + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0700\n", + "__partition_msdos /dev/null --type 82\n", + ]) + manifest_fd.close() + + self.config.run_initial_manifest() + + # FIXME: check that only __partition_msdos objects are created! + + self.assertFalse(failed) + + +### OLD FROM CONFIG ############################################################ + def test_initial_manifest_different_parameter(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0700\n", + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_added(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + '\n', + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_removed(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + "\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_non_existent_command(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "thereisdefinitelynosuchcommend"]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_twice(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + try: + self.config.run_initial_manifest() + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + diff --git a/lib/cdist/test/test_path.py b/lib/cdist/test/test_path.py new file mode 100644 index 00000000..f86c8fad --- /dev/null +++ b/lib/cdist/test/test_path.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import shutil +import sys +import tempfile +import unittest + +import cdist.path +import cdist.test + +class Path(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.init_manifest = os.path.join(self.temp_dir, "manifest") + self.path = cdist.path.Path("localhost", "root", "ssh root@localhost", + initial_manifest=self.init_manifest, + base_dir=self.temp_dir) + + os.mkdir(self.path.conf_dir) + os.mkdir(self.path.type_base_dir) + + self.install_type_name = "__install_test" + self.config_type_name = "__config_test" + + # Create install type + self.install_type = os.path.join(self.path.type_base_dir, self.install_type_name) + os.mkdir(self.install_type) + open(os.path.join(self.install_type, "install"), "w").close() + + # Create config type + self.config_type = os.path.join(self.path.type_base_dir, self.config_type_name) + os.mkdir(self.config_type) + + def tearDown(self): + self.path.cleanup() + shutil.rmtree(self.temp_dir) + + def test_type_detection(self): + """Check that a type is identified as install or configuration correctly""" + + self.assertTrue(self.path.is_install_type(self.install_type)) + self.assertFalse(self.path.is_install_type(self.config_type)) + + def test_manifest_uses_install_types_only(self): + """Check that objects created from manifest are only of install type""" + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + self.install_type_name + "testid\n", + self.config_type_name + "testid\n", + ]) + manifest_fd.close() + + self.install.run_initial_manifest() + + # FIXME: check that only __partition_msdos objects are created! + + self.assertFalse(failed) diff --git a/lib/cdist/type.py b/lib/cdist/type.py new file mode 100644 index 00000000..e1c5f589 --- /dev/null +++ b/lib/cdist/type.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import os +log = logging.getLogger(__name__) + +class Type(object): + + def __init__(self, path, remote_path): + self.path = path + self.remote_path = remote_path + + def list_explorers(self): + """Return list of available explorers""" + dir = os.path.join(self.path, "explorer") + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + log.debug("Explorers for %s in %s: %s", type, dir, list) + + return list + + def is_install(self): + """Check whether a type is used for installation (if not: for configuration)""" + return os.path.isfile(os.path.join(self.path, "install")) + + def remote_explorer_dir(self): + """Return remote directory that holds the explorers of a type""" + return os.path.join(self.remote_path, "explorer")