From 5d256b21b1e844026426f4ab22207952ecf35b7f Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Sun, 2 Oct 2011 22:33:56 +0200 Subject: [PATCH 01/54] Initial init_script type --- conf/type/__init_script/gencode-remote | 30 ++++++++++++++++++++++ conf/type/__init_script/parameter/optional | 1 + conf/type/__init_script/parameter/required | 1 + 3 files changed, 32 insertions(+) create mode 100644 conf/type/__init_script/gencode-remote create mode 100644 conf/type/__init_script/parameter/optional create mode 100644 conf/type/__init_script/parameter/required diff --git a/conf/type/__init_script/gencode-remote b/conf/type/__init_script/gencode-remote new file mode 100644 index 00000000..d9e56970 --- /dev/null +++ b/conf/type/__init_script/gencode-remote @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2010-2011 Daniel Roth (dani-cdist@d-roth.li) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +if [ -f "$__object/parameter/service" ]; then + service=$(cat "$__object/parameter/service") +else + service="/$__object_id" +fi + +mode=$(cat "$__object/parameter/mode") + +echo "/etc/init.d/${service} ${mode}" diff --git a/conf/type/__init_script/parameter/optional b/conf/type/__init_script/parameter/optional new file mode 100644 index 00000000..24e10984 --- /dev/null +++ b/conf/type/__init_script/parameter/optional @@ -0,0 +1 @@ +service diff --git a/conf/type/__init_script/parameter/required b/conf/type/__init_script/parameter/required new file mode 100644 index 00000000..17ab372f --- /dev/null +++ b/conf/type/__init_script/parameter/required @@ -0,0 +1 @@ +mode From 5f1afb08f6387111a754fef0bbdc1e7abe093257 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 3 Oct 2011 17:33:19 +0200 Subject: [PATCH 02/54] man page --- conf/type/__init_script/man.text | 49 ++++++++++++++++++++++ conf/type/__init_script/parameter/optional | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 conf/type/__init_script/man.text diff --git a/conf/type/__init_script/man.text b/conf/type/__init_script/man.text new file mode 100644 index 00000000..34065260 --- /dev/null +++ b/conf/type/__init_script/man.text @@ -0,0 +1,49 @@ +cdist-type__init_script(7) +============================== +Daniel Roth + + +NAME +---- +cdist-type__init_script - Use the init scripts + + +DESCRIPTION +----------- +This type can be used to control your init scripts. + + +REQUIRED PARAMETERS +------------------- +mode:: + Specifies what shall be done with the init script (usually one of 'start'|'stop'|'restart'|'reload' or 'force-reload') + + +OPTIONAL PARAMETERS +------------------- +script:: + If supplied, use this as the init-script. + Otherwise the object_id is used. The script will be pretended with '/etc/init.d/' + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Reloads the configuration for lighttpd +__init_script lighttpd --mode force-reload + +# Reloads the configuration for lighttpd +__init_script lighty --script lighttpd --mode force-reload +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Daniel Roth. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/conf/type/__init_script/parameter/optional b/conf/type/__init_script/parameter/optional index 24e10984..84f7e31d 100644 --- a/conf/type/__init_script/parameter/optional +++ b/conf/type/__init_script/parameter/optional @@ -1 +1 @@ -service +script From 342cbca533a5da451555769d55cd83b7cb0959c1 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 3 Oct 2011 18:00:07 +0200 Subject: [PATCH 03/54] additional parameter base_dir --- conf/type/__init_script/gencode-remote | 16 ++++++++++++---- conf/type/__init_script/man.text | 4 +++- conf/type/__init_script/parameter/optional | 1 + 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/conf/type/__init_script/gencode-remote b/conf/type/__init_script/gencode-remote index d9e56970..9b493cfc 100644 --- a/conf/type/__init_script/gencode-remote +++ b/conf/type/__init_script/gencode-remote @@ -19,12 +19,20 @@ # # -if [ -f "$__object/parameter/service" ]; then - service=$(cat "$__object/parameter/service") +if [ -f "$__object/parameter/script" ]; then + script=$(cat "$__object/parameter/script") else - service="/$__object_id" + script="/$__object_id" fi +if [ -f "$__object/parameter/base_dir" ]; then + base_dir=$(cat "$__object/parameter/base_dir") +else + base_dir="/etc/init.d" +fi + + + mode=$(cat "$__object/parameter/mode") -echo "/etc/init.d/${service} ${mode}" +echo "${base_dir}/${script} ${mode}" diff --git a/conf/type/__init_script/man.text b/conf/type/__init_script/man.text index 34065260..898943a5 100644 --- a/conf/type/__init_script/man.text +++ b/conf/type/__init_script/man.text @@ -23,8 +23,10 @@ OPTIONAL PARAMETERS ------------------- script:: If supplied, use this as the init-script. - Otherwise the object_id is used. The script will be pretended with '/etc/init.d/' + Otherwise the object_id is used. +base_dir:: + If supplied, this type uses this directory instead of '/etc/init.d'. The parameter will not need an ending slash. EXAMPLES -------- diff --git a/conf/type/__init_script/parameter/optional b/conf/type/__init_script/parameter/optional index 84f7e31d..5551a8f2 100644 --- a/conf/type/__init_script/parameter/optional +++ b/conf/type/__init_script/parameter/optional @@ -1 +1,2 @@ script +base_dir From 2147480fcd418c21b3a539e622d9ad46641f6a6f Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 3 Oct 2011 18:35:45 +0200 Subject: [PATCH 04/54] base_dir is /etc/rc.d for archlinux /etc/init.d otherwise --- conf/type/__init_script/gencode-remote | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/conf/type/__init_script/gencode-remote b/conf/type/__init_script/gencode-remote index 9b493cfc..ff823cad 100644 --- a/conf/type/__init_script/gencode-remote +++ b/conf/type/__init_script/gencode-remote @@ -28,11 +28,13 @@ fi if [ -f "$__object/parameter/base_dir" ]; then base_dir=$(cat "$__object/parameter/base_dir") else - base_dir="/etc/init.d" + os="$(cat "$__global/explorer/os")" + case "$os" in + archlinux) base_dir="/etc/rc.d" ;; + *) base_dir="/etc/init.d" + esac fi - - mode=$(cat "$__object/parameter/mode") echo "${base_dir}/${script} ${mode}" From 40d5b9cb05f93a8fc770fd595aadc8d9cd0ce40c Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Mon, 3 Oct 2011 21:39:07 +0200 Subject: [PATCH 05/54] bsd init location --- conf/type/__init_script/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__init_script/gencode-remote b/conf/type/__init_script/gencode-remote index ff823cad..d212feb7 100644 --- a/conf/type/__init_script/gencode-remote +++ b/conf/type/__init_script/gencode-remote @@ -30,7 +30,7 @@ if [ -f "$__object/parameter/base_dir" ]; then else os="$(cat "$__global/explorer/os")" case "$os" in - archlinux) base_dir="/etc/rc.d" ;; + archlinux|netbsd|macosx|freebsd|openbsd) base_dir="/etc/rc.d" ;; *) base_dir="/etc/init.d" esac fi From 9dfd6a27ad2b0e771e03d633a20248b2c7a0e102 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Wed, 5 Oct 2011 09:15:20 +0200 Subject: [PATCH 06/54] fixed broken man page --- conf/type/__init_script/man.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__init_script/man.text b/conf/type/__init_script/man.text index 898943a5..c33ff7b7 100644 --- a/conf/type/__init_script/man.text +++ b/conf/type/__init_script/man.text @@ -1,5 +1,5 @@ cdist-type__init_script(7) -============================== +========================== Daniel Roth From 820eea56fb5a2cde0784eadedf6bf5710061a534 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 09:27:16 +0200 Subject: [PATCH 07/54] install finish Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 2 +- lib/cdist/type.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 9a84c2cf..39ee79c6 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -68,7 +68,7 @@ class ConfigInstall: log.info("Running global explorers") explorers = self.path.list_global_explorers() if(len(explorers) == 0): - raise CdistError("No explorers found in", self.path.global_explorer_dir) + raise CdistError("No explorers found in ", self.path.global_explorer_dir) self.path.transfer_global_explorers() for explorer in explorers: diff --git a/lib/cdist/type.py b/lib/cdist/type.py index e1c5f589..e6c35ad1 100644 --- a/lib/cdist/type.py +++ b/lib/cdist/type.py @@ -46,6 +46,10 @@ class Type(object): """Check whether a type is used for installation (if not: for configuration)""" return os.path.isfile(os.path.join(self.path, "install")) + def explorer_dir(self): + """Return remote directory that holds the explorers of a type""" + return os.path.join(self.remote_path, "explorer") + def remote_explorer_dir(self): """Return remote directory that holds the explorers of a type""" return os.path.join(self.remote_path, "explorer") From 5b70ff5694bfa18ad745987d9aee7c32a131eeb3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 12:07:50 +0200 Subject: [PATCH 08/54] whiteboard to file Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-10-06 | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 doc/dev/logs/2011-10-06 diff --git a/doc/dev/logs/2011-10-06 b/doc/dev/logs/2011-10-06 new file mode 100644 index 00000000..14edab51 --- /dev/null +++ b/doc/dev/logs/2011-10-06 @@ -0,0 +1,30 @@ +GlobalExplorer + list_explorers() + list_explorers_names() + + base_dir + __init__(name) + out_dir + env + name = id + path + return_code + return_value + +-------------------------------------------------------------------------------- +Exec: + +normal: + +scp /from/where $USER@$HOST:REMOTE_BASE/cdist-internal +ssh $USER@$HOST MY_CMD_THAT_NEEDS_TO_RUN_IN_BIN_SH (including ENV) + +sudo: + +scp $USER@$HOST:REMOTE_BASE/cdist-internal +ssh $USER@$HOST sudo MY_CMD_THAT_NEEDS_TO_RUN_IN_BIN_SH (including ENV) + +chroot: + +[sudo] cp file /chroot/THE_HOST_BASE/REMOTE_BASE/cdist-internal +[sudo] chroot /chroot MY_CMD_THAT_NEEDS_TO_RUN_IN_BIN_SH (including ENV) From 2ec2ab26ce95b967a82577409e42fdf745adba41 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 12:34:34 +0200 Subject: [PATCH 09/54] update path with changes from yesterday (system crash) Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 5a2b64d2..2fcf3aea 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -117,7 +117,7 @@ class Path: shutil.rmtree(self.cache_dir) shutil.move(self.temp_dir, self.cache_dir) - + def __init_out_dirs(self): """Initialise output directory structure""" os.mkdir(self.out_dir) From aa1c13898aa6b66cb833036af43aef7facb0d363 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 13:00:00 +0200 Subject: [PATCH 10/54] add ideas about remote exec/copy Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot | 45 +++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot diff --git a/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot b/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot new file mode 100644 index 00000000..1dd9039d --- /dev/null +++ b/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot @@ -0,0 +1,45 @@ +Commands needed: + conf/cmd/remote_exec + conf/cmd/copy + +If ! conf/cmd/remote_exec: + use builtin +If ! conf/cmd/copy: + use builtin + + -> Depend on session! + +Builtin: + cdist.exec.run_or_fail(["scp", "-qr", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + +self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + + self.remote_user = remote_user + self.remote_prefix = remote_prefix + +-------------------------------------------------------------------------------- +What is in a session? + + base_dir + target_host + +-------------------------------------------------------------------------------- +remote_user + pseudo-static, can be hardcoded again +-------------------------------------------------------------------------------- + +Result: + +os.environ['__remote_exec'] = ["ssh", "-l", "root" ] +os.environ['__remote_copy'] = ["scp", "-o" "User=root" ] + +args for __remote_exec + $1 = hostname + $2 - ... = stuff to be executed in /bin/sh on remote side + +args for __remote_copy + $1 = file here + $2 = hostname:destination From c0f04cab0f5e9ed0d0b70114111f46c6ad64dc90 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 13:20:05 +0200 Subject: [PATCH 11/54] more hints on env Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot | 22 ++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot b/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot index 1dd9039d..91f5d204 100644 --- a/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot +++ b/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot @@ -7,6 +7,12 @@ If ! conf/cmd/remote_exec: If ! conf/cmd/copy: use builtin +-------------------------------------------------------------------------------- + +--cmd-dir? +$__cdist_cmd_dir + +-------------------------------------------------------------------------------- -> Depend on session! Builtin: @@ -34,12 +40,26 @@ remote_user Result: os.environ['__remote_exec'] = ["ssh", "-l", "root" ] -os.environ['__remote_copy'] = ["scp", "-o" "User=root" ] + +os.environ['__remote_exec'] = ["ssh", "-o", "User=root" ] +os.environ['__remote_copy'] = ["scp", "-o", "User=root" ] args for __remote_exec $1 = hostname $2 - ... = stuff to be executed in /bin/sh on remote side + $2 - $7 = env + $7 - 12 = cmd + args for __remote_copy $1 = file here $2 = hostname:destination + +-------------------------------------------------------------------------------- +There needs to be an easy way to change those cmds! +-------------------------------------------------------------------------------- +Env-Passing: + _a=b test -> test can access $_a + _a=b test $_a -> $1 = "", because _a is *not* set within the shell + _a=b; test -> can access $_a + _a=b; test $_a -> $1 == "b" From eaf2b28fd7b9af740b511657ad66d2e3f52fa3cb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 13:34:28 +0200 Subject: [PATCH 12/54] more prefix ideas Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot b/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot index 91f5d204..ec89a999 100644 --- a/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot +++ b/doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot @@ -44,6 +44,28 @@ os.environ['__remote_exec'] = ["ssh", "-l", "root" ] os.environ['__remote_exec'] = ["ssh", "-o", "User=root" ] os.environ['__remote_copy'] = ["scp", "-o", "User=root" ] + +__remote_exec=~/sudossh __remote_copy=... cdist config localhost + +~/sudossh hostname $@... +~/sudocopy a hostname:b + +~/chrootssh +~/chrootcopy + + + +a) + 3 cmd verzeichnnise: cdist, sudo, chroot + pro aufruf variable ändern + +b) + 1 dir, mit zeug + pro aufruf variablen ändern + + +conf/cmd/remote_exec + args for __remote_exec $1 = hostname $2 - ... = stuff to be executed in /bin/sh on remote side From ecc4fc10d73b8c56d2203e67803b3154a8a066f0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 13:34:45 +0200 Subject: [PATCH 13/54] setup __remote_exec and __remote_copy variables Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 51615c28..43d4bf51 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -297,6 +297,9 @@ def config(args): time_start = datetime.datetime.now() + os.environ['__remote_exec'] = ["ssh", "-o", "User=root" ] + os.environ['__remote_copy'] = ["scp", "-o", "User=root" ] + for host in args.host: c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) if args.parallel: From 278c379e077b398685db1ebff5d72ddcc4f9b78e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 13:37:12 +0200 Subject: [PATCH 14/54] use os.environ['__remote_exec'] in exec* Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 9cedefcc..aaa2c13f 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -20,6 +20,7 @@ # import logging +import os import subprocess log = logging.getLogger(__name__) @@ -32,7 +33,7 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): args[0][:0] = [ "/bin/sh", "-e" ] if remote_prefix: - args[0][:0] = remote_prefix + args[0][:0] = os.environ['__remote_exec'] log.debug("Shell exec cmd: %s", args) @@ -43,8 +44,8 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: log.error("Code that raised the error:\n") - if remote_prefix: - run_or_fail(["cat", script], remote_prefix=remote_prefix) + + run_or_fail(["cat", script], remote_prefix=remote_prefix) else: try: @@ -60,7 +61,7 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): def run_or_fail(*args, remote_prefix=False, **kargs): if remote_prefix: - args[0][:0] = remote_prefix + args[0][:0] = os.environ['__remote_exec'] log.debug("Exec: " + " ".join(*args)) try: From eea6a38f3342d38973fcc79ebdf4204dc94cc831 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 13:39:08 +0200 Subject: [PATCH 15/54] cat only locally :-) Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index aaa2c13f..bb375404 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -45,7 +45,8 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): except subprocess.CalledProcessError: log.error("Code that raised the error:\n") - run_or_fail(["cat", script], remote_prefix=remote_prefix) + if remote_prefix: + run_or_fail(["cat", script], remote_prefix=remote_prefix) else: try: From df630e1fdf0da5049a802717e039baa75a2f734f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 13:40:46 +0200 Subject: [PATCH 16/54] user remote_prefix=True now Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 43d4bf51..81c5b3fc 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -38,23 +38,16 @@ class Config: def __init__(self, target_host, initial_manifest=False, - remote_user="root", home=None, exec_path=sys.argv[0], debug=False): self.target_host = target_host self.debug = debug - self.remote_user = remote_user self.exec_path = exec_path - # FIXME: broken - construct elsewhere! - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] - self.path = cdist.path.Path(self.target_host, initial_manifest=initial_manifest, - remote_user=self.remote_user, - remote_prefix=self.remote_prefix, base_dir=home, debug=debug) @@ -78,7 +71,7 @@ class Config: cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) cmd.append(self.path.remote_global_explorer_path(explorer)) - cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=True) output_fd.close() def run_type_explorer(self, cdist_object): @@ -105,7 +98,7 @@ class Config: log.debug("%s exploring %s using %s storing to %s", cdist_object, explorer, remote_cmd, output) - cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=True) output_fd.close() def link_emulator(self): @@ -235,8 +228,7 @@ class Config: remote_remote_code = os.path.join(remote_dir, "code-remote") if os.path.isfile(local_remote_code): self.path.transfer_file(local_remote_code, remote_remote_code) - # FIXME: remote_prefix - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) def stage_prepare(self): """Do everything for a deploy, minus the actual code stage""" @@ -297,8 +289,8 @@ def config(args): time_start = datetime.datetime.now() - os.environ['__remote_exec'] = ["ssh", "-o", "User=root" ] - os.environ['__remote_copy'] = ["scp", "-o", "User=root" ] + os.environ['__remote_exec'] = "ssh -o User=root" + os.environ['__remote_copy'] = "scp -o User=root" for host in args.host: c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) From 59bee1410147bfc40eb6387d2363e27921e859a2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 14:02:37 +0200 Subject: [PATCH 17/54] begin to eliminate remote_user/prefix from path Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index e416c42d..721764bf 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -57,8 +57,6 @@ class Path: def __init__(self, target_host, - remote_user, - remote_prefix, initial_manifest=False, base_dir=None, debug=False): @@ -72,9 +70,6 @@ class Path: self.temp_dir = tempfile.mkdtemp() self.target_host = target_host - self.remote_user = remote_user - self.remote_prefix = remote_prefix - self.conf_dir = os.path.join(self.base_dir, "conf") self.cache_base_dir = os.path.join(self.base_dir, "cache") self.cache_dir = os.path.join(self.cache_base_dir, target_host) @@ -122,10 +117,10 @@ class Path: def remote_mkdir(self, directory): """Create directory on remote side""" - cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix) + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) def remove_remote_dir(self, destination): - cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix) + cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" From 5a7e4b2f322aaf06fcb422b72fabbcda3ae9bbf8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 15:48:44 +0200 Subject: [PATCH 18/54] quiet ssh/scp Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 81c5b3fc..7b59d416 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -289,8 +289,8 @@ def config(args): time_start = datetime.datetime.now() - os.environ['__remote_exec'] = "ssh -o User=root" - os.environ['__remote_copy'] = "scp -o User=root" + os.environ['__remote_exec'] = "ssh -o User=root -q" + os.environ['__remote_copy'] = "scp -o User=root -q" for host in args.host: c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) From 919f67184a01bde6f45d3e36e1d94ff5a9503cbf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 15:49:08 +0200 Subject: [PATCH 19/54] do not reference scp anymore Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 721764bf..a2b47610 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -125,17 +125,13 @@ class Path: def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) - cdist.exec.run_or_fail(["scp", "-qr", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) + cdist.exec.run_or_fail([os.environ['__remote_copy'], "-r", source, + self.target_host + ":" + destination]) def transfer_file(self, source, destination): """Transfer file""" - cdist.exec.run_or_fail(["scp", "-q", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) + cdist.exec.run_or_fail([os.environ['__remote_copy'], source, + self.target_host + ":" + destination]) def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" From 26278b1e326a4b8951e6df433b9b352985968f71 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 16:28:21 +0200 Subject: [PATCH 20/54] expose target host from config Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 7b59d416..c027da17 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -43,6 +43,8 @@ class Config: debug=False): self.target_host = target_host + os.environ['target_host'] = target_host + self.debug = debug self.exec_path = exec_path From 8b561fd63ca886d20644d1e8f977258c9074947c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 16:30:06 +0200 Subject: [PATCH 21/54] fixup correct split/insert of os.environ() for target_host and args Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 8 ++++++-- lib/cdist/path.py | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index bb375404..a9b8d147 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -33,7 +33,9 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): args[0][:0] = [ "/bin/sh", "-e" ] if remote_prefix: - args[0][:0] = os.environ['__remote_exec'] + remote_prefix = os.environ['__remote_exec'].split() + remote_prefix.append(os.environ['target_host']) + args[0][:0] = remote_prefix log.debug("Shell exec cmd: %s", args) @@ -62,7 +64,9 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): def run_or_fail(*args, remote_prefix=False, **kargs): if remote_prefix: - args[0][:0] = os.environ['__remote_exec'] + remote_prefix = os.environ['__remote_exec'].split() + remote_prefix.append(os.environ['target_host']) + args[0][:0] = remote_prefix log.debug("Exec: " + " ".join(*args)) try: diff --git a/lib/cdist/path.py b/lib/cdist/path.py index a2b47610..85d1a87f 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -125,13 +125,13 @@ class Path: def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) - cdist.exec.run_or_fail([os.environ['__remote_copy'], "-r", source, - self.target_host + ":" + destination]) + cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + + ["-r", source, self.target_host + ":" + destination]) def transfer_file(self, source, destination): """Transfer file""" - cdist.exec.run_or_fail([os.environ['__remote_copy'], source, - self.target_host + ":" + destination]) + cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + + [source, self.target_host + ":" + destination]) def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" From 063fc61291edc18b3c0205526f903dbee47b257c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 16:41:28 +0200 Subject: [PATCH 22/54] remove legacy manpage of cdist-type-emulator Signed-off-by: Nico Schottelius --- doc/man/man1/cdist-type-emulator.text | 56 --------------------------- 1 file changed, 56 deletions(-) delete mode 100644 doc/man/man1/cdist-type-emulator.text diff --git a/doc/man/man1/cdist-type-emulator.text b/doc/man/man1/cdist-type-emulator.text deleted file mode 100644 index 507c1054..00000000 --- a/doc/man/man1/cdist-type-emulator.text +++ /dev/null @@ -1,56 +0,0 @@ -cdist-type-emulator(1) -====================== -Nico Schottelius - - -NAME ----- -cdist-type-emulator - Emulate type and record parameters and dependencies - - -SYNOPSIS --------- -cdist-type-emulator [TYPE ARGS] - - -DESCRIPTION ------------ -cdist-type-emulator is normally called through a link to it of the -name of a specifc type. It saves the given parameters into -a parameters directory and the requirements into a require file. - -It checks whether the parameters are valid: - -- are required parameter given? -- are all other required parameters specified as optional? - - -EXAMPLES --------- -Your manifest may contain stuff like this: - - --------------------------------------------------------------------------------- -__addifnosuchline /tmp/linetest --line "test" - -__motd --------------------------------------------------------------------------------- - -In both cases, cdist-type-emulator is called instead of a real type. -In the first case, the object id "/tmp/linetest" is recorded and the -parameter "line" stored with the content "test". - -In the second case, __motd must be decleared as a singleton, as the -object id is missing. - - -SEE ALSO --------- -- cdist(7) -- cdist-type-build-emulation(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). From afa3920ca987347c50ffd44f9b4d11a3d28f37d3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 16:42:45 +0200 Subject: [PATCH 23/54] remove more obsolete manpages Signed-off-by: Nico Schottelius --- doc/man/man1/cdist-config.text | 48 ----------------------------- doc/man/man1/cdist-env.text | 49 ------------------------------ doc/man/man1/cdist-quickstart.text | 47 ---------------------------- 3 files changed, 144 deletions(-) delete mode 100644 doc/man/man1/cdist-config.text delete mode 100644 doc/man/man1/cdist-env.text delete mode 100644 doc/man/man1/cdist-quickstart.text diff --git a/doc/man/man1/cdist-config.text b/doc/man/man1/cdist-config.text deleted file mode 100644 index 0c8b0735..00000000 --- a/doc/man/man1/cdist-config.text +++ /dev/null @@ -1,48 +0,0 @@ -cdist-config(1) -=============== -Nico Schottelius - - -NAME ----- -cdist-config - Read basic cdist configuration - - -DESCRIPTION ------------ -Cdist-config is sourced by cdist programs and provides hints on where to find -types, manifests, etc. Generally speaking, it's just usable from within the -core and is only of interest for cdist-developers. - - -ENVIRONMENT VARIABLES ---------------------- -The following list contains environment variables that are known -to be changed by users in various situations. To change the variable, -use your current shell and export it, so all cdist-binaries know about it. - -__cdist_tmp_base_dir:: - Normally this points to /tmp. In case /tmp is not suitable for - cdist (i.e. has noexec flag setup) you can change this variable - to point to a better location. - - -EXAMPLES --------- - -If /tmp has the noexec flag, you can use $HOME/.tmp for instance: - --------------------------------------------------------------------------------- -export __cdist_tmp_base_dir=$HOME/.tmp --------------------------------------------------------------------------------- - - -SEE ALSO --------- -cdist(7) - - -COPYING -------- -Copyright \(C) 2010-2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-env.text b/doc/man/man1/cdist-env.text deleted file mode 100644 index 9a736133..00000000 --- a/doc/man/man1/cdist-env.text +++ /dev/null @@ -1,49 +0,0 @@ -cdist-env(1) -============ -Nico Schottelius - - -NAME ----- -cdist-env - Setup environment for using cdist - - -SYNOPSIS --------- -cdist-env - - -DESCRIPTION ------------ -cdist-env outputs two strings suitable for usage in your current shell, -so you can use cdist from the checkout. cdist-env essentially helps you -to easily setup PATH and MANPATH. - -If you've multiple checkouts of cdist and run cdist-env from the various -checkouts, a new run will prepend the last directory, thus ensures you -can run it multiple times and does what one expects. - -EXAMPLES --------- -For use in bourne shell variants (like dash, bash, ksh) as well as -in csh variants (csh, tcsh): - --------------------------------------------------------------------------------- -eval `./bin/cdist-env` --------------------------------------------------------------------------------- - -For bourne shell, there is also a shorter version: --------------------------------------------------------------------------------- -. ./bin/cdist-env --------------------------------------------------------------------------------- - - -SEE ALSO --------- -cdist(7) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-quickstart.text b/doc/man/man1/cdist-quickstart.text deleted file mode 100644 index 087fd2d5..00000000 --- a/doc/man/man1/cdist-quickstart.text +++ /dev/null @@ -1,47 +0,0 @@ -cdist-quickstart(1) -=================== -Nico Schottelius - -NAME ----- -cdist-quickstart - Make use of cinit in 5 minutes - - -SYNOPSIS --------- -cdist-quickstart - - -DESCRIPTION ------------ -cdist-quickstart is an interactive guide to cdist. It should be one -of the first tools you use when you begin with cdist. - - -EXAMPLES --------- -To use cdist-quickstart, add the bin directory to your PATH, execute -cdist-quickstart and enjoy cdist: - --------------------------------------------------------------------------------- -# Bourne shell example -export PATH=$(pwd -P)/bin:$PATH - -# Alternatively, usable for csh and bsh, set's up PATH and MANPATH -eval `./bin/cdist-env` - -# Let's go! -cdist-quickstart --------------------------------------------------------------------------------- - - -SEE ALSO --------- -- cdist(7) -- cdist-env(1) - - -COPYING -------- -Copyright \(C) 2010-2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). From 7c11fa09378e63c49a52114f80f5d9e00919453f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 16:49:27 +0200 Subject: [PATCH 24/54] introduce unclear changes (names not yet decided) Signed-off-by: Nico Schottelius --- doc/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog b/doc/changelog index 632ddb45..798c90e0 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,6 +1,7 @@ 2.0.3: * Improved logging, added --verbose, by more quiet by default * Bugfix __user: Correct quoting (Steven Armstrong) + * FIXME: Support for __remote_exec and __remote_copy 2.0.2: 2011-09-27 * Add support for detection of OpenWall Linux (Matthias Teege) From 1c8a14339759aece4029e8ca8452b22bca3a9753 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 16:52:56 +0200 Subject: [PATCH 25/54] commit previously missed change Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 429 +++++++++++++++++------------------- 1 file changed, 202 insertions(+), 227 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 39ee79c6..4829061b 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -31,276 +31,251 @@ import cdist.path log = logging.getLogger(__name__) -CODE_HEADER = "#!/bin/sh -e\n" +CODE_HEADER = "#!/bin/sh -e\n" class ConfigInstall: - """Class to hold install and config methods""" + """Cdist main class to hold arbitrary data""" - def __init__(self, target_host, - initial_manifest=False, - remote_user="root", - home=None, - exec_path=sys.argv[0], - debug=False): + def __init__(self, target_host, + initial_manifest=False, + home=None, + exec_path=sys.argv[0], + debug=False): - self.target_host = target_host - self.debug = debug - self.remote_user = remote_user - self.exec_path = exec_path + self.target_host = target_host + os.environ['target_host'] = target_host - # FIXME: broken - construct elsewhere! - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + self.debug = debug + self.exec_path = exec_path - self.path = cdist.path.Path(self.target_host, - initial_manifest=initial_manifest, - remote_user=self.remote_user, - remote_prefix=self.remote_prefix, - base_dir=home, - debug=debug) - - self.objects_prepared = [] + self.path = cdist.path.Path(self.target_host, + initial_manifest=initial_manifest, + base_dir=home, + debug=debug) + + self.objects_prepared = [] - def cleanup(self): - self.path.cleanup() + def cleanup(self): + self.path.cleanup() - def run_global_explorers(self): - """Run global explorers""" - log.info("Running global explorers") - explorers = self.path.list_global_explorers() - if(len(explorers) == 0): - raise CdistError("No explorers found in ", self.path.global_explorer_dir) + def run_global_explores(self): + """Run global explorers""" + log.info("Running global explorers") + explorers = self.path.list_global_explorers() + if(len(explorers) == 0): + raise CdistError("No explorers found in", self.path.global_explorer_dir) - self.path.transfer_global_explorers() - for explorer in explorers: - output = self.path.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.path.remote_global_explorer_path(explorer)) + self.path.transfer_global_explorers() + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.path.remote_global_explorer_path(explorer)) - cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=True) + output_fd.close() -# FIXME: where to call this from? - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" - type = self.path.get_type_from_object(cdist_object) - self.path.transfer_type_explorers(type) + type = self.path.get_type_from_object(cdist_object) + self.path.transfer_type_explorers(type) - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) - # Need to transfer at least the parameters for objects to be useful - self.path.transfer_object_parameter(cdist_object) + # Need to transfer at least the parameters for objects to be useful + self.path.transfer_object_parameter(cdist_object) - # FIXME: Broken due to refactoring into type.py - explorers = self.path.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() + explorers = self.path.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=True) + output_fd.close() - def link_emulator(self): - """Link emulator to types""" - cdist.emulator.link(self.exec_path, - self.path.bin_dir, self.path.list_types()) + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") - self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) - self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - self.link_emulator() + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + self.link_emulator() - def run_initial_manifest(self): - """Run the initial manifest""" - log.info("Running initial manifest %s", self.path.initial_manifest) - env = { "__manifest" : self.path.manifest_dir } - self.run_manifest(self.path.initial_manifest, extra_env=env) + def run_initial_manifest(self): + """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) + env = { "__manifest" : self.path.manifest_dir } + self.run_manifest(self.path.initial_manifest, extra_env=env) - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.path.get_type_from_object(cdist_object) - manifest = self.path.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.path.object_dir(cdist_object), - "__object_id": self.path.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.path.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.path.get_type_from_object(cdist_object) + manifest = self.path.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.path.object_dir(cdist_object), + "__object_id": self.path.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.path.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.path.bin_dir + ":" + env['PATH'] + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.path.bin_dir + ":" + env['PATH'] - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - - # Submit debug flag to manifest, can be used by emulator and types - if self.debug: - env['__debug'] = "yes" + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + + # Submit debug flag to manifest, can be used by emulator and types + if self.debug: + env['__debug'] = "yes" - # Required for recording source - env['__cdist_manifest'] = manifest + # Required for recording source + env['__cdist_manifest'] = manifest - # Required to find types - env['__cdist_type_base_dir'] = self.path.type_base_dir + # Required to find types + env['__cdist_type_base_dir'] = self.path.type_base_dir - # Other environment stuff - if extra_env: - env.update(extra_env) + # Other environment stuff + if extra_env: + env.update(extra_env) - cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) + cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.path.object_dir(cdist_object), "require") - requirements = cdist.path.file_to_list(file) - type = self.path.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.path.object_dir(cdist_object), "require") + requirements = cdist.path.file_to_list(file) + type = self.path.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - env["__object"] = self.path.object_dir(cdist_object) - env["__object_id"] = self.path.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.path.type_dir(type) + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + env["__object"] = self.path.object_dir(cdist_object) + env["__object_id"] = self.path.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.path.type_dir(type) - if mode == "gencode": - paths = [ - self.path.type_dir(type, "gencode-local"), - self.path.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.path.object_dir(cdist_object), - os.path.basename(bin)[3:]) + if mode == "gencode": + paths = [ + self.path.type_dir(type, "gencode-local"), + self.path.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.path.object_dir(cdist_object), + os.path.basename(bin)[3:]) - outfile_fd = open(outfile, "w") + outfile_fd = open(outfile, "w") - # Need to flush to ensure our write is done before stdout write - # FIXME: CODE_HEADER needed in our sh -e scenario? - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() - cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() - status = os.stat(outfile) + status = os.stat(outfile) - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - # Mark object as changed - open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() + # Mark object as changed + open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() - if mode == "code": - local_dir = self.path.object_dir(cdist_object) - remote_dir = self.path.remote_object_dir(cdist_object) + if mode == "code": + local_dir = self.path.object_dir(cdist_object) + remote_dir = self.path.remote_object_dir(cdist_object) - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - cdist.exec.run_or_fail([bin]) - + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + cdist.exec.run_or_fail([bin]) + - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.path.transfer_file(local_remote_code, remote_remote_code) - # FIXME: remote_prefix - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explorers() - self.run_initial_manifest() - - log.info("Running object manifests and type explorers") + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.path.transfer_file(local_remote_code, remote_remote_code) + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + log.info("Running object manifests and type explorers") - old_objects = [] - objects = self.path.list_objects() + old_objects = [] + objects = self.path.list_objects() - # Continue process until no new objects are created anymore - while old_objects != objects: - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - # FIXME: run_type_explorer: - # object can return type - # type has explorers - # path knows about where to save explorer output - # type = self.path.objects[object].type() - # self.path.types['type'].explorers() - # for explorer in explorers: - # output = cdist.exec.run_debug_or_fail_shell(explorer) - # if output: - # write_output_to(output, os.path.join(self.path.objects[object].explorer_dir(),explorer) ) - # - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) + # Continue process until no new objects are created anymore + while old_objects != objects: + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) - objects = self.path.list_objects() + objects = self.path.list_objects() - def stage_run(self): - """The final (and real) step of deployment""" - log.info("Generating and executing code") - # Now do the final steps over the existing objects - for cdist_object in self.path.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") + def stage_run(self): + """The final (and real) step of deployment""" + log.info("Generating and executing code") + # Now do the final steps over the existing objects + for cdist_object in self.path.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() - self.stage_prepare() - self.stage_run() + self.stage_prepare() + self.stage_run() - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) From 8341e0cc0f964ad8e3c9b301e3c548b953fe7d9a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 16:54:47 +0200 Subject: [PATCH 26/54] think about the __init_script type (probably raise discussion) Signed-off-by: Nico Schottelius --- other/types_pending_inclusion/__init_script/README | 4 ++++ .../types_pending_inclusion}/__init_script/gencode-remote | 0 .../types_pending_inclusion}/__init_script/man.text | 0 .../types_pending_inclusion}/__init_script/parameter/optional | 0 .../types_pending_inclusion}/__init_script/parameter/required | 0 5 files changed, 4 insertions(+) create mode 100644 other/types_pending_inclusion/__init_script/README rename {conf/type => other/types_pending_inclusion}/__init_script/gencode-remote (100%) rename {conf/type => other/types_pending_inclusion}/__init_script/man.text (100%) rename {conf/type => other/types_pending_inclusion}/__init_script/parameter/optional (100%) rename {conf/type => other/types_pending_inclusion}/__init_script/parameter/required (100%) diff --git a/other/types_pending_inclusion/__init_script/README b/other/types_pending_inclusion/__init_script/README new file mode 100644 index 00000000..f2621f12 --- /dev/null +++ b/other/types_pending_inclusion/__init_script/README @@ -0,0 +1,4 @@ +Moved out of conf/type/ to think about whether this type makes sense or not. + +Cdist describes the state and using an init_script may be useful, but +should only be used conditionally. diff --git a/conf/type/__init_script/gencode-remote b/other/types_pending_inclusion/__init_script/gencode-remote similarity index 100% rename from conf/type/__init_script/gencode-remote rename to other/types_pending_inclusion/__init_script/gencode-remote diff --git a/conf/type/__init_script/man.text b/other/types_pending_inclusion/__init_script/man.text similarity index 100% rename from conf/type/__init_script/man.text rename to other/types_pending_inclusion/__init_script/man.text diff --git a/conf/type/__init_script/parameter/optional b/other/types_pending_inclusion/__init_script/parameter/optional similarity index 100% rename from conf/type/__init_script/parameter/optional rename to other/types_pending_inclusion/__init_script/parameter/optional diff --git a/conf/type/__init_script/parameter/required b/other/types_pending_inclusion/__init_script/parameter/required similarity index 100% rename from conf/type/__init_script/parameter/required rename to other/types_pending_inclusion/__init_script/parameter/required From 220611604b5ecc8c01a2f592c99f1ff793c7b26e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 17:00:09 +0200 Subject: [PATCH 27/54] +: Signed-off-by: Nico Schottelius --- lib/cdist/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 864b4f37..1f325a8c 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -29,7 +29,7 @@ class Error(Exception): class MissingEnvironmentVariableError(Error): """Raised when a required environment variable is not set.""" - def __init__(self, name) + def __init__(self, name): self.name = name def __str__(self): From 60b5f8783c56b405c4b13105dbd4bfeeb9671b6e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 17:02:37 +0200 Subject: [PATCH 28/54] revert changes from upcoming install branch Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 7 +++---- lib/cdist/install.py | 3 --- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 657714a4..0ae41174 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -22,13 +22,15 @@ import datetime import logging -log = logging.getLogger(__name__) +import sys import cdist.emulator import cdist.path CODE_HEADER = "#!/bin/sh -e\n" +log = logging.getLogger(__name__) + class Config: """Cdist main class to hold arbitrary data""" @@ -276,9 +278,6 @@ class Config: self.target_host, duration.total_seconds()) -class Config(cdist.config_install.ConfigInstall): - pass - def config(args): """Configure remote system""" process = {} diff --git a/lib/cdist/install.py b/lib/cdist/install.py index 5a35626d..87714fa8 100644 --- a/lib/cdist/install.py +++ b/lib/cdist/install.py @@ -27,9 +27,6 @@ import cdist.config_install log = logging.getLogger(__name__) -Class Install(cdist.config_install.ConfigInstall): - pass - def install(args): """Install remote system""" process = {} From df5de24b72cf98c6ca091b11fbf827758dd78d1c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 18:26:55 +0200 Subject: [PATCH 29/54] begin use of cdist.core.Object.list_objects() Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index c7c592a5..b6d7451b 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -29,6 +29,8 @@ import sys import cdist.emulator import cdist.path +import cdist.core + log = logging.getLogger(__name__) CODE_HEADER = "#!/bin/sh -e\n" @@ -108,14 +110,6 @@ class ConfigInstall: cdist.emulator.link(self.exec_path, self.path.bin_dir, self.path.list_types()) - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) - self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - self.link_emulator() - def run_initial_manifest(self): """Run the initial manifest""" log.info("Running initial manifest %s", self.path.initial_manifest) @@ -241,11 +235,11 @@ class ConfigInstall: log.info("Running object manifests and type explorers") old_objects = [] - objects = self.path.list_objects() + objects = cdist.core.Object.list_objects() # Continue process until no new objects are created anymore while old_objects != objects: - old_objects = list(objects) + old_objects = objects for cdist_object in objects: if cdist_object in self.objects_prepared: log.debug("Skipping rerun of object %s", cdist_object) @@ -255,7 +249,7 @@ class ConfigInstall: self.run_type_manifest(cdist_object) self.objects_prepared.append(cdist_object) - objects = self.path.list_objects() + objects = cdist.core.Object.list_objects() def stage_run(self): """The final (and real) step of deployment""" @@ -285,3 +279,11 @@ class ConfigInstall: self.deploy_to() self.cleanup() + ### Cleaned / check functions: Round 1 :-) ################################# + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + self.link_emulator() From 2157cef2a6048a61ef9708269b2dc4a0c1ed38d2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:22:08 +0200 Subject: [PATCH 30/54] make cdist.MissingEnvironmentVariableError print the key and not fail :-) Signed-off-by: Nico Schottelius --- lib/cdist/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 1f325a8c..fcfa3693 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -33,4 +33,4 @@ class MissingEnvironmentVariableError(Error): self.name = name def __str__(self): - return 'Missing required environment variable: {0.name}'.format(o) + return 'Missing required environment variable: ' + str(self.name) From 5c9694215ca4943c5209062d9394c5ee1194df7e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:26:58 +0200 Subject: [PATCH 31/54] setup '__cdist_out_dir' in path for use in Object Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index c45ddcd6..73a90302 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -68,6 +68,8 @@ class Path: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) self.temp_dir = tempfile.mkdtemp() + os.environ['__cdist_out_dir'] = self.temp_dir + self.target_host = target_host # Input directories From 344bfb06030f4ebe4e59efe755d642f52893765a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:40:19 +0200 Subject: [PATCH 32/54] was the object prepared/ran? Signed-off-by: Nico Schottelius --- lib/cdist/core/object.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/cdist/core/object.py b/lib/cdist/core/object.py index 127bf038..544f7565 100644 --- a/lib/cdist/core/object.py +++ b/lib/cdist/core/object.py @@ -89,6 +89,10 @@ class Object(object): self.__parameters = None self.__requirements = None + + # Whether this object was prepared/ran + self.prepared = False + self.ran = False def __repr__(self): return '' % self.name From 8305f07aab6c7c9b57ce2d69045a4550b3fcfadd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:43:07 +0200 Subject: [PATCH 33/54] stage prepare uses new object interface Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 48 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index b6d7451b..b1db2855 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -226,31 +226,6 @@ class ConfigInstall: self.path.transfer_file(local_remote_code, remote_remote_code) cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - log.info("Running object manifests and type explorers") - - old_objects = [] - objects = cdist.core.Object.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - old_objects = objects - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) - - objects = cdist.core.Object.list_objects() - def stage_run(self): """The final (and real) step of deployment""" log.info("Generating and executing code") @@ -287,3 +262,26 @@ class ConfigInstall: self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) self.link_emulator() + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + log.info("Running object manifests and type explorers") + + # Continue process until no new objects are created anymore + new_objects_created = True + while new_objects_created: + new_objects_created = False + for cdist_object in cdist.core.Object.list_objects(): + if cdist_object.prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + cdist_object.prepared = True + new_objects_created = True From ccb2ffcae620d829d090729ff7232f3c35df25f1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:47:55 +0200 Subject: [PATCH 34/54] remove toplevel object/type Signed-off-by: Nico Schottelius --- lib/cdist/object.py | 51 ----------------------------------------- lib/cdist/type.py | 55 --------------------------------------------- 2 files changed, 106 deletions(-) delete mode 100644 lib/cdist/object.py delete mode 100644 lib/cdist/type.py diff --git a/lib/cdist/object.py b/lib/cdist/object.py deleted file mode 100644 index 0a282dc2..00000000 --- a/lib/cdist/object.py +++ /dev/null @@ -1,51 +0,0 @@ -# -*- coding: utf-8 -*- -# -# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import os -import logging -log = logging.getLogger(__name__) - - - -class Object(object): - - def __init__(self, path, remote_path, object_fq): - self.path = path - self.remote_path = remote_path - self.object_fq = object_fq - self.type = self.object_fq.split(os.sep)[0] - self.object_id = self.object_fq.split(os.sep)[1:] - self.parameter_dir = os.path.join(self.path, "parameter") - self.remote_object_parameter_dir = os.path.join(self.remote_path, "parameter") - self.object_code_paths = [ - os.path.join(self.path, "code-local"), - os.path.join(self.path, "code-remote")] - - @property - def type_explorer_output_dir(self): - """Returns and creates dir of the output for a type explorer""" - if not self.__type_explorer_output_dir: - dir = os.path.join(self.path, "explorer") - if not os.path.isdir(dir): - os.mkdir(dir) - self.__type_explorer_output_dir = dir - return self.__type_explorer_output_dir - diff --git a/lib/cdist/type.py b/lib/cdist/type.py deleted file mode 100644 index e6c35ad1..00000000 --- a/lib/cdist/type.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -log = logging.getLogger(__name__) - -class Type(object): - - def __init__(self, path, remote_path): - self.path = path - self.remote_path = remote_path - - def list_explorers(self): - """Return list of available explorers""" - dir = os.path.join(self.path, "explorer") - if os.path.isdir(dir): - list = os.listdir(dir) - else: - list = [] - - log.debug("Explorers for %s in %s: %s", type, dir, list) - - return list - - def is_install(self): - """Check whether a type is used for installation (if not: for configuration)""" - return os.path.isfile(os.path.join(self.path, "install")) - - def explorer_dir(self): - """Return remote directory that holds the explorers of a type""" - return os.path.join(self.remote_path, "explorer") - - def remote_explorer_dir(self): - """Return remote directory that holds the explorers of a type""" - return os.path.join(self.remote_path, "explorer") From ae9eba80a9e369e91d5b9a69c2af0fe88c5769fc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:50:45 +0200 Subject: [PATCH 35/54] use require, not __require in emulator Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 38a58f8c..519afd24 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -127,7 +127,7 @@ def run(argv): param_fd.close() # Record requirements - if "__require" in os.environ: + if "require" in os.environ: requirements = os.environ['__require'] log.debug(object_id + ":Writing requirements: " + requirements) require_fd = open(os.path.join(object_dir, "require"), "a") From 52268d032a11afe0c6abfc473314be654571be69 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:51:42 +0200 Subject: [PATCH 36/54] ++todos Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 11c734f9..42dec573 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,3 +1,10 @@ +2.0.3: + +- fix emulator +- introduce tests for $require + +-------------------------------------------------------------------------------- + - Fix / rewrite cdist-quickstart - write tutorial!!!!!!!!! From d07a191867b31efc914248181de207fbaef37fc3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:54:41 +0200 Subject: [PATCH 37/54] add logger to config Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 6bf9d782..942d2917 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -27,6 +27,8 @@ import sys import cdist.config_install +log = logging.getLogger(__name__) + class Config(cdist.config_install.ConfigInstall): pass From a87e6f22b5251c364e86d74da535d7aa52b8ce3f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 19:59:07 +0200 Subject: [PATCH 38/54] use new cdist.Error exception Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index b1db2855..3b6c79a1 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -65,7 +65,7 @@ class ConfigInstall: log.info("Running global explorers") explorers = self.path.list_global_explorers() if(len(explorers) == 0): - raise CdistError("No explorers found in", self.path.global_explorer_dir) + raise cdist.Error("No explorers found in", self.path.global_explorer_dir) self.path.transfer_global_explorers() for explorer in explorers: @@ -159,6 +159,8 @@ class ConfigInstall: def object_run(self, cdist_object, mode): """Run gencode or code for an object""" log.debug("Running %s from %s", mode, cdist_object) + + # FIXME: replace with new object interface file=os.path.join(self.path.object_dir(cdist_object), "require") requirements = cdist.path.file_to_list(file) type = self.path.get_type_from_object(cdist_object) @@ -169,7 +171,7 @@ class ConfigInstall: # # Setup env Variable: - # + # env = os.environ.copy() env['__target_host'] = self.target_host env['__global'] = self.path.out_dir @@ -230,7 +232,7 @@ class ConfigInstall: """The final (and real) step of deployment""" log.info("Generating and executing code") # Now do the final steps over the existing objects - for cdist_object in self.path.list_objects(): + for cdist_object in cdist.core.Object.list_objects(): log.debug("Run object: %s", cdist_object) self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") From 14126cb4b85a88a1dbd55af19c975243877d19dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:04:44 +0200 Subject: [PATCH 39/54] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 42dec573..966ead7a 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,8 +1,12 @@ 2.0.3: - fix emulator -- introduce tests for $require +- introduce tests: + - does $require work? + - $whatever should fail if there is no global explorer directory +- Create GlobalExplorer + -------------------------------------------------------------------------------- - Fix / rewrite cdist-quickstart From 14082f5c0049e8eba532034bf601dfc500f71392 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:05:24 +0200 Subject: [PATCH 40/54] string, not tuple in exception Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 3b6c79a1..f3dac289 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -65,7 +65,7 @@ class ConfigInstall: log.info("Running global explorers") explorers = self.path.list_global_explorers() if(len(explorers) == 0): - raise cdist.Error("No explorers found in", self.path.global_explorer_dir) + raise cdist.Error("No explorers found in " + self.path.global_explorer_dir) self.path.transfer_global_explorers() for explorer in explorers: @@ -251,12 +251,13 @@ class ConfigInstall: self.target_host, duration.total_seconds()) + + ### Cleaned / check functions: Round 1 :-) ################################# def deploy_and_cleanup(self): """Do what is most often done: deploy & cleanup""" self.deploy_to() self.cleanup() - ### Cleaned / check functions: Round 1 :-) ################################# def init_deploy(self): """Ensure the base directories are cleaned up""" log.debug("Creating clean directory structure") From d4bf98cf0c5b9c5026f136b92c973c6a9d5a93c4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:08:46 +0200 Subject: [PATCH 41/54] +tests needed Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 966ead7a..85620098 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -4,6 +4,7 @@ - introduce tests: - does $require work? - $whatever should fail if there is no global explorer directory + - emulator may only be called with __ as prefix - fail otherwise! - Create GlobalExplorer From 1db4cd48d24fecd4e92b4a941de3f86ef291c910 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:11:23 +0200 Subject: [PATCH 42/54] -typo Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index f3dac289..1f590d07 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -55,12 +55,10 @@ class ConfigInstall: base_dir=home, debug=debug) - self.objects_prepared = [] - def cleanup(self): self.path.cleanup() - def run_global_explores(self): + def run_global_explorers(self): """Run global explorers""" log.info("Running global explorers") explorers = self.path.list_global_explorers() @@ -269,7 +267,7 @@ class ConfigInstall: def stage_prepare(self): """Do everything for a deploy, minus the actual code stage""" self.init_deploy() - self.run_global_explores() + self.run_global_explorers() self.run_initial_manifest() log.info("Running object manifests and type explorers") @@ -285,6 +283,5 @@ class ConfigInstall: else: self.run_type_explorer(cdist_object) self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) cdist_object.prepared = True new_objects_created = True From 609efcbfd1af1b2e679c5a0c708941c3b35caedd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:14:34 +0200 Subject: [PATCH 43/54] Remove datetime from config_install, break code and make it more beautiful Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 1f590d07..3be7e0b1 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -20,7 +20,6 @@ # # -import datetime import logging import os import stat From 84867db20a08aa7c742d28e891a1ffea328f90de Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:14:48 +0200 Subject: [PATCH 44/54] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 85620098..bfbdd8c3 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -10,6 +10,9 @@ -------------------------------------------------------------------------------- +- insert prefix into logger to distinguish between modules + - in debug/info only? + - Fix / rewrite cdist-quickstart - write tutorial!!!!!!!!! From 13d47f3cf4fd2a083218073e6fc7f79fcc57e3b9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:20:35 +0200 Subject: [PATCH 45/54] remove datetime from config to wrong position Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++++++++++ lib/cdist/config_install.py | 11 +---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 0bf3ed9c..36dc3d5c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -21,6 +21,7 @@ # import argparse +import datetime import logging import os import re @@ -105,6 +106,8 @@ if __name__ == "__main__": try: logging.basicConfig(format='%(levelname)s: %(message)s') + time_start = datetime.datetime.now() + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): import cdist.emulator cdist.emulator.run(sys.argv) @@ -117,6 +120,14 @@ if __name__ == "__main__": import cdist.path commandline() + + time_end = datetime.datetime.now() + duration = time_end - time_start + # FIXME: move into runner + # log.info("Finished run of %s in %s seconds", self.target_host, + # duration.total_seconds()) + log.info("Finished run in %s seconds", duration.total_seconds()) + except KeyboardInterrupt: sys.exit(0) except cdist.Error as e: diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 3be7e0b1..7d0e2c9a 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -234,22 +234,13 @@ class ConfigInstall: self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") + ### Cleaned / check functions: Round 1 :-) ################################# def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() - self.stage_prepare() self.stage_run() - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - - - ### Cleaned / check functions: Round 1 :-) ################################# def deploy_and_cleanup(self): """Do what is most often done: deploy & cleanup""" self.deploy_to() From 579bd45c8765f44560f2279a063bf97c46dc050f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:24:40 +0200 Subject: [PATCH 46/54] cleanup more stuff in config_install Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 7d0e2c9a..ac988425 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -225,6 +225,7 @@ class ConfigInstall: self.path.transfer_file(local_remote_code, remote_remote_code) cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) + ### Cleaned / check functions: Round 1 :-) ################################# def stage_run(self): """The final (and real) step of deployment""" log.info("Generating and executing code") @@ -234,7 +235,6 @@ class ConfigInstall: self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") - ### Cleaned / check functions: Round 1 :-) ################################# def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to " + self.target_host) @@ -271,6 +271,7 @@ class ConfigInstall: log.debug("Skipping rerun of object %s", cdist_object) continue else: + log.debug("Preparing object: " + cdist_object) self.run_type_explorer(cdist_object) self.run_type_manifest(cdist_object) cdist_object.prepared = True From be1d4afd589e3265089d736b9a75e2e191d94776 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:27:07 +0200 Subject: [PATCH 47/54] add log to object Signed-off-by: Nico Schottelius --- lib/cdist/core/object.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cdist/core/object.py b/lib/cdist/core/object.py index 544f7565..8ad2a9eb 100644 --- a/lib/cdist/core/object.py +++ b/lib/cdist/core/object.py @@ -19,12 +19,14 @@ # # +import logging import os import collections import cdist import cdist.core.property +log = logging.getLogger(__name__) DOT_CDIST = '.cdist' From 9b5b94577fd75f6433b045c4dfa18770e80a9dde Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:32:47 +0200 Subject: [PATCH 48/54] add debug Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 2 ++ lib/cdist/emulator.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index ac988425..29154f84 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -262,6 +262,8 @@ class ConfigInstall: log.info("Running object manifests and type explorers") + log.debug("Searching for objects in " + cdist.core.Object.base_dir()) + # Continue process until no new objects are created anymore new_objects_created = True while new_objects_created: diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 519afd24..51b2ecc1 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -77,6 +77,8 @@ def run(argv): object_dir = os.path.join(global_dir, "object", type, object_id, cdist.path.DOT_CDIST) + log.debug("Object output dir = " + object_dir) + param_out_dir = os.path.join(object_dir, "parameter") object_source_file = os.path.join(object_dir, "source") From 6bd64437213a177924f699b22d5e5b72ec8653c9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:32:59 +0200 Subject: [PATCH 49/54] fix out dir for object Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 73a90302..98572b81 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -68,7 +68,6 @@ class Path: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) self.temp_dir = tempfile.mkdtemp() - os.environ['__cdist_out_dir'] = self.temp_dir self.target_host = target_host @@ -93,6 +92,8 @@ class Path: self.object_base_dir = os.path.join(self.out_dir, "object") self.bin_dir = os.path.join(self.out_dir, "bin") + os.environ['__cdist_out_dir'] = self.out_dir + # List of type explorers transferred self.type_explorers_transferred = {} From 354e15015a9385a7cae7b4e0a83c3599a69a6a69 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:34:36 +0200 Subject: [PATCH 50/54] use full.path.to.cdist.core.Type(), Steven ;-) Signed-off-by: Nico Schottelius --- lib/cdist/core/object.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/cdist/core/object.py b/lib/cdist/core/object.py index 8ad2a9eb..dbb6542d 100644 --- a/lib/cdist/core/object.py +++ b/lib/cdist/core/object.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # -# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -67,7 +68,7 @@ class Object(object): for object_name in cls.list_object_names(): type_name = object_name.split(os.sep)[0] object_id = os.sep.join(object_name.split(os.sep)[1:]) - yield cls(Type(type_name), object_id=object_id) + yield cls(cdist.core.Type(type_name), object_id=object_id) @classmethod def list_type_names(cls): From c0d6d4d5309d46530090ed0ef9460c86fc7237d1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:37:55 +0200 Subject: [PATCH 51/54] use name, not object Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 29154f84..3c11d6bb 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -273,7 +273,7 @@ class ConfigInstall: log.debug("Skipping rerun of object %s", cdist_object) continue else: - log.debug("Preparing object: " + cdist_object) + log.debug("Preparing object: " + cdist_object.name) self.run_type_explorer(cdist_object) self.run_type_manifest(cdist_object) cdist_object.prepared = True From b891818b9cd34ec59878eeb04f1880f4a006a0de Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:42:31 +0200 Subject: [PATCH 52/54] remove obsolete get_type_from_object() and use Object.type Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 2 +- lib/cdist/path.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 3c11d6bb..2071a0e9 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -78,7 +78,7 @@ class ConfigInstall: def run_type_explorer(self, cdist_object): """Run type specific explorers for objects""" - type = self.path.get_type_from_object(cdist_object) + type = cdist_object.type self.path.transfer_type_explorers(type) cmd = [] diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 98572b81..d82ae974 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -192,11 +192,6 @@ class Path: return object_paths - # FIXME: Object - def get_type_from_object(self, cdist_object): - """Returns the first part (i.e. type) of an object""" - return cdist_object.split(os.sep)[0] - # FIXME: Object def get_object_id_from_object(self, cdist_object): """Returns everything but the first part (i.e. object_id) of an object""" From 35a7757e1aa6f1f7d6722a6bd903ce631aefb24a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:51:31 +0200 Subject: [PATCH 53/54] rewrite path to include calls to non-existent methods in type (fix needed, sar :-) Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index d82ae974..deeb6b0d 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -255,19 +255,25 @@ class Path: # Stays here - FIXME: adjust to type code, loop over types! def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" - if type in self.type_explorers_transferred: + if type.transferred: log.debug("Skipping retransfer for explorers of %s", type) return else: # Do not retransfer - self.type_explorers_transferred[type] = 1 + type.transferred = True - src = self.type_dir(type, "explorer") - remote_base = os.path.join(REMOTE_TYPE_DIR, type) - dst = self.remote_type_explorer_dir(type) + # FIXME: need to get explorer path from type! + src = type.explorer_path() + dst = type.remote_explorer_path() + # FIXME: where to construct remote path? here? + # remote_base = os.path.join(REMOTE_TYPE_DIR, type.name) + # dst = self.remote_type_explorer_dir(type) # Only continue, if there is at least the directory - if os.path.isdir(src): + #if os.path.isdir(src): + + # Transfer if there is at least one explorer + if len(type.explorers) > 0: # Ensure that the path exists - self.remote_mkdir(remote_base) + self.remote_mkdir(dst) self.transfer_dir(src, dst) From 6f58b18c8cff14d3ce736dc6a605a222ccf0d913 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Oct 2011 20:52:34 +0200 Subject: [PATCH 54/54] remove legacy code, fail until type supports us Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index deeb6b0d..f7d2a8ae 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -262,16 +262,10 @@ class Path: # Do not retransfer type.transferred = True - # FIXME: need to get explorer path from type! + # FIXME: Can be explorer_path or explorer_dir, I don't care. src = type.explorer_path() dst = type.remote_explorer_path() - # FIXME: where to construct remote path? here? - # remote_base = os.path.join(REMOTE_TYPE_DIR, type.name) - # dst = self.remote_type_explorer_dir(type) - # Only continue, if there is at least the directory - #if os.path.isdir(src): - # Transfer if there is at least one explorer if len(type.explorers) > 0: # Ensure that the path exists