From 7c566309246bba675f17f82378cd596f53ecd5cc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 13:13:56 +0200 Subject: [PATCH 01/10] prepare installation routine Signed-off-by: Nico Schottelius --- bin/cdist | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index f7ad2f72..49985c01 100755 --- a/bin/cdist +++ b/bin/cdist @@ -604,6 +604,10 @@ def banner(args): print(BANNER) sys.exit(0) +def install(args): + """Install remote system""" + process = {} + def config(args): """Configure remote system""" process = {} @@ -648,25 +652,33 @@ if __name__ == "__main__": add_help=False) parser['banner'].set_defaults(func=banner) - # Config - parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most']]) - parser['config'].add_argument('host', nargs='+', + # Config and install (common stuff) + parser['configinstall'] = argparse.ArgumentParser(add_help=False) + parser['configinstall'].add_argument('host', nargs='+', help='one or more hosts to operate on') - parser['config'].add_argument('-c', '--cdist-home', + parser['configinstall'].add_argument('-c', '--cdist-home', help='Change cdist home (default: .. from bin directory)', action='store') - parser['config'].add_argument('-i', '--initial-manifest', + parser['configinstall'].add_argument('-i', '--initial-manifest', help='Path to a cdist manifest', dest='manifest', required=False) - parser['config'].add_argument('-p', '--parallel', + parser['configinstall'].add_argument('-p', '--parallel', help='Operate on multiple hosts in parallel', action='store_true', dest='parallel') - parser['config'].add_argument('-s', '--sequential', + parser['configinstall'].add_argument('-s', '--sequential', help='Operate on multiple hosts sequentially (default)', action='store_false', dest='parallel') + + # Config + parser['config'] = parser['sub'].add_parser('config', + parents=[parser['most'], parser['configinstall']]) parser['config'].set_defaults(func=config) + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['most'], parser['configinstall']]) + parser['install'].set_defaults(func=install) + for p in parser: parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" From b2546fb236c1608a3d01840992d5330a0959b35f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 13:24:40 +0200 Subject: [PATCH 02/10] split deploy to into two stages Signed-off-by: Nico Schottelius --- bin/cdist | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 49985c01..a4570a34 100755 --- a/bin/cdist +++ b/bin/cdist @@ -555,10 +555,8 @@ class Cdist: self.transfer_file(local_remote_code, remote_remote_code) self.run_or_fail([remote_remote_code], remote=True) - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" self.init_deploy() self.run_global_explores() self.run_initial_manifest() @@ -580,14 +578,24 @@ class Cdist: self.objects_prepared.append(cdist_object) objects = self.list_objects() - + + def stage_run(self): + """The final (and real) step of deployment""" log.debug("Actual run objects") # Now do the final steps over the existing objects - for cdist_object in objects: + for cdist_object in self.list_objects(): log.debug("Run object: %s", cdist_object) self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + time_end = datetime.datetime.now() duration = time_end - time_start log.info("Finished run of %s in %s seconds", From 182074bb9d2908cdd16b2ce8d70fd4797843058a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 13:34:10 +0200 Subject: [PATCH 03/10] remove unused type template (to be reimported, if someone has a use for it) Signed-off-by: Nico Schottelius --- bin/cdist-type-template | 83 --------------------------- doc/man/man1/cdist-type-template.text | 30 ---------- 2 files changed, 113 deletions(-) delete mode 100755 bin/cdist-type-template delete mode 100644 doc/man/man1/cdist-type-template.text diff --git a/bin/cdist-type-template b/bin/cdist-type-template deleted file mode 100755 index 6d8a3f15..00000000 --- a/bin/cdist-type-template +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Create a new type from scratch -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -eu - -__cdist_type="$1"; shift -__cdist_my_type_dir="$(__cdist_type_dir "$__cdist_type")" - -if [ -d "$__cdist_my_type_dir" ]; then - __cdist_usage "Type $__cdist_type already exists" -fi - -echo "Creating type $__cdist_type in $__cdist_my_type_dir ..." -# Base -mkdir -p "$__cdist_my_type_dir" - -# Parameter -mkdir -p "$(__cdist_type_parameter_dir "$__cdist_type")" -touch "$(__cdist_type_parameter_dir "$__cdist_type")/${__cdist_name_parameter_required}" -touch "$(__cdist_type_parameter_dir "$__cdist_type")/${__cdist_name_parameter_optional}" - -# Manifest -cat "$__cdist_abs_mydir/../doc/dev/header" - << eof > "$__cdist_my_type_dir/${__cdist_name_manifest}" - -# -# This is the manifest, which can be used to create other objects like this: -# __file /path/to/destination --source /from/where/ -# -# To tell cdist to make use of it, you need to make it executable (chmod +x) -# -# - -eof - -# Gencode remote -cat "$__cdist_abs_mydir/../doc/dev/header" - << eof > "$(__cdist_type_dir "$__cdist_type")/${__cdist_name_gencode}-${__cdist_name_gencode_remote}" - -# -# This file should generate code on stdout, which will be collected by cdist -# and run on the target. -# -# To tell cdist to make use of it, you need to make it executable (chmod +x) -# -# - -eof - -cat "$__cdist_abs_mydir/../doc/dev/header" - << eof > "$(__cdist_type_dir "$__cdist_type")/${__cdist_name_gencode}-${__cdist_name_gencode_local}" - -# -# This file should generate code on stdout, which will be collected by cdist -# and run on the same machine cdist-deploy-to is executed. -# -# To tell cdist to make use of it, you need to make it executable (chmod +x) -# -# - -eof - -# Explorer -mkdir -p "$__cdist_my_type_dir/${__cdist_name_explorer}" diff --git a/doc/man/man1/cdist-type-template.text b/doc/man/man1/cdist-type-template.text deleted file mode 100644 index bbd31409..00000000 --- a/doc/man/man1/cdist-type-template.text +++ /dev/null @@ -1,30 +0,0 @@ -cdist-type-template(1) -====================== -Nico Schottelius - - -NAME ----- -cdist-type-template - Create a new type - - -SYNOPSIS --------- -cdist-type-template NAME - - -DESCRIPTION ------------ -cdist-type-template creates a new type and adds the usual files to it. -It is thought to be helpful when writing new types. - - -SEE ALSO --------- -cdist(7) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). From d0abfbf01b31ba1573783d0c6f78241a2ac4268a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 14:17:45 +0200 Subject: [PATCH 04/10] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index beae3cb2..deab8801 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,9 +1,20 @@ +- Initial install support + - setup $__install = "" for + manifest(s) + + - run standard manifest (?) + - creates initial objects + - only those having the installer flag? + - requires changegs to cdist-type-emulator! + - Goto Rewrite cdist-type-emulator + + - run all other manifests + - creates all objects + - what about type explorer? + - Support parallel execution - and maximum number of parallel runs (-p X) - error handling / report failed hosts -- Support different home instead of ../ -- Bug: os.path.join() may be wrong for the remote side! - -> does not matter for now! - Rewrite cdist-type-emulator - Remove legacy code in cdist @@ -11,8 +22,6 @@ - Remove man1/cdist-type-emulator.text - Remove the PATH=... part from the README -- Replace bin/cdist-type-template - - Allow manifest to be read from stdin - Create new video for cdist 2.0.0 http://www.youtube.com/watch?v=PRMjzy48eTI From 76f3792885ed008523fa044b7657465552f6edf3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 14:47:50 +0200 Subject: [PATCH 05/10] prepare argv[0] parsing for emulator use Signed-off-by: Nico Schottelius --- bin/cdist | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index a4570a34..327d86be 100755 --- a/bin/cdist +++ b/bin/cdist @@ -25,6 +25,7 @@ import datetime import logging import multiprocessing import os +import re import subprocess import shutil import stat @@ -55,12 +56,12 @@ REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" +TYPE_PREFIX = "__" VERSION = "2.0.0" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() - class TypeEmulator: def __init__(self, name): self.name = name @@ -612,10 +613,6 @@ def banner(args): print(BANNER) sys.exit(0) -def install(args): - """Install remote system""" - process = {} - def config(args): """Configure remote system""" process = {} @@ -640,7 +637,16 @@ def config(args): log.info("Total processing time for %s host(s): %s", len(args.host), (time_end - time_start).total_seconds()) -if __name__ == "__main__": +def install(args): + """Install remote system""" + process = {} + +def emulator(): + """Emulate type commands (i.e. __file and co)""" + process = {} + +def commandline(): + """Parse command line""" # Construct parser others can reuse parser = {} # Options _all_ parsers have in common @@ -702,3 +708,9 @@ if __name__ == "__main__": args.func(args) except KeyboardInterrupt: sys.exit(0) + +if __name__ == "__main__": + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): + emulator() + else: + commandline() From 4956c015c15d7284c00a0e8962e79001d396f25f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 14:50:43 +0200 Subject: [PATCH 06/10] make __type available for type gencode Signed-off-by: Nico Schottelius --- bin/cdist | 1 + doc/man/cdist-reference.text.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 327d86be..40b8dd5e 100755 --- a/bin/cdist +++ b/bin/cdist @@ -514,6 +514,7 @@ class Cdist: env["__object"] = self.object_dir(cdist_object) env["__object_id"] = self.get_object_id_from_object(cdist_object) env["__object_fq"] = cdist_object + env["__type"] = self.type_dir(get_type_from_object(cdist_object)) if mode == "gencode": paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index 218b0f57..e38f157d 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -182,7 +182,7 @@ __target_user:: Currently static in core. __type:: Path to the current type. - Available for: type manifest + Available for: type manifest, type gencode __type_explorer:: Directory that contains the type explorers. Available for: type explorer From dadc3ffe941dd115bc4a77ec68e07279041cce5f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 15:01:33 +0200 Subject: [PATCH 07/10] self.... Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 40b8dd5e..05eb8c00 100755 --- a/bin/cdist +++ b/bin/cdist @@ -514,7 +514,7 @@ class Cdist: env["__object"] = self.object_dir(cdist_object) env["__object_id"] = self.get_object_id_from_object(cdist_object) env["__object_fq"] = cdist_object - env["__type"] = self.type_dir(get_type_from_object(cdist_object)) + env["__type"] = self.type_dir(self.get_type_from_object(cdist_object)) if mode == "gencode": paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) @@ -644,7 +644,7 @@ def install(args): def emulator(): """Emulate type commands (i.e. __file and co)""" - process = {} + type = os.path.basename(sys.argv[0]) def commandline(): """Parse command line""" From eb3982b059f70ef3460c3ab796cae3a89469923b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 15:10:24 +0200 Subject: [PATCH 08/10] introduce more generic type_attribute_path() and replace type_manifest_path() Signed-off-by: Nico Schottelius --- bin/cdist | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 05eb8c00..3708e6a6 100755 --- a/bin/cdist +++ b/bin/cdist @@ -335,9 +335,9 @@ class Cdist: return [os.path.join(self.type_base_dir, type, "gencode-local"), os.path.join(self.type_base_dir, type, "gencode-remote")] - def type_manifest_path(self, type): - """Return path to manifest of type""" - return os.path.join(self.type_base_dir, type, "manifest") + def type_attribute_path(self, type, attribute): + """Return path of an attribute of a type""" + return os.path.join(self.type_base_dir, type, attribute) def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" @@ -446,7 +446,7 @@ class Cdist: def run_type_manifest(self, cdist_object): """Run manifest for a specific object""" type = self.get_type_from_object(cdist_object) - manifest = self.type_manifest_path(type) + manifest = self.type_attribute_path(type, "manifest") log.debug("%s: Running %s", cdist_object, manifest) if os.path.exists(manifest): From d979bdc5149fee855edfafb7a26445ceec73f5d9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 15:12:12 +0200 Subject: [PATCH 09/10] also replace self.type_explorer_dir Signed-off-by: Nico Schottelius --- bin/cdist | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 3708e6a6..2e094fcf 100755 --- a/bin/cdist +++ b/bin/cdist @@ -253,7 +253,7 @@ class Cdist: def list_type_explorers(self, type): """Return list of available explorers for a specific type""" - dir = self.type_explorer_dir(type) + dir = self.type_attribute_path(type, "explorer") if os.path.isdir(dir): list = os.listdir(dir) else: @@ -326,10 +326,6 @@ class Cdist: """Return directory the type""" return os.path.join(self.type_base_dir, type) - def type_explorer_dir(self, type): - """Return directory that holds the explorers of a type""" - return os.path.join(self.type_dir(type), "explorer") - def type_gencode_paths(self, type): """Return paths to gencode scripts of type""" return [os.path.join(self.type_base_dir, type, "gencode-local"), @@ -366,7 +362,7 @@ class Cdist: # Do not retransfer self.type_explorers_transferred[type] = 1 - src = self.type_explorer_dir(type) + src = self.type_attribute_path(type, attribute) remote_base = os.path.join(REMOTE_TYPE_DIR, type) dst = self.remote_type_explorer_dir(type) From 40d494ace28023ba7c42a6bdef5baef2abab79c5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 15:20:59 +0200 Subject: [PATCH 10/10] and now replace the new cool function type_attribute_path() with easy type_dir Signed-off-by: Nico Schottelius --- bin/cdist | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/bin/cdist b/bin/cdist index 2e094fcf..39452352 100755 --- a/bin/cdist +++ b/bin/cdist @@ -253,7 +253,7 @@ class Cdist: def list_type_explorers(self, type): """Return list of available explorers for a specific type""" - dir = self.type_attribute_path(type, "explorer") + dir = self.type_dir(type, "explorer") if os.path.isdir(dir): list = os.listdir(dir) else: @@ -322,18 +322,9 @@ class Cdist: return objects - def type_dir(self, type): + def type_dir(self, type, *args): """Return directory the type""" - return os.path.join(self.type_base_dir, type) - - def type_gencode_paths(self, type): - """Return paths to gencode scripts of type""" - return [os.path.join(self.type_base_dir, type, "gencode-local"), - os.path.join(self.type_base_dir, type, "gencode-remote")] - - def type_attribute_path(self, type, attribute): - """Return path of an attribute of a type""" - return os.path.join(self.type_base_dir, type, attribute) + return os.path.join(self.type_base_dir, type, *args) def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" @@ -362,7 +353,7 @@ class Cdist: # Do not retransfer self.type_explorers_transferred[type] = 1 - src = self.type_attribute_path(type, attribute) + src = self.type_dir(type, "explorer") remote_base = os.path.join(REMOTE_TYPE_DIR, type) dst = self.remote_type_explorer_dir(type) @@ -442,7 +433,7 @@ class Cdist: def run_type_manifest(self, cdist_object): """Run manifest for a specific object""" type = self.get_type_from_object(cdist_object) - manifest = self.type_attribute_path(type, "manifest") + manifest = self.type_dir(type, "manifest") log.debug("%s: Running %s", cdist_object, manifest) if os.path.exists(manifest): @@ -496,6 +487,7 @@ class Cdist: """Run gencode or code for an object""" log.debug("Running %s from %s", mode, cdist_object) requirements = self.list_object_requirements(cdist_object) + type = self.get_type_from_object(cdist_object) for requirement in requirements: log.debug("Object %s requires %s", cdist_object, requirement) @@ -510,10 +502,13 @@ class Cdist: env["__object"] = self.object_dir(cdist_object) env["__object_id"] = self.get_object_id_from_object(cdist_object) env["__object_fq"] = cdist_object - env["__type"] = self.type_dir(self.get_type_from_object(cdist_object)) + env["__type"] = self.type_dir(type) if mode == "gencode": - paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) + paths = [ + self.type_dir(type, "gencode-local"), + self.type_dir(type, "gencode-remote") + ] for bin in paths: if os.path.isfile(bin): # omit "gen" from gencode and