Merge remote-tracking branch 'telmich/master' into oo-restructure

bin/cdist

@@ -21,6 +21,7 @@
 #
 
 import argparse
+import datetime
 import logging
 import os
 import re
@@ -105,6 +106,8 @@ if __name__ == "__main__":
     try:
         logging.basicConfig(format='%(levelname)s: %(message)s')
 
+        time_start = datetime.datetime.now()
+
         if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])):
             import cdist.emulator
             cdist.emulator.run(sys.argv)
@@ -117,6 +120,14 @@ if __name__ == "__main__":
             import cdist.path
 
             commandline()
+
+        time_end = datetime.datetime.now()
+        duration = time_end - time_start
+        # FIXME: move into runner
+        # log.info("Finished run of %s in %s seconds", self.target_host,
+        #     duration.total_seconds())
+        log.info("Finished run in %s seconds", duration.total_seconds())
+
     except KeyboardInterrupt:
         sys.exit(0)
     except cdist.Error as e:

doc/changelog

@@ -1,6 +1,7 @@
 2.0.3:
 	* Improved logging, added --verbose, by more quiet by default
 	* Bugfix __user: Correct quoting (Steven Armstrong)
+	* FIXME: Support for __remote_exec and __remote_copy
 
 2.0.2: 2011-09-27
 	* Add support for detection of OpenWall Linux (Matthias Teege)

doc/dev/logs/2011-10-06

@@ -0,0 +1,30 @@
+GlobalExplorer
+   list_explorers()
+   list_explorers_names()
+
+   base_dir
+   __init__(name)
+   out_dir
+   env
+   name = id
+   path
+   return_code
+   return_value
+
+--------------------------------------------------------------------------------
+Exec:
+
+normal:
+
+scp /from/where $USER@$HOST:REMOTE_BASE/cdist-internal
+ssh $USER@$HOST MY_CMD_THAT_NEEDS_TO_RUN_IN_BIN_SH (including ENV)
+
+sudo:
+
+scp $USER@$HOST:REMOTE_BASE/cdist-internal
+ssh $USER@$HOST sudo MY_CMD_THAT_NEEDS_TO_RUN_IN_BIN_SH (including ENV)
+
+chroot:
+
+[sudo] cp file /chroot/THE_HOST_BASE/REMOTE_BASE/cdist-internal
+[sudo] chroot /chroot MY_CMD_THAT_NEEDS_TO_RUN_IN_BIN_SH (including ENV)

doc/dev/logs/2011-10-06.ssh_scp_sudo_chroot

@@ -0,0 +1,87 @@
+Commands needed:
+   conf/cmd/remote_exec
+   conf/cmd/copy
+
+If ! conf/cmd/remote_exec:
+   use builtin
+If ! conf/cmd/copy:
+   use builtin
+
+--------------------------------------------------------------------------------
+
+--cmd-dir?
+$__cdist_cmd_dir
+
+--------------------------------------------------------------------------------
+   -> Depend on session!
+
+Builtin:
+        cdist.exec.run_or_fail(["scp", "-qr", source, 
+                                self.remote_user + "@" + 
+                                self.target_host + ":" + 
+                                destination])
+
+self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host]
+
+        self.remote_user = remote_user
+        self.remote_prefix = remote_prefix
+
+--------------------------------------------------------------------------------
+What is in a session?
+
+   base_dir
+   target_host
+
+--------------------------------------------------------------------------------
+remote_user 
+   pseudo-static, can be hardcoded again
+--------------------------------------------------------------------------------
+
+Result:
+
+os.environ['__remote_exec'] = ["ssh", "-l", "root" ]
+
+os.environ['__remote_exec'] = ["ssh", "-o", "User=root" ]
+os.environ['__remote_copy'] = ["scp", "-o", "User=root" ]
+
+
+__remote_exec=~/sudossh __remote_copy=... cdist config localhost
+
+~/sudossh hostname $@...
+~/sudocopy a hostname:b
+
+~/chrootssh
+~/chrootcopy
+
+
+
+a)
+   3 cmd verzeichnnise: cdist, sudo, chroot
+   pro aufruf variable ändern
+
+b)
+   1 dir, mit zeug
+   pro aufruf variablen ändern
+
+
+conf/cmd/remote_exec 
+
+args for __remote_exec
+   $1 = hostname
+   $2 - ...  = stuff to be executed in /bin/sh on remote side
+   $2 - $7 = env
+   $7 - 12 = cmd
+
+
+args for __remote_copy
+   $1 = file here
+   $2 = hostname:destination
+
+--------------------------------------------------------------------------------
+There needs to be an easy way to change those cmds!
+--------------------------------------------------------------------------------
+Env-Passing:
+   _a=b test -> test can access $_a
+   _a=b test $_a -> $1 = "", because _a is *not* set within the shell
+   _a=b; test -> can access $_a
+   _a=b; test $_a -> $1 == "b"

doc/dev/todo/niconext

@@ -1,3 +1,18 @@
+2.0.3:
+
+- fix emulator
+- introduce tests:
+   - does $require work?
+   - $whatever should fail if there is no global explorer directory
+   - emulator may only be called with __ as prefix - fail otherwise!
+
+- Create GlobalExplorer
+   
+--------------------------------------------------------------------------------
+
+- insert prefix into logger to distinguish between modules
+   - in debug/info only?
+
 - Fix / rewrite cdist-quickstart
 
 - write tutorial!!!!!!!!!

doc/man/man1/cdist-config.text

@@ -1,48 +0,0 @@
-cdist-config(1)
-===============
-Nico Schottelius <nico-cdist--@--schottelius.org>
-
-
-NAME
-----
-cdist-config - Read basic cdist configuration
-
-
-DESCRIPTION
------------
-Cdist-config is sourced by cdist programs and provides hints on where to find
-types, manifests, etc. Generally speaking, it's just usable from within the
-core and is only of interest for cdist-developers.
-
-
-ENVIRONMENT VARIABLES
----------------------
-The following list contains environment variables that are known
-to be changed by users in various situations. To change the variable,
-use your current shell and export it, so all cdist-binaries know about it.
-
-__cdist_tmp_base_dir::
-   Normally this points to /tmp. In case /tmp is not suitable for
-   cdist (i.e. has noexec flag setup) you can change this variable
-   to point to a better location.
-
-
-EXAMPLES
---------
-
-If /tmp has the noexec flag, you can use $HOME/.tmp for instance:
-
---------------------------------------------------------------------------------
-export __cdist_tmp_base_dir=$HOME/.tmp
---------------------------------------------------------------------------------
-
-
-SEE ALSO
---------
-cdist(7)
-
-
-COPYING
--------
-Copyright \(C) 2010-2011 Nico Schottelius. Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).

doc/man/man1/cdist-env.text

@@ -1,49 +0,0 @@
-cdist-env(1)
-============
-Nico Schottelius <nico-cdist--@--schottelius.org>
-
-
-NAME
-----
-cdist-env - Setup environment for using cdist
-
-
-SYNOPSIS
---------
-cdist-env
-
-
-DESCRIPTION
------------
-cdist-env outputs two strings suitable for usage in your current shell,
-so you can use cdist from the checkout. cdist-env essentially helps you
-to easily setup PATH and MANPATH.
-
-If you've multiple checkouts of cdist and run cdist-env from the various
-checkouts, a new run will prepend the last directory, thus ensures you
-can run it multiple times and does what one expects.
-
-EXAMPLES
---------
-For use in bourne shell variants (like dash, bash, ksh) as well as
-in csh variants (csh, tcsh):
-
---------------------------------------------------------------------------------
-eval `./bin/cdist-env`
---------------------------------------------------------------------------------
-
-For bourne shell, there is also a shorter version:
---------------------------------------------------------------------------------
-. ./bin/cdist-env
---------------------------------------------------------------------------------
-
-
-SEE ALSO
---------
-cdist(7)
-
-
-COPYING
--------
-Copyright \(C) 2011 Nico Schottelius. Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).

doc/man/man1/cdist-quickstart.text

@@ -1,47 +0,0 @@
-cdist-quickstart(1)
-===================
-Nico Schottelius <nico-cdist--@--schottelius.org>
-
-NAME
-----
-cdist-quickstart - Make use of cinit in 5 minutes
-
-
-SYNOPSIS
---------
-cdist-quickstart
-
-
-DESCRIPTION
------------
-cdist-quickstart is an interactive guide to cdist. It should be one
-of the first tools you use when you begin with cdist.
-
-
-EXAMPLES
---------
-To use cdist-quickstart, add the bin directory to your PATH, execute
-cdist-quickstart and enjoy cdist:
-
---------------------------------------------------------------------------------
-# Bourne shell example
-export PATH=$(pwd -P)/bin:$PATH
-
-# Alternatively, usable for csh and bsh, set's up PATH and MANPATH
-eval `./bin/cdist-env`
-
-# Let's go!
-cdist-quickstart
---------------------------------------------------------------------------------
-
-
-SEE ALSO
---------
-- cdist(7)
-- cdist-env(1)
-
-
-COPYING
--------
-Copyright \(C) 2010-2011 Nico Schottelius. Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).

doc/man/man1/cdist-type-emulator.text

@@ -1,56 +0,0 @@
-cdist-type-emulator(1)
-======================
-Nico Schottelius <nico-cdist--@--schottelius.org>
-
-
-NAME
-----
-cdist-type-emulator - Emulate type and record parameters and dependencies
-
-
-SYNOPSIS
---------
-cdist-type-emulator [TYPE ARGS]
-
-
-DESCRIPTION
------------
-cdist-type-emulator is normally called through a link to it of the
-name of a specifc type. It saves the given parameters into
-a parameters directory and the requirements into a require file.
-
-It checks whether the parameters are valid:
-
-- are required parameter given?
-- are all other required parameters specified as optional?
-
-
-EXAMPLES
---------
-Your manifest may contain stuff like this:
-
-
---------------------------------------------------------------------------------
-__addifnosuchline /tmp/linetest --line "test"
-
-__motd
---------------------------------------------------------------------------------
-
-In both cases, cdist-type-emulator is called instead of a real type.
-In the first case, the object id "/tmp/linetest" is recorded and the
-parameter "line" stored with the content "test".
-
-In the second case, __motd must be decleared as a singleton, as the
-object id is missing.
-
-
-SEE ALSO
---------
-- cdist(7)
-- cdist-type-build-emulation(1)
-
-
-COPYING
--------
-Copyright \(C) 2011 Nico Schottelius. Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).

lib/cdist/__init__.py

@@ -33,4 +33,4 @@ class MissingEnvironmentVariableError(Error):
         self.name = name
 
     def __str__(self):
-        return 'Missing required environment variable: {0.name}'.format(o)
+        return 'Missing required environment variable: ' + str(self.name)

lib/cdist/config.py

@@ -22,10 +22,13 @@
 
 import datetime
 import logging
-log = logging.getLogger(__name__)
+import os
+import sys
 
 import cdist.config_install
 
+log = logging.getLogger(__name__)
+
 class Config(cdist.config_install.ConfigInstall):
     pass
 
@@ -35,6 +38,9 @@ def config(args):
 
     time_start = datetime.datetime.now()
 
+    os.environ['__remote_exec'] = "ssh -o User=root -q"
+    os.environ['__remote_copy'] = "scp -o User=root -q"
+
     for host in args.host:
         c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug)
         if args.parallel:

lib/cdist/config_install.py

@@ -20,7 +20,6 @@
 #
 #
 
-import datetime
 import logging
 import os
 import stat
@@ -29,278 +28,253 @@ import sys
 import cdist.emulator
 import cdist.path
 
+import cdist.core
+
 log = logging.getLogger(__name__)
 
-CODE_HEADER                  = "#!/bin/sh -e\n"
+CODE_HEADER = "#!/bin/sh -e\n"
 
 class ConfigInstall:
- """Class to hold install and config methods"""
+    """Cdist main class to hold arbitrary data"""
 
- def __init__(self, target_host, 
+    def __init__(self, target_host, 
-                 initial_manifest=False,
+                    initial_manifest=False,
-                 remote_user="root",
+                    home=None,
-                 home=None,
+                    exec_path=sys.argv[0],
-                 exec_path=sys.argv[0],
+                    debug=False):
-                 debug=False):
 
-     self.target_host    = target_host
+        self.target_host    = target_host
-     self.debug          = debug
+        os.environ['target_host'] = target_host
-     self.remote_user    = remote_user
-     self.exec_path      = exec_path
 
-     # FIXME: broken - construct elsewhere!
+        self.debug          = debug
-     self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host]
+        self.exec_path      = exec_path
 
-     self.path = cdist.path.Path(self.target_host, 
+        self.path = cdist.path.Path(self.target_host, 
-                     initial_manifest=initial_manifest,
+                        initial_manifest=initial_manifest,
-                     remote_user=self.remote_user,
+                        base_dir=home,
-                     remote_prefix=self.remote_prefix,
+                        debug=debug)
-                     base_dir=home,
-                     debug=debug)
         
-     self.objects_prepared = []
+    def cleanup(self):
+        self.path.cleanup()
 
- def cleanup(self):
+    def run_global_explorers(self):
-     self.path.cleanup()
+        """Run global explorers"""
+        log.info("Running global explorers")
+        explorers = self.path.list_global_explorers()
+        if(len(explorers) == 0):
+            raise cdist.Error("No explorers found in " + self.path.global_explorer_dir)
 
- def run_global_explorers(self):
+        self.path.transfer_global_explorers()
-     """Run global explorers"""
+        for explorer in explorers:
-     log.info("Running global explorers")
+            output = self.path.global_explorer_output_path(explorer)
-     explorers = self.path.list_global_explorers()
+            output_fd = open(output, mode='w')
-     if(len(explorers) == 0):
+            cmd = []
-         raise CdistError("No explorers found in", self.path.global_explorer_dir)
+            cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR)
+            cmd.append(self.path.remote_global_explorer_path(explorer))
 
-     self.path.transfer_global_explorers()
+            cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=True)
-     for explorer in explorers:
+            output_fd.close()
-         output = self.path.global_explorer_output_path(explorer)
-         output_fd = open(output, mode='w')
-         cmd = []
-         cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR)
-         cmd.append(self.path.remote_global_explorer_path(explorer))
 
-         cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
+    def run_type_explorer(self, cdist_object):
-         output_fd.close()
+        """Run type specific explorers for objects"""
 
-# FIXME: where to call this from?
+        type = cdist_object.type
- def run_type_explorer(self, cdist_object):
+        self.path.transfer_type_explorers(type)
-     """Run type specific explorers for objects"""
 
-     type = self.path.get_type_from_object(cdist_object)
+        cmd = []
-     self.path.transfer_type_explorers(type)
+        cmd.append("__explorer="        + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR)
+        cmd.append("__type_explorer="   + self.path.remote_type_explorer_dir(type))
+        cmd.append("__object="          + self.path.remote_object_dir(cdist_object))
+        cmd.append("__object_id="       + self.path.get_object_id_from_object(cdist_object))
+        cmd.append("__object_fq="       + cdist_object)
 
-     cmd = []
+        # Need to transfer at least the parameters for objects to be useful
-     cmd.append("__explorer="        + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR)
+        self.path.transfer_object_parameter(cdist_object)
-     cmd.append("__type_explorer="   + self.path.remote_type_explorer_dir(type))
-     cmd.append("__object="          + self.path.remote_object_dir(cdist_object))
-     cmd.append("__object_id="       + self.path.get_object_id_from_object(cdist_object))
-     cmd.append("__object_fq="       + cdist_object)
 
-     # Need to transfer at least the parameters for objects to be useful
+        explorers = self.path.list_type_explorers(type)
-     self.path.transfer_object_parameter(cdist_object)
+        for explorer in explorers:
+            remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)]
+            output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer)
+            output_fd = open(output, mode='w')
+            log.debug("%s exploring %s using %s storing to %s", 
+                            cdist_object, explorer, remote_cmd, output)
                         
-     # FIXME: Broken due to refactoring into type.py
+            cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=True)
-     explorers = self.path.list_type_explorers(type)
+            output_fd.close()
-     for explorer in explorers:
-         remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)]
-         output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer)
-         output_fd = open(output, mode='w')
-         log.debug("%s exploring %s using %s storing to %s", 
-                         cdist_object, explorer, remote_cmd, output)
 
-         cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
+    def link_emulator(self):
-         output_fd.close()
+        """Link emulator to types"""
+        cdist.emulator.link(self.exec_path,
+            self.path.bin_dir, self.path.list_types())
 
- def link_emulator(self):
+    def run_initial_manifest(self):
-     """Link emulator to types"""
+        """Run the initial manifest"""
-     cdist.emulator.link(self.exec_path,
+        log.info("Running initial manifest %s", self.path.initial_manifest)
-         self.path.bin_dir, self.path.list_types())
+        env = {  "__manifest" : self.path.manifest_dir }
+        self.run_manifest(self.path.initial_manifest, extra_env=env)
 
- def init_deploy(self):
+    def run_type_manifest(self, cdist_object):
-     """Ensure the base directories are cleaned up"""
+        """Run manifest for a specific object"""
-     log.debug("Creating clean directory structure")
+        type = self.path.get_type_from_object(cdist_object)
+        manifest = self.path.type_dir(type, "manifest")
         
-     self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR)
+        log.debug("%s: Running %s", cdist_object, manifest)
-     self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR)
+        if os.path.exists(manifest):
-     self.link_emulator()
+            env = { "__object" :    self.path.object_dir(cdist_object), 
+                    "__object_id":  self.path.get_object_id_from_object(cdist_object),
+                    "__object_fq":  cdist_object,
+                    "__type":       self.path.type_dir(type)
+                    }
+            self.run_manifest(manifest, extra_env=env)
 
- def run_initial_manifest(self):
+    def run_manifest(self, manifest, extra_env=None):
-     """Run the initial manifest"""
+        """Run a manifest"""
-     log.info("Running initial manifest %s", self.path.initial_manifest)
+        log.debug("Running manifest %s, env=%s", manifest, extra_env)
-     env = {  "__manifest" : self.path.manifest_dir }
+        env = os.environ.copy()
-     self.run_manifest(self.path.initial_manifest, extra_env=env)
+        env['PATH'] = self.path.bin_dir + ":" + env['PATH']
 
- def run_type_manifest(self, cdist_object):
+        # Information required in every manifest
-     """Run manifest for a specific object"""
+        env['__target_host']            = self.target_host
-     type = self.path.get_type_from_object(cdist_object)
+        env['__global']                 = self.path.out_dir
-     manifest = self.path.type_dir(type, "manifest")
         
-     log.debug("%s: Running %s", cdist_object, manifest)
+        # Submit debug flag to manifest, can be used by emulator and types
-     if os.path.exists(manifest):
+        if self.debug:
-         env = { "__object" :    self.path.object_dir(cdist_object), 
+            env['__debug']                  = "yes"
-                 "__object_id":  self.path.get_object_id_from_object(cdist_object),
-                 "__object_fq":  cdist_object,
-                 "__type":       self.path.type_dir(type)
-                 }
-         self.run_manifest(manifest, extra_env=env)
 
- def run_manifest(self, manifest, extra_env=None):
+        # Required for recording source
-     """Run a manifest"""
+        env['__cdist_manifest']         = manifest
-     log.debug("Running manifest %s, env=%s", manifest, extra_env)
-     env = os.environ.copy()
-     env['PATH'] = self.path.bin_dir + ":" + env['PATH']
 
-     # Information required in every manifest
+        # Required to find types
-     env['__target_host']            = self.target_host
+        env['__cdist_type_base_dir']    = self.path.type_base_dir
-     env['__global']                 = self.path.out_dir
 
-     # Submit debug flag to manifest, can be used by emulator and types
+        # Other environment stuff
-     if self.debug:
+        if extra_env:
-         env['__debug']                  = "yes"
+            env.update(extra_env)
 
-     # Required for recording source
+        cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env)
-     env['__cdist_manifest']         = manifest
 
-     # Required to find types
+    def object_run(self, cdist_object, mode):
-     env['__cdist_type_base_dir']    = self.path.type_base_dir
+        """Run gencode or code for an object"""
+        log.debug("Running %s from %s", mode, cdist_object)
 
-     # Other environment stuff
+        # FIXME: replace with new object interface
-     if extra_env:
+        file=os.path.join(self.path.object_dir(cdist_object), "require")
-         env.update(extra_env)
+        requirements = cdist.path.file_to_list(file)
+        type = self.path.get_type_from_object(cdist_object)
             
-     cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env)
+        for requirement in requirements:
+            log.debug("Object %s requires %s", cdist_object, requirement)
+            self.object_run(requirement, mode=mode)
 
- def object_run(self, cdist_object, mode):
+        #
-     """Run gencode or code for an object"""
+        # Setup env Variable:
-     log.debug("Running %s from %s", mode, cdist_object)
+        #
-     file=os.path.join(self.path.object_dir(cdist_object), "require")
+        env = os.environ.copy()
-     requirements = cdist.path.file_to_list(file)
+        env['__target_host']    = self.target_host
-     type = self.path.get_type_from_object(cdist_object)
+        env['__global']         = self.path.out_dir
+        env["__object"]         = self.path.object_dir(cdist_object)
+        env["__object_id"]      = self.path.get_object_id_from_object(cdist_object)
+        env["__object_fq"]      = cdist_object
+        env["__type"]           = self.path.type_dir(type)
 
-     for requirement in requirements:
+        if mode == "gencode":
-         log.debug("Object %s requires %s", cdist_object, requirement)
+            paths = [
-         self.object_run(requirement, mode=mode)
+                self.path.type_dir(type, "gencode-local"),
+                self.path.type_dir(type, "gencode-remote")
+            ]
+            for bin in paths:
+                if os.path.isfile(bin):
+                    # omit "gen" from gencode and use it for output base
+                    outfile=os.path.join(self.path.object_dir(cdist_object), 
+                        os.path.basename(bin)[3:])
 
-     #
+                    outfile_fd = open(outfile, "w")
-     # Setup env Variable:
-     # 
-     env = os.environ.copy()
-     env['__target_host']    = self.target_host
-     env['__global']         = self.path.out_dir
-     env["__object"]         = self.path.object_dir(cdist_object)
-     env["__object_id"]      = self.path.get_object_id_from_object(cdist_object)
-     env["__object_fq"]      = cdist_object
-     env["__type"]           = self.path.type_dir(type)
 
-     if mode == "gencode":
+                    # Need to flush to ensure our write is done before stdout write
-         paths = [
+                    outfile_fd.write(CODE_HEADER)
-             self.path.type_dir(type, "gencode-local"),
+                    outfile_fd.flush()
-             self.path.type_dir(type, "gencode-remote")
-         ]
-         for bin in paths:
-             if os.path.isfile(bin):
-                 # omit "gen" from gencode and use it for output base
-                 outfile=os.path.join(self.path.object_dir(cdist_object), 
-                     os.path.basename(bin)[3:])
 
-                 outfile_fd = open(outfile, "w")
+                    cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd)
+                    outfile_fd.close()
 
-                 # Need to flush to ensure our write is done before stdout write
+                    status = os.stat(outfile)
-                 # FIXME: CODE_HEADER needed in our sh -e scenario?
-                 outfile_fd.write(CODE_HEADER)
-                 outfile_fd.flush()
 
-                 cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd)
+                    # Remove output if empty, else make it executable
-                 outfile_fd.close()
+                    if status.st_size == len(CODE_HEADER):
+                        os.unlink(outfile)
+                    else:
+                        # Add header and make executable - identically to 0o700
+                        os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
 
-                 status = os.stat(outfile)
+                        # Mark object as changed
-
+                        open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close()
-                 # Remove output if empty, else make it executable
-                 if status.st_size == len(CODE_HEADER):
-                     os.unlink(outfile)
-                 else:
-                     # Add header and make executable - identically to 0o700
-                     os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
-
-                     # Mark object as changed
-                     open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close()
 
 
-     if mode == "code":
+        if mode == "code":
-         local_dir   = self.path.object_dir(cdist_object)
+            local_dir   = self.path.object_dir(cdist_object)
-         remote_dir  = self.path.remote_object_dir(cdist_object)
+            remote_dir  = self.path.remote_object_dir(cdist_object)
 
-         bin = os.path.join(local_dir, "code-local")
+            bin = os.path.join(local_dir, "code-local")
-         if os.path.isfile(bin):
+            if os.path.isfile(bin):
-             cdist.exec.run_or_fail([bin])
+                cdist.exec.run_or_fail([bin])
                 
 
-         local_remote_code = os.path.join(local_dir, "code-remote")
+            local_remote_code = os.path.join(local_dir, "code-remote")
-         remote_remote_code = os.path.join(remote_dir, "code-remote")
+            remote_remote_code = os.path.join(remote_dir, "code-remote")
-         if os.path.isfile(local_remote_code):
+            if os.path.isfile(local_remote_code):
-             self.path.transfer_file(local_remote_code, remote_remote_code)
+                self.path.transfer_file(local_remote_code, remote_remote_code)
-             # FIXME: remote_prefix
+                cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True)
-             cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix)
                 
- def stage_prepare(self):
+    ### Cleaned / check functions: Round 1 :-) #################################
-     """Do everything for a deploy, minus the actual code stage"""
+    def stage_run(self):
-     self.init_deploy()
+        """The final (and real) step of deployment"""
-     self.run_global_explorers()
+        log.info("Generating and executing code")
-     self.run_initial_manifest()
+        # Now do the final steps over the existing objects
+        for cdist_object in cdist.core.Object.list_objects():
+            log.debug("Run object: %s", cdist_object)
+            self.object_run(cdist_object, mode="gencode")
+            self.object_run(cdist_object, mode="code")
 
-     log.info("Running object manifests and type explorers")
+    def deploy_to(self):
+        """Mimic the old deploy to: Deploy to one host"""
+        log.info("Deploying to " + self.target_host)
+        self.stage_prepare()
+        self.stage_run()
 
-     old_objects = []
+    def deploy_and_cleanup(self):
-     objects = self.path.list_objects()
+        """Do what is most often done: deploy & cleanup"""
+        self.deploy_to()
+        self.cleanup()
 
-     # Continue process until no new objects are created anymore
+    def init_deploy(self):
-     while old_objects != objects:
+        """Ensure the base directories are cleaned up"""
-         old_objects = list(objects)
+        log.debug("Creating clean directory structure")
-         for cdist_object in objects:
-             if cdist_object in self.objects_prepared:
-                 log.debug("Skipping rerun of object %s", cdist_object)
-                 continue
-             else:
-                 # FIXME: run_type_explorer:
-                 # object can return type
-                 # type has explorers
-                 # path knows about where to save explorer output
-                 # type = self.path.objects[object].type()
-                 # self.path.types['type'].explorers()
-                 # for explorer in explorers:
-                 #  output = cdist.exec.run_debug_or_fail_shell(explorer) 
-                 #  if output:
-                 #      write_output_to(output, os.path.join(self.path.objects[object].explorer_dir(),explorer) )
-                 # 
-                 self.run_type_explorer(cdist_object)
-                 self.run_type_manifest(cdist_object)
-                 self.objects_prepared.append(cdist_object)
 
-         objects = self.path.list_objects()
+        self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR)
+        self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR)
+        self.link_emulator()
     
- def stage_run(self):
+    def stage_prepare(self):
-     """The final (and real) step of deployment"""
+        """Do everything for a deploy, minus the actual code stage"""
-     log.info("Generating and executing code")
+        self.init_deploy()
-     # Now do the final steps over the existing objects
+        self.run_global_explorers()
-     for cdist_object in self.path.list_objects():
+        self.run_initial_manifest()
-         log.debug("Run object: %s", cdist_object)
-         self.object_run(cdist_object, mode="gencode")
-         self.object_run(cdist_object, mode="code")
         
- def deploy_to(self):
+        log.info("Running object manifests and type explorers")
-     """Mimic the old deploy to: Deploy to one host"""
-     log.info("Deploying to " + self.target_host)
-     time_start = datetime.datetime.now()
 
-     self.stage_prepare()
+        log.debug("Searching for objects in " + cdist.core.Object.base_dir())
-     self.stage_run()
 
-     time_end = datetime.datetime.now()
+        # Continue process until no new objects are created anymore
-     duration = time_end - time_start
+        new_objects_created = True
-     log.info("Finished run of %s in %s seconds", 
+        while new_objects_created:
-         self.target_host,
+            new_objects_created = False
-         duration.total_seconds())
+            for cdist_object in cdist.core.Object.list_objects():
-
+                if cdist_object.prepared:
- def deploy_and_cleanup(self):
+                    log.debug("Skipping rerun of object %s", cdist_object)
-     """Do what is most often done: deploy & cleanup"""
+                    continue
-     self.deploy_to()
+                else:
-     self.cleanup()
+                    log.debug("Preparing object: " + cdist_object.name)
+                    self.run_type_explorer(cdist_object)
+                    self.run_type_manifest(cdist_object)
+                    cdist_object.prepared = True
+                    new_objects_created = True

lib/cdist/core/object.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -19,12 +20,14 @@
 #
 #
 
+import logging
 import os
 import collections
 
 import cdist
 import cdist.core.property
 
+log = logging.getLogger(__name__)
 
 DOT_CDIST = '.cdist'
 
@@ -65,7 +68,7 @@ class Object(object):
         for object_name in cls.list_object_names():
             type_name = object_name.split(os.sep)[0]
             object_id = os.sep.join(object_name.split(os.sep)[1:])
-            yield cls(Type(type_name), object_id=object_id)
+            yield cls(cdist.core.Type(type_name), object_id=object_id)
 
     @classmethod
     def list_type_names(cls):
@@ -90,6 +93,10 @@ class Object(object):
         self.__parameters = None
         self.__requirements = None
 
+        # Whether this object was prepared/ran
+        self.prepared = False
+        self.ran = False
+        
     def __repr__(self):
         return '<Object %s>' % self.name
 

lib/cdist/emulator.py

@@ -77,6 +77,8 @@ def run(argv):
 
     object_dir = os.path.join(global_dir, "object", type,
                             object_id, cdist.path.DOT_CDIST)
+    log.debug("Object output dir = " + object_dir)
+
     param_out_dir = os.path.join(object_dir, "parameter")
 
     object_source_file = os.path.join(object_dir, "source")
@@ -127,7 +129,7 @@ def run(argv):
                 param_fd.close()
 
     # Record requirements
-    if "__require" in os.environ:
+    if "require" in os.environ:
         requirements = os.environ['__require']
         log.debug(object_id + ":Writing requirements: " + requirements)
         require_fd = open(os.path.join(object_dir, "require"), "a")

lib/cdist/exec.py

@@ -20,6 +20,7 @@
 #
 
 import logging
+import os
 import subprocess
 
 log = logging.getLogger(__name__)
@@ -32,6 +33,8 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs):
     args[0][:0] = [ "/bin/sh", "-e" ]
 
     if remote_prefix:
+        remote_prefix = os.environ['__remote_exec'].split()
+        remote_prefix.append(os.environ['target_host'])
         args[0][:0] = remote_prefix
 
     log.debug("Shell exec cmd: %s", args)
@@ -43,6 +46,7 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs):
         subprocess.check_call(*args, **kargs)
     except subprocess.CalledProcessError:
         log.error("Code that raised the error:\n")
+
         if remote_prefix:
             run_or_fail(["cat", script], remote_prefix=remote_prefix)
 
@@ -60,6 +64,8 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs):
 
 def run_or_fail(*args, remote_prefix=False, **kargs):
     if remote_prefix:
+        remote_prefix = os.environ['__remote_exec'].split()
+        remote_prefix.append(os.environ['target_host'])
         args[0][:0] = remote_prefix
 
     log.debug("Exec: " + " ".join(*args))

lib/cdist/install.py

@@ -27,9 +27,6 @@ import cdist.config_install
 log = logging.getLogger(__name__)
 
 
-Class Install(cdist.config_install.ConfigInstall):
-    pass
-
 def install(args):
     """Install remote system"""
     process = {}

lib/cdist/object.py

@@ -1,51 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import os
-import logging
-log = logging.getLogger(__name__)
-
-
-
-class Object(object):
-
-    def __init__(self, path, remote_path, object_fq):
-        self.path = path
-        self.remote_path = remote_path
-        self.object_fq = object_fq
-        self.type = self.object_fq.split(os.sep)[0]
-        self.object_id = self.object_fq.split(os.sep)[1:]
-        self.parameter_dir = os.path.join(self.path, "parameter")
-        self.remote_object_parameter_dir = os.path.join(self.remote_path, "parameter")
-        self.object_code_paths = [
-            os.path.join(self.path, "code-local"),
-            os.path.join(self.path, "code-remote")]
-
-    @property
-    def type_explorer_output_dir(self):
-        """Returns and creates dir of the output for a type explorer"""
-        if not self.__type_explorer_output_dir:
-            dir = os.path.join(self.path, "explorer")
-            if not os.path.isdir(dir):
-                os.mkdir(dir)
-            self.__type_explorer_output_dir = dir
-        return self.__type_explorer_output_dir
-

lib/cdist/path.py

@@ -57,8 +57,6 @@ class Path:
 
     def __init__(self,
                 target_host,
-                remote_user,
-                remote_prefix,
                 initial_manifest=False,
                 base_dir=None,
                 debug=False):
@@ -70,10 +68,8 @@ class Path:
             self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
 
         self.temp_dir = tempfile.mkdtemp()
-        self.target_host = target_host
 
-        self.remote_user = remote_user
+        self.target_host = target_host
-        self.remote_prefix = remote_prefix
 
         # Input directories
         self.conf_dir               = os.path.join(self.base_dir, "conf")
@@ -96,6 +92,8 @@ class Path:
         self.object_base_dir = os.path.join(self.out_dir, "object")
         self.bin_dir = os.path.join(self.out_dir, "bin")
 
+        os.environ['__cdist_out_dir'] = self.out_dir
+
         # List of type explorers transferred
         self.type_explorers_transferred = {}
 
@@ -135,28 +133,24 @@ class Path:
     # FIXME: belongs to here - clearify remote*
     def remote_mkdir(self, directory):
         """Create directory on remote side"""
-        cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix)
+        cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True)
 
     # FIXME: belongs to here - clearify remote*
     def remove_remote_dir(self, destination):
-        cdist.exec.run_or_fail(["rm", "-rf",  destination], remote_prefix=self.remote_prefix)
+        cdist.exec.run_or_fail(["rm", "-rf",  destination], remote_prefix=True)
 
     # FIXME: belongs to here - clearify remote*
     def transfer_dir(self, source, destination):
         """Transfer directory and previously delete the remote destination"""
         self.remove_remote_dir(destination)
-        cdist.exec.run_or_fail(["scp", "-qr", source, 
+        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
-                                self.remote_user + "@" + 
+            ["-r", source, self.target_host + ":" + destination])
-                                self.target_host + ":" + 
-                                destination])
 
     # FIXME: belongs to here - clearify remote*
     def transfer_file(self, source, destination):
         """Transfer file"""
-        cdist.exec.run_or_fail(["scp", "-q", source, 
+        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
-                                self.remote_user + "@" +
+            [source, self.target_host + ":" + destination])
-                                self.target_host + ":" +
-                                destination])
 
     # FIXME: Explorer or stays
     def global_explorer_output_path(self, explorer):
@@ -198,11 +192,6 @@ class Path:
 
         return object_paths
 
-    # FIXME: Object
-    def get_type_from_object(self, cdist_object):
-        """Returns the first part (i.e. type) of an object"""
-        return cdist_object.split(os.sep)[0]
-
     # FIXME: Object
     def get_object_id_from_object(self, cdist_object):
         """Returns everything but the first part (i.e. object_id) of an object"""
@@ -266,19 +255,19 @@ class Path:
     # Stays here - FIXME: adjust to type code, loop over types!
     def transfer_type_explorers(self, type):
         """Transfer explorers of a type, but only once"""
-        if type in self.type_explorers_transferred:
+        if type.transferred:
             log.debug("Skipping retransfer for explorers of %s", type)
             return
         else:
             # Do not retransfer
-            self.type_explorers_transferred[type] = 1
+            type.transferred = True
 
-        src = self.type_dir(type, "explorer")
+        # FIXME: Can be explorer_path or explorer_dir, I don't care.
-        remote_base = os.path.join(REMOTE_TYPE_DIR, type)
+        src = type.explorer_path()
-        dst = self.remote_type_explorer_dir(type)
+        dst = type.remote_explorer_path()
 
-        # Only continue, if there is at least the directory
+        # Transfer if there is at least one explorer
-        if os.path.isdir(src):
+        if len(type.explorers) > 0:
             # Ensure that the path exists
-            self.remote_mkdir(remote_base)
+            self.remote_mkdir(dst)
             self.transfer_dir(src, dst)

lib/cdist/type.py

@@ -1,51 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2011 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import logging
-import os
-log = logging.getLogger(__name__)
-
-class Type(object):
-
-    def __init__(self, path, remote_path):
-        self.path = path
-        self.remote_path = remote_path
-
-    def list_explorers(self):
-        """Return list of available explorers"""
-        dir = os.path.join(self.path, "explorer")
-        if os.path.isdir(dir):
-            list = os.listdir(dir)
-        else:
-            list = []
-
-        log.debug("Explorers for %s in %s: %s", type, dir, list)
-
-        return list
-
-    def is_install(self):
-        """Check whether a type is used for installation (if not: for configuration)"""
-        return os.path.isfile(os.path.join(self.path, "install"))
-
-    def remote_explorer_dir(self):
-        """Return remote directory that holds the explorers of a type"""
-        return os.path.join(self.remote_path, "explorer")

other/types_pending_inclusion/__init_script/README

@@ -0,0 +1,4 @@
+Moved out of conf/type/ to think about whether this type makes sense or not.
+
+Cdist describes the state and using an init_script may be useful, but
+should only be used conditionally.

other/types_pending_inclusion/__init_script/gencode-remote

@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+# 2010-2011 Daniel Roth (dani-cdist@d-roth.li)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+if [ -f "$__object/parameter/script" ]; then
+   script=$(cat "$__object/parameter/script")
+else
+   script="/$__object_id"
+fi
+
+if [ -f "$__object/parameter/base_dir" ]; then
+   base_dir=$(cat "$__object/parameter/base_dir")
+else
+  os="$(cat "$__global/explorer/os")"
+  case "$os" in
+    archlinux|netbsd|macosx|freebsd|openbsd) base_dir="/etc/rc.d" ;;
+    *) base_dir="/etc/init.d"
+  esac
+fi
+
+mode=$(cat "$__object/parameter/mode")
+
+echo "${base_dir}/${script} ${mode}"

other/types_pending_inclusion/__init_script/man.text

@@ -0,0 +1,51 @@
+cdist-type__init_script(7)
+==========================
+Daniel Roth <dani-cdist--@--d-roth.li>
+
+
+NAME
+----
+cdist-type__init_script - Use the init scripts
+
+
+DESCRIPTION
+-----------
+This type can be used to control your init scripts.
+
+
+REQUIRED PARAMETERS
+-------------------
+mode::
+   Specifies what shall be done with the init script (usually one of 'start'|'stop'|'restart'|'reload' or 'force-reload')
+
+
+OPTIONAL PARAMETERS
+-------------------
+script::
+   If supplied, use this as the init-script.
+   Otherwise the object_id is used.
+
+base_dir::
+   If supplied, this type uses this directory instead of '/etc/init.d'. The parameter will not need an ending slash.
+
+EXAMPLES
+--------
+
+--------------------------------------------------------------------------------
+# Reloads the configuration for lighttpd 
+__init_script lighttpd --mode force-reload
+
+# Reloads the configuration for lighttpd 
+__init_script lighty --script lighttpd --mode force-reload
+--------------------------------------------------------------------------------
+
+
+SEE ALSO
+--------
+- cdist-type(7)
+
+
+COPYING
+-------
+Copyright \(C) 2011 Daniel Roth. Free use of this software is
+granted under the terms of the GNU General Public License version 3 (GPLv3).

other/types_pending_inclusion/__init_script/parameter/optional

@@ -0,0 +1,2 @@
+script
+base_dir

other/types_pending_inclusion/__init_script/parameter/required

@@ -0,0 +1 @@
+mode