Merge branch 'weired_idea'

bin/cdist

@@ -0,0 +1,607 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import argparse
+import logging
+import os
+import subprocess
+import shutil
+import stat
+import sys
+import tempfile
+
+BANNER = """
+             ..          .       .x+=:.        s
+           dF           @88>    z`    ^%      :8
+          '88bu.        %8P        .   <k    .88
+      .   '*88888bu      .       .@8Ned8"   :888ooo
+ .udR88N    ^"*8888N   .@88u   .@^%8888"  -*8888888
+<888'888k  beWE "888L ''888E` x88:  `)8b.   8888
+9888 'Y"   888E  888E   888E  8888N=*8888   8888
+9888       888E  888E   888E   %8"    R88   8888
+9888       888E  888F   888E    @8Wou 9%   .8888Lu=
+?8888u../ .888N..888    888&  .888888P`    ^%888*
+ "8888P'   `"888*""     R888" `   ^"F        'Y"
+   "P'        ""         ""
+"""
+
+# Given paths from installation
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+CONF_DIR                   = os.path.join(BASE_DIR, "conf")
+GLOBAL_EXPLORER_DIR        = os.path.join(CONF_DIR, "explorer")
+LIB_DIR                    = os.path.join(BASE_DIR, "lib")
+MANIFEST_DIR               = os.path.join(CONF_DIR, "manifest")
+TYPE_DIR                   = os.path.join(CONF_DIR, "type")
+
+REMOTE_BASE_DIR            = "/var/lib/cdist"
+REMOTE_CONF_DIR            = os.path.join(REMOTE_BASE_DIR, "conf")
+REMOTE_OBJECT_DIR          = os.path.join(REMOTE_BASE_DIR, "object")
+REMOTE_TYPE_DIR            = os.path.join(REMOTE_CONF_DIR, "type")
+REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer")
+
+DOT_CDIST                  = ".cdist"
+VERSION                    = "2.0.0"
+
+
+#class Context(object):
+#
+#   def __init__(self, target_host):
+#      self.target_host = target_host
+#
+#   # class variable
+#   user_selber_shuld_wenn_aendert = 'bla'
+#
+#   # read only, aber statisch
+#   @property
+#   def remote_base_directory(self):
+#      return "/var/lib/cdist"
+#   @property.setter
+#
+#   @property
+#   def special_foo(self):
+#      return 'foo/{0}'.format(self.target_host)
+#
+
+
+logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+log = logging.getLogger()
+
+# List types
+def list_types():
+   return os.listdir(TYPE_DIR)
+
+def banner():
+   """Guess what :-)"""
+   print(BANNER)
+
+
+class Cdist:
+   """Cdist main class to hold arbitrary data"""
+
+   def __init__(self, target_host, initial_manifest=False, remote_user="root"):
+      self.target_host = target_host
+      self.remote_prefix = ["ssh", "root@" + self.target_host]
+
+      # Setup directory paths
+      self.temp_dir = tempfile.mkdtemp()
+
+      self.out_dir = os.path.join(self.temp_dir, "out")
+      os.mkdir(self.out_dir)
+
+      self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer")
+      os.mkdir(self.global_explorer_out_dir)
+
+      self.object_base_dir = os.path.join(self.out_dir, "object")
+
+      # Setup binary directory + contents
+      self.bin_dir = os.path.join(self.out_dir, "bin")
+      os.mkdir(self.bin_dir)
+      self.link_type_to_emulator()
+
+      # List of type explorers transferred
+      self.type_explorers_transferred = {}
+
+      self.remote_user = remote_user
+
+      # Mostly static, but can be overwritten on user demand
+      if initial_manifest:
+         self.initial_manifest = initial_manifest
+      else:
+         self.initial_manifest = os.path.join(MANIFEST_DIR, "init")
+
+   def cleanup(self):
+      # Do not use in __del__:
+      # http://docs.python.org/reference/datamodel.html#customization
+      # "other globals referenced by the __del__() method may already have been deleted 
+      # or in the process of being torn down (e.g. the import machinery shutting down)"
+      #
+      print("I should cleanup " + self.temp_dir)
+      # shutil.rmtree(self.temp_dir)
+
+   def exit_error(self, *args):
+      log.error(*args)
+      sys.exit(1)
+
+   def remote_mkdir(self, directory):
+      """Create directory on remote side"""
+      self.run_or_fail(["mkdir", "-p", directory], remote=True)
+
+   def remote_cat(filename):
+      """Use cat on the remote side for output"""
+      self.run_or_fail(["cat", filename], remote=True)
+
+   def shell_run_or_debug_fail(self, script, *args, **kargs):
+      # Manually execute /bin/sh, because sh -e does what we want
+      # and sh -c -e does not exit if /bin/false called
+      args[0][:0] = [ "/bin/sh", "-e" ]
+
+      if "remote" in kargs:
+         if kargs["remote"]:
+            args[0][:0] = self.remote_prefix
+            remote = true
+
+         del kargs["remote"]
+
+      log.debug("Shell exec cmd: %s", args)
+      log.debug("Shell exec env: %s", kargs['env'])
+      try:
+         subprocess.check_call(*args, **kargs)
+      except subprocess.CalledProcessError:
+         log.error("Code that raised the error:\n")
+         if remote:
+            remote_cat(script)
+         else:
+            script_fd = open(script)
+            print(script_fd.read())
+            script_fd.close()
+
+         self.exit_error("Non-Zero exit code exit of " + " ".join(*args))
+
+   def run_or_fail(self, *args, **kargs):
+      if "remote" in kargs:
+         if kargs["remote"]:
+            args[0][:0] = self.remote_prefix
+
+         del kargs["remote"]
+
+      log.debug("Exec: " + " ".join(*args))
+      try:
+         subprocess.check_call(*args, **kargs)
+      except subprocess.CalledProcessError:
+         self.exit_error("Command failed: " + " ".join(*args))
+
+   def remove_remote_dir(self, destination):
+      self.run_or_fail(["rm", "-rf",  destination], remote=True)
+
+   def transfer_dir(self, source, destination):
+      """Transfer directory and previously delete the remote destination"""
+      self.remove_remote_dir(destination)
+      self.run_or_fail(["scp", "-qr", source, 
+                        self.remote_user + "@" + 
+                        self.target_host + ":" + 
+                        destination])
+
+   def transfer_file(self, source, destination):
+      """Transfer file"""
+      self.run_or_fail(["scp", "-q", source, 
+                        self.remote_user + "@" +
+                        self.target_host + ":" +
+                        destination])
+
+   def global_explorer_output_path(self, explorer):
+      """Returns path of the output for a global explorer"""
+      return os.path.join(self.global_explorer_out_dir, explorer)
+
+   def type_explorer_output_dir(self, cdist_object):
+      """Returns and creates dir of the output for a type explorer"""
+      dir = os.path.join(self.object_dir(cdist_object), "explorer")
+      if not os.path.isdir(dir):
+         os.mkdir(dir)
+
+      return dir
+
+   def remote_global_explorer_path(self, explorer):
+      """Returns path to the remote explorer"""
+      return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer)
+
+   def list_global_explorers(self):
+      """Return list of available explorers"""
+      return os.listdir(GLOBAL_EXPLORER_DIR)
+
+   def list_type_explorers(self, type):
+      """Return list of available explorers for a specific type"""
+      dir = self.type_explorer_dir(type)
+      if os.path.isdir(dir):
+         list = os.listdir(dir)
+      else:
+         list = []
+
+      log.debug("Explorers for %s in %s: %s", type, dir, list)
+
+      return list
+
+   def list_object_paths(self, starting_point = False):
+      """Return list of paths of existing objects"""
+      object_paths = []
+
+      if not starting_point:
+         starting_point = self.object_base_dir
+
+      for content in os.listdir(starting_point):
+         full_path = os.path.join(starting_point, content)
+         if os.path.isdir(full_path):
+            object_paths.extend(self.list_object_paths(starting_point = full_path))
+
+         # Directory contains .cdist -> is an object
+         if content == DOT_CDIST:
+            object_paths.append(starting_point)
+
+      return object_paths
+
+   def get_type_from_object(self, cdist_object):
+      """Returns the first part (i.e. type) of an object"""
+      return cdist_object.split(os.sep)[0]
+
+   def get_object_id_from_object(self, cdist_object):
+      """Returns everything but the first part (i.e. object_id) of an object"""
+      return os.sep.join(cdist_object.split(os.sep)[1:])
+
+   def object_dir(self, cdist_object):
+      """Returns the full path to the object (including .cdist)"""
+      return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST)
+
+   def remote_object_dir(self, cdist_object):
+      """Returns the remote full path to the object (including .cdist)"""
+      return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST)
+
+   def object_parameter_dir(self, cdist_object):
+      """Returns the dir to the object parameter"""
+      return os.path.join(self.object_dir(cdist_object), "parameter")
+
+   def remote_object_parameter_dir(self, cdist_object):
+      """Returns the remote dir to the object parameter"""
+      return os.path.join(self.remote_object_dir(cdist_object), "parameter")
+
+   def object_code_paths(self, cdist_object):
+      """Return paths to code scripts of object"""
+      return [os.path.join(self.object_dir(cdist_object), "code-local"),
+              os.path.join(self.object_dir(cdist_object), "code-remote")]
+
+   def list_objects(self, starting_point = False):
+      """Return list of existing objects"""
+
+      if not starting_point:
+         starting_point = self.object_base_dir
+
+      object_paths = self.list_object_paths(starting_point)
+      objects = []
+
+      for path in object_paths:
+         objects.append(os.path.relpath(path, starting_point))
+
+      return objects
+
+   def type_explorer_dir(self, type):
+      """Return directory that holds the explorers of a type"""
+      return os.path.join(TYPE_DIR, type, "explorer")
+
+   def type_gencode_paths(self, type):
+      """Return paths to gencode scripts of type"""
+      return [os.path.join(TYPE_DIR, type, "gencode-local"),
+              os.path.join(TYPE_DIR, type, "gencode-remote")]
+
+   def type_manifest_path(self, type):
+      """Return path to manifest of type"""
+      return os.path.join(TYPE_DIR, type, "manifest")
+
+   def remote_type_explorer_dir(self, type):
+      """Return remote directory that holds the explorers of a type"""
+      return os.path.join(REMOTE_TYPE_DIR, type, "explorer")
+
+   def transfer_object_parameter(self, cdist_object):
+      """Transfer the object parameter to the remote destination"""
+      # Create base path before using mkdir -p
+      self.remote_mkdir(self.remote_object_parameter_dir(cdist_object))
+
+      # Synchronise parameter dir afterwards
+      self.transfer_dir(self.object_parameter_dir(cdist_object), 
+                        self.remote_object_parameter_dir(cdist_object))
+
+   def transfer_object_code(self, cdist_object):
+      FIXME
+      """Transfer the object code to the remote destination"""
+      # Create base path before using mkdir -p
+      self.remote_mkdir(self.remote_object_parameter_dir(cdist_object))
+
+      # Synchronise parameter dir afterwards
+      self.transfer_file(self.object_code_path(cdist_object), 
+                        self.remote_object_parameter_dir(cdist_object))
+
+   def transfer_global_explorers(self):
+      """Transfer the global explorers"""
+      self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR)
+
+   def transfer_type_explorers(self, type):
+      """Transfer explorers of a type, but only once"""
+      if type in self.type_explorers_transferred:
+         log.debug("Skipping retransfer for explorers of %s", type)
+         return
+      else:
+         # Do not retransfer
+         self.type_explorers_transferred[type] = 1
+
+      src = self.type_explorer_dir(type)
+      remote_base = os.path.join(REMOTE_TYPE_DIR, type)
+      dst = self.remote_type_explorer_dir(type)
+
+      # Only continue, if there is at least the directory
+      if os.path.isdir(src):
+         # Ensure that the path exists
+         self.remote_mkdir(remote_base)
+         self.transfer_dir(src, dst)
+
+
+   def link_type_to_emulator(self):
+      """Link type names to cdist-type-emulator"""
+      for type in list_types():
+         source = os.path.join(LIB_DIR, "cdist-type-emulator")
+         destination = os.path.join(self.bin_dir, type)
+         log.debug("Linking %s to %s", source, destination)
+         os.symlink(source, destination)
+
+   def run_global_explores(self):
+      """Run global explorers"""
+      explorers = self.list_global_explorers()
+      if(len(explorers) == 0):
+         self.exit_error("No explorers found in", GLOBAL_EXPLORER_DIR)
+
+      self.transfer_global_explorers()
+      for explorer in explorers:
+         output = self.global_explorer_output_path(explorer)
+         output_fd = open(output, mode='w')
+         cmd = []
+         cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR)
+         cmd.append(self.remote_global_explorer_path(explorer))
+
+         self.run_or_fail(cmd, stdout=output_fd, remote=True)
+         output_fd.close()
+
+   def run_type_explorer(self, cdist_object):
+      """Run type specific explorers for objects"""
+      # Based on bin/cdist-object-explorer-run
+
+      # Transfering explorers for this type
+      type = self.get_type_from_object(cdist_object)
+      self.transfer_type_explorers(type)
+
+      cmd = []
+      cmd.append("__explorer="      + REMOTE_GLOBAL_EXPLORER_DIR)
+      cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type))
+      cmd.append("__object="        + self.remote_object_dir(cdist_object))
+      cmd.append("__object_id="     + self.get_object_id_from_object(cdist_object))
+      cmd.append("__object_fq="     + cdist_object)
+
+      # Need to transfer at least the parameters for objects to be useful
+      self.transfer_object_parameter(cdist_object)
+
+      explorers = self.list_type_explorers(type)
+      for explorer in explorers:
+         remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)]
+         output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer)
+         output_fd = open(output, mode='w')
+         log.debug("%s exploring %s using %s storing to %s", 
+                  cdist_object, explorer, remote_cmd, output)
+                  
+         self.run_or_fail(remote_cmd, stdout=output_fd, remote=True)
+         output_fd.close()
+
+   def init_deploy(self):
+      log.debug("Creating clean directory structure")
+
+      # Ensure there is no old stuff, neither local nor remote
+   #   remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"])
+   #
+   #   # Create base directories
+   #   remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"])
+   #
+   #   # Link configuraion source directory - consistent with remote
+   #   run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"])
+
+   def run_initial_manifest(self):
+      """Run the initial manifest"""
+      env = {  "__manifest" : MANIFEST_DIR }
+      self.run_manifest(self.initial_manifest, extra_env=env)
+
+   def run_type_manifest(self, cdist_object):
+      """Run manifest for a specific object"""
+      type = self.get_type_from_object(cdist_object)
+      manifest = self.type_manifest_path(type)
+      
+      log.debug("%s: Running %s", cdist_object, manifest)
+      # FIXME: add more sensible checks for manifest
+      if os.path.exists(manifest):
+         env = {  "__object" :   self.object_dir(cdist_object), 
+                  "__object_id": self.get_object_id_from_object(cdist_object),
+                  "__object_fq": cdist_object,
+                  "__type":      type
+               }
+         self.run_manifest(manifest, extra_env=env)
+
+   def run_manifest(self, manifest, extra_env=None):
+      """Run a manifest"""
+      log.debug("Running manifest %s, env=%s", manifest, extra_env)
+      env = os.environ.copy()
+      env['PATH'] = self.bin_dir + ":" + env['PATH']
+
+      env['__target_host']    = self.target_host
+      env['__global']         = self.out_dir
+      
+      # Legacy stuff to make cdist-type-emulator work
+      env['__cdist_conf_dir']       = CONF_DIR
+      env['__cdist_core_dir']       = os.path.join(BASE_DIR, "core")
+      env['__cdist_local_base_dir'] = self.temp_dir
+      env['__cdist_manifest']       = self.initial_manifest
+
+      # Other environment stuff
+      if extra_env:
+         env.update(extra_env)
+
+      self.shell_run_or_debug_fail(manifest, [manifest], env=env)
+
+   def list_object_requirements(self, cdist_object):
+      """Return list of requirements for specific object"""
+      file=os.path.join(self.object_dir(cdist_object), "require")
+
+      if os.path.isfile(file):
+         file_fd = open(file, "r")
+         requirements = file_fd.readlines()
+         file_fd.close()
+
+         # Remove \n from all lines
+         requirements = map(lambda s: s.strip(), requirements)
+
+         log.debug("Requirements for %s: %s", cdist_object, requirements)
+      else:
+         requirements = []
+
+      return requirements
+
+   def object_run(self, cdist_object, mode):
+      """Run gencode or code for an object"""
+      log.debug("Running %s from %s", mode, cdist_object)
+      requirements = self.list_object_requirements(cdist_object)
+         
+      for requirement in requirements:
+         log.debug("Object %s requires %s", cdist_object, requirement)
+         self.object_run(requirement, mode=mode)
+
+      #
+      # Setup env Variable:
+      # 
+      env = os.environ.copy()
+      env['__target_host'] = self.target_host
+      env['__global']      = self.out_dir
+      env["__object"]      = self.object_dir(cdist_object)
+      env["__object_id"]   = self.get_object_id_from_object(cdist_object)
+      env["__object_fq"]   = cdist_object
+
+      if mode == "gencode":
+         paths = self.type_gencode_paths(self.get_type_from_object(cdist_object))
+         for bin in paths:
+            if os.path.isfile(bin):
+               # omit "gen" from gencode and 
+               outfile=os.path.join(self.object_dir(cdist_object), 
+                  os.path.basename(bin)[3:])
+
+               outfile_fd = open(outfile, "w")
+               self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd)
+               outfile_fd.close()
+
+               status = os.stat(outfile)
+
+               # Remove output if empty, else make it executable
+               if status.st_size == 0:
+                  os.unlink(outfile)
+               else:
+                  os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
+
+      if mode == "code":
+#         paths = self.object_code_paths(cdist_object)
+         local_dir   = self.object_dir(cdist_object)
+         remote_dir  = self.remote_object_dir(cdist_object)
+
+         bin = os.path.join(local_dir, "code-local")
+         if os.path.isfile(bin):
+            self.run_or_fail([bin], remote=False)
+            
+
+         local_remote_code = os.path.join(local_dir, "code-remote")
+         remote_remote_code = os.path.join(remote_dir, "code-remote")
+         if os.path.isfile(local_remote_code):
+            self.transfer_file(local_remote_code, remote_remote_code)
+            self.run_or_fail([remote_remote_code], remote=True)
+            
+   def deploy_to(self):
+      """Mimic the old deploy to: Deploy to one host"""
+      log.info("Deploying to " + self.target_host)
+      self.init_deploy()
+      self.run_global_explores()
+      self.run_initial_manifest()
+      
+      old_objects = []
+      objects = self.list_objects()
+
+      # Continue process until no new objects are created anymore
+      while old_objects != objects:
+         log.debug("Prepare stage")
+         old_objects = list(objects)
+         # FIXME: do not rerun existing objects!
+         for cdist_object in objects:
+            self.run_type_explorer(cdist_object)
+            self.run_type_manifest(cdist_object)
+
+         objects = self.list_objects()
+      
+      log.debug("Actual run objects")
+      # Now do the final steps over the existing objects
+      for cdist_object in objects:
+         log.debug("Run object: %s", cdist_object)
+         self.object_run(cdist_object, mode="gencode")
+         self.object_run(cdist_object, mode="code")
+
+      log.info("Finished run of %s", self.target_host)
+
+if __name__ == "__main__":
+   parser = argparse.ArgumentParser(description='cdist ' + VERSION)
+   parser.add_argument('host', nargs='*', help='one or more hosts to operate on')
+   parser.add_argument('-b', '--banner',
+       help='Show cdist banner',
+       action='store_true', dest='banner')
+   parser.add_argument('-d', '--debug', help='Set log level to debug',
+       action='store_true')
+   parser.add_argument('-i', '--initial-manifest', 
+       help='Path to a cdist manifest or - to read from stdin',
+       dest='manifest', required=False)
+   parser.add_argument('-p', '--parallel',
+       help='Operate on multiple hosts in parallel',
+       action='store_true', dest='parallel')
+   parser.add_argument('-s', '--sequential',
+       help='Operate on multiple hosts sequentially',
+       action='store_false', dest='parallel')
+   
+   args = parser.parse_args(sys.argv[1:])
+   if args.debug:
+       logging.root.setLevel(logging.DEBUG)
+
+   if args.banner:
+      banner()
+      sys.exit(0)
+   
+   try:
+      log.debug(args)
+
+      for host in args.host:
+         c = Cdist(host, initial_manifest=args.manifest)
+         c.deploy_to()
+         c.cleanup()
+   except KeyboardInterrupt:
+       sys.exit(0)

bin/cdist-deploy-to

@@ -43,16 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host "
 # See cdist-stages(7)
 #
 
-# Prepare local and remote directories
-__cdist_init_deploy           "$__cdist_target_host"
-
-# Transfer cdist executables
-__cdist_echo info "Transferring cdist binaries to the target host "
-cdist-dir push                "$__cdist_target_host" \
-   "${__cdist_abs_mydir}" "${__cdist_remote_bin_dir}"
-cdist-explorer-run-global     "$__cdist_target_host"
-cdist-manifest-run-init       "$__cdist_target_host"
-cdist-object-all              "$__cdist_target_host" cdist-object-prepare
 cdist-object-all              "$__cdist_target_host" cdist-object-run
 cdist-cache                   "$__cdist_target_host"
 

core/__cdist_cache

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2010 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# Save the configuration tree into the local cache
+#
+
+__cdist_cache()
+{
+   # Create base to move into
+   mkdir -p "${__cdist_local_base_cache_dir}"
+
+   __cdist_echo info \
+      "Caching to $(__cdist_host_cache_dir "$__cdist_target_host")"
+   rm -rf "$(__cdist_host_cache_dir "$__cdist_target_host")"
+   mv "$__cdist_local_base_dir" \
+      "$(__cdist_host_cache_dir "$__cdist_target_host")"
+}

core/__cdist_dir

@@ -23,29 +23,24 @@
 # Pull a directory from a target, both sides have the same name (i.e. explorers)
 #
 
+__cdist_dir()
+{
+   [ $# -eq 3 ] || __cdist_usage "<push|pull> <src dir> <dst dir>"
 
-. cdist-config
+   # ${3%/*} will be the destination directory, so no subdirectories
-[ $# -eq 4 ] || __cdist_usage "<push|pull> <target host> <src dir> <dst dir>"
+   # of the same name are created, if the directory is already existing
-set -ue
 
-__cdist_action="$1"; shift
+   if [ "$1" = "push" ]; then
-__cdist_target_host="$1"; shift
+      # FIXME: add error handling with __cdist_run_remote_... or so
-__cdist_src_dir="$1"; shift
-__cdist_dst_dir="$1"; shift
-
-# This will be the destination directory, so no subdirectories
-# of the same name are created, if the directory is already existing
-__cdist_top_dir="${__cdist_dst_dir%/*}"
-
-if [ "$__cdist_action" = "push" ]; then
       ssh "${__cdist_remote_user}@${__cdist_target_host}" \
-      "mkdir -p \"${__cdist_dst_dir}\""
+         "mkdir -p \"$3\""
-   scp -qr "$__cdist_src_dir" \
+      scp -qr "$2" \
-      "${__cdist_remote_user}@${__cdist_target_host}:${__cdist_top_dir}"
+         "${__cdist_remote_user}@${__cdist_target_host}:${3%/*}"
-elif [ "$__cdist_action" = "pull" ]; then
+   elif [ "$1" = "pull" ]; then
-   mkdir -p "${__cdist_dst_dir}"
+      mkdir -p "$3"
-   scp -qr "${__cdist_remote_user}@${__cdist_target_host}:${__cdist_src_dir}" \
+      scp -qr "${__cdist_remote_user}@${__cdist_target_host}:$2" \
-      "${__cdist_top_dir}"
+         "${3%/*}"
-else
+   else
-   __cdist_exit_err "Unknown action $__cdist_action"
+      __cdist_exit_err "Unknown action $1"
-fi
+   fi
+}

core/__cdist_dir_listing

@@ -0,0 +1,30 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# List files in a directory, if it exists
+#
+# We only create output, if there's at least one entry
+# and can thus be used as a boolean ;-)
+#
+
+__cdist_dir_listing()
+{
+   [ -d "$1" ] && ls -1 "$1"
+}

core/__cdist_echo

@@ -0,0 +1,56 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# echo / syslog alike function
+#
+
+__cdist_echo()
+{
+   __cdist_echo_type="$1"; shift
+
+   set +u
+   if [ "$__cdist_object_self" ]; then
+      __cdist_echo_prefix="${__cdist_object_self}:"
+   else
+      __cdist_echo_prefix="core: "
+   fi
+   set -u
+
+   case "$__cdist_echo_type" in
+      debug)
+         if [ "$__cdist_debug" = 1 ]; then
+            echo $__cdist_echo_prefix "DEBUG: $@"
+         fi
+      ;;
+      info)
+         echo $__cdist_echo_prefix "$@"
+      ;;
+      warn)
+         echo $__cdist_echo_prefix "Warning: $@"
+      ;;
+      error)
+         echo $__cdist_echo_prefix "Error: $@" >&2
+      ;;
+      *)
+         echo "CORE BUG, who created the broken commit in $0?" >&2
+         exit 23
+      ;;
+   esac
+}

core/__cdist_exit_err

@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Print error and exit (perror() alike)
+#
+
+__cdist_exit_err()
+{
+   __cdist_echo error "$@"
+   exit 1
+}

core/__cdist_explorer_run

@@ -0,0 +1,79 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Run explorers - FIXME: this function is ugly
+#
+
+__cdist_explorer_run()
+{
+   [ $# -eq 5 ] || __cdist_usage "<global|type> <local explorer dir> <remote explorer dir> <remote out dir> <local out dir>"
+
+   # Ensure there is at least one explorer
+   num="$(ls -1 "$__cdist_explorer_dir" | wc -l)"
+   if [ "$num" -lt 1 ]; then
+      __cdist_exit_err "${__cdist_explorer_dir}: Contains no explorers"
+   fi
+
+   # Check whether to setup variable for type explorer
+   case "$1" in
+      global)
+      ;;
+      type)
+         # FIXME: think about how and where this gets setup!
+         "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\""
+      ;;
+   esac
+
+   # Transfer explorers
+   __cdist_dir push "$2" "$3"
+
+   # Create output directory
+   __cdist_run_remote mkdir -p "$4"
+
+   # Execute all explorers - FIXME: isolate cd call?
+   cd "$2";
+   # FIXME: cleanup double variable, no need when in directory
+   for __cdist_explorer_run_explorer in *; do
+      __cdist_explorer_explorer_name="${__cdist_explorer_run_explorer##*/}"
+
+      if [ -f "$__cdist_explorer_run_explorer" ]; then
+         if [ ! -x "$__cdist_explorer_run_explorer" ]; then
+            __cdist_exit_err "Explorer \"$__cdist_explorer_run_explorer\" exists, but is not executable."
+         fi
+
+      else
+         if [ -e "$__cdist_explorer_run_explorer" ]; then
+            __cdist_exit_err "Explorer \"$__cdist_explorer_run_explorer\" exists, but is not a file."
+         fi
+      fi
+
+      # FIXME: no need for remote out dir probably?
+      # or should we leave it and continue using __cdist_dir pull?
+      __cdist_run_remote \
+         "export $__cdist_name_var_explorer=\"$__cdist_remote_explorer_dir\";" \
+         "export $__cdist_name_var_global=\"$__cdist_remote_out_dir\";" \
+         "$3/$__cdist_explorer_run_explorer" ">" \
+            "$4/$__cdist_explorer_run_explorer" || \
+               __cdist_exit_err "Explorer $__cdist_explorer_run_explorer failed."
+   done
+
+   # Transfer results back
+   __cdist_dir pull "$4" "$5"
+}

core/__cdist_explorer_run_global

@@ -17,27 +17,16 @@
 # You should have received a copy of the GNU General Public License
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
-# Build pseudo binaries for type emulation
+#
+# Copy & run the global explorers, i.e. not bound to types
 #
 
-. cdist-config
+__cdist_explorer_run_global()
-[ $# -eq 1 ] || __cdist_usage "<out dir>"
+{
-set -eu
+   __cdist_echo info "Running global explorers "
 
-__cdist_output_dir="$1"; shift
+   # run the global explorers remotely
-
+   __cdist_explorer_run global \
-__cdist_type_emulator="$__cdist_abs_mydir/cdist-type-emulator"
+      "$__cdist_explorer_dir" "$__cdist_remote_explorer_dir" \
-
+      "$__cdist_remote_out_explorer_dir" "$__cdist_out_explorer_dir"
-if [ ! -d "${__cdist_type_dir}" ]; then
+}
-   __cdist_exit_err "$__cdist_type_dir must exist and contain available types"
-fi
-
-# Get Types
-cd "${__cdist_type_dir}"
-ls -1 > "${__cdist_tmp_file}"
-
-# Create binaries
-mkdir -p "${__cdist_output_dir}"
-while read type; do
-   ln -sf "${__cdist_type_emulator}" "${__cdist_output_dir}/${type}"
-done < "${__cdist_tmp_file}"

core/__cdist_is_executable

@@ -0,0 +1,44 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Test whether something is executable (that should be executable) or
+# is missing
+#
+
+__cdist_is_executable()
+{
+   [ $# -eq 1 ] || __cdist_exit_err "<executable>"
+
+   if [ -e "$1" ]; then
+      if [ -f "$1" ]; then
+         if [ -x "$1" ]; then
+            # Exists and is a correct executable
+            true
+         else
+            __cdist_exit_err "$1 exists, but is not executable."
+         fi
+      else
+         __cdist_exit_err "$1 exists, but is not a file."
+      fi
+   else
+      # Does not exist
+      false
+   fi
+}

core/__cdist_kill_on_interrupt

@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# Run the given command for each created object.
+#
+
+# Does not work in children, will be called again in every script!
+# Use only in interactive "front end" scripts
+__cdist_kill_on_interrupt()
+{
+   __cdist_tmp_removal
+   kill 0
+   exit 1
+}

core/__cdist_manifest_run

@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# Let's build a cconfig tree from a configuration
+# And save it into the cache tree
+#
+
+__cdist_manifest_run()
+{
+   [ $# -eq 1 ] || __cdist_usage "<manifest>"
+
+   __cdist_manifest="$1"; shift
+
+   ################################################################################
+   # Export information for cdist-type-emulator or manifest
+   #
+
+   # Config dir should not get reset - FIXME: why did I do this?
+   export __cdist_conf_dir
+
+   # Used to record the source in the object
+   export __cdist_manifest
+
+   # Export information for manifests - __cdist_out_dir comes from cdist-config
+   export $__cdist_name_var_global="$__cdist_out_dir"
+
+   ################################################################################
+   # The actual run
+   #
+
+   # Ensure binaries are existing - FIXME: move error handling into __cdist_type_build_emulation
+   __cdist_type_build_emulation \
+      || __cdist_exit_err "Failed to build type emulation binaries"
+
+   __cdist_run_shell "${__cdist_manifest}"
+}

core/__cdist_manifest_run_init

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2010 Nico Schottelius (nico-cdist at schottelius.org)
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -22,18 +22,11 @@
 # And save it into the cache tree
 #
 
-. cdist-config
+__cdist_manifest_run_init()
-[ $# -eq 1 ] || __cdist_usage "<target host>"
+{
-set -u
+   # FIXME: probably do not export but always set explicitly?
+   export $__cdist_name_var_manifest="$__cdist_manifest_dir"
 
-__cdist_target_host="$1"; shift
+   __cdist_echo info "Running initial manifest for $__cdist_target_host "
-
+   __cdist_manifest_run "$__cdist_manifest_init"
-# Create base to move into
+}
-mkdir -p "${__cdist_local_base_cache_dir}"
-
-# Now determine absolute path
-__cdist_ddir="$(__cdist_host_cache_dir "$__cdist_target_host")"
-
-__cdist_echo info "Saving cache to $__cdist_ddir "
-rm -rf "$__cdist_ddir"
-mv "$__cdist_local_base_dir" "$__cdist_ddir"

core/__cdist_object_all

@@ -0,0 +1,60 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# Run the given command for each created object.
+#
+
+__cdist_object_all()
+{
+   [ $# -eq 1 ] || __cdist_usage "<command>"
+
+   __cdist_object_all_object_all_command="$1"; shift
+
+   __cdist_object_all_object_all_objects="$__cdist_tmp_dir/objects"
+
+   # Ensure object dir exists, so marker can be created
+   mkdir -p "${__cdist_out_object_dir}"
+
+   # FIXME: : - why do we use a file?
+   #    core/__cdist_object_manifest_run:         touch "$__cdist_objects_created"
+
+   # Loop until we do not create new objects anymore
+   # which is equal to all objects have been run
+   touch "$__cdist_objects_created"
+   while [ -f "$__cdist_objects_created" ]; do
+      # Assume we're done after this run
+      rm "$__cdist_objects_created"
+
+      # Get listing of objects
+      __cdist_object_list "$__cdist_out_object_dir" > \
+         "$__cdist_object_all_object_all_objects"
+
+      # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP
+      while read __cdist_object_all_object; do
+         set -- "$@" "$__cdist_object_all_object"
+      done < "$__cdist_object_all_object_all_objects"
+
+      while [ $# -gt 0 ]; do
+         __cdist_object_all_object="$1"; shift
+         $__cdist_object_all_object_all_command "$__cdist_object_all_object"
+      done
+   done
+}

core/__cdist_object_code_run

@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Exec the code for the given object locally and remote
+#
+
+__cdist_object_code_run()
+{
+   [ $# -eq 1 ] || __cdist_exit_err "<object>"
+
+
+   if [ ! -d "$(__cdist_object_dir "$1")" ]; then
+      __cdist_exit_err "Object undefined"
+   fi  
+
+   # Code local
+   export __cdist_out_object_dir="$__cdist_out_object_dir"
+   __cdist_echo debug "Trying to run local code"
+   if __cdist_is_executable \
+      "$(__cdist_object_code "$1" "${__cdist_name_gencode_local}")"; then
+      __cdist_run_shell \
+         "$(__cdist_object_code "$1" "${__cdist_name_gencode_local}")"
+   else
+      __cdist_echo debug "Local code: none"
+   fi
+
+   # Code remote
+   __cdist_echo debug "Trying to run remote code"
+   if __cdist_is_executable \
+      "$(__cdist_object_code "$1" "${__cdist_name_gencode_remote}")"; then
+
+      __cdist_run_remote $(__cdist_remote_object_code "$1")
+   else
+      __cdist_echo debug "Remote code: none"
+   fi
+}

core/__cdist_object_explorer_run

@@ -0,0 +1,89 @@
+#!/bin/sh
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# Run the explorers for the given object on the target host.
+#
+
+# FIXME: many cleanups needed before going production!
+
+__cdist_object_explorer_run()
+{
+   __cdist_object_self="$1"; shift
+
+   __cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")"
+   __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")"
+
+   # Check if type of object has >= 1 explorer
+   __cdist_has_explorer="$(__cdist_dir_listing "$(__cdist_type_explorer_dir "$__cdist_type")" | wc -l)"
+   # Run the type explorers for the current object if any
+   if [ "$__cdist_has_explorer" -ge 1 ]; then
+      if ! __cdist_type_explorer_pushed "$__cdist_type"; then
+         # FIXME: variables!
+         src_dir="$(__cdist_type_explorer_dir "$__cdist_type")"
+         dst_dir="$(__cdist_remote_type_explorer_dir "$__cdist_type")"
+         __cdist_echo info "Transfering explorers for $__cdist_type "
+         __cdist_dir push "$src_dir" "$dst_dir"
+         __cdist_type_explorer_pushed_add "$__cdist_type"
+      fi
+
+      __cdist_echo info "Running explorers"
+      # Copy object parameters
+      __cdist_dir push \
+         "$(__cdist_object_parameter_dir "$__cdist_object_self")"         \
+         "$(__cdist_remote_object_parameter_dir "$__cdist_object_self")"
+
+      # Execute explorers
+      # FIXME: STOPPED:
+      #  - remove cdist-remote-explorer-run
+      #  - problem: new variables / need to run explorer directly?
+      #     -> or put logic into __cdist_explorer_run
+      #        -> think about having _one_ wrapper script for remote to execute
+      #           shell functions
+
+      # Create remote output directory
+      __cdist_run_remote mkdir -p "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")"
+
+      cd "$(__cdist_type_explorer_dir "$__cdist_type")"
+
+
+      for __cdist_object_explorer_run_explorer in *; do
+         __cdist_run_remote                                          \
+            "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \
+            "$__cdist_name_var_object_id=\"$__cdist_object_id\""     \
+            "$__cdist_name_var_self=\"$__cdist_object_self\""        \
+            "$(__cdist_remote_type_explorer_dir "$__cdist_type")/$__cdist_object_explorer_run_explorer"    \
+            ">" "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")/$__cdist_object_explorer_run_explorer"
+      done
+
+#       __cdist_run_remote                                          \
+#          "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \
+#          "$__cdist_name_var_object_id=\"$__cdist_object_id\""     \
+#          "$__cdist_name_var_self=\"$__cdist_object_self\""        \
+#          cdist-remote-explorer-run                                \
+#          "$__cdist_name_var_type_explorer"                        \
+#          "$(__cdist_remote_type_explorer_dir "$__cdist_type")"    \
+#          "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")"
+
+      # Copy back results
+      __cdist_dir pull "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" \
+         "$(__cdist_object_type_explorer_dir "$__cdist_object_self")"
+   fi
+}

core/__cdist_object_gencode

@@ -0,0 +1,66 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Generate code from one object (object must be relative path!)
+# WARNING: OUTPUT ON STDOUT, ERRORS NEED TO BE ON STDERR!
+#
+
+# FIXME: check variable names:
+# either prefix or use global or use functions directly
+# functions looks good, they are cheap anyway!
+
+__cdist_object_gencode()
+{
+   [ $# -eq 2 ] || __cdist_usage "<object>" "<type>"
+
+   __cdist_object_self="$1"; shift
+   __cdist_gencode_type="$1"; shift
+
+   __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")"
+   __cdist_type_gencode="$(__cdist_type_gencode "$__cdist_type" "$__cdist_gencode_type")"
+   __cdist_code_output="$(__cdist_object_code "$__cdist_object_self" "$__cdist_gencode_type")"
+
+   # export variables for the gencode script
+   export __object_id="$(__cdist_object_id_from_object "$__cdist_object_self")"
+   export __object="$(__cdist_object_dir "$__cdist_object_self")"
+   export __global="$__cdist_out_dir"
+
+   if [ -x "$__cdist_type_gencode" ]; then
+      __cdist_run_shell "$__cdist_type_gencode" > "$__cdist_tmp_file"
+   else
+      if [ -e "$__cdist_type_gencode" ]; then
+         __cdist_exit_err "$__cdist_type_gencode exists, but is not executable"
+      fi
+
+      # Ensure it's empty, if there is no gencode
+      : > "$__cdist_tmp_file"
+   fi
+
+   # Only create code, if gencode created output
+   if [ "$(wc -l < "$__cdist_tmp_file")" -gt 0 ]; then
+      cat - "$__cdist_tmp_file" << eof > "$__cdist_code_output"
+#
+# The following code was generated by $__cdist_type_gencode
+#
+
+eof
+      chmod u+x "${__cdist_code_output}"
+   fi
+}

core/__cdist_object_gencode_run

@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# 2010 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# For the given object create the code to be executed on the target.
+#
+
+__cdist_object_gencode_run()
+{
+   __cdist_object_gencode_run_object="$1"; shift
+
+   __cdist_echo info "Generating local code "
+   __cdist_object_gencode "$__cdist_object_gencode_run_object" \
+      "${__cdist_name_gencode_local}"
+
+   __cdist_echo info "Generating remote code "
+   __cdist_object_gencode "$__cdist_object_gencode_run_object" \
+   "${__cdist_name_gencode_remote}"
+}

core/__cdist_object_list

@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Print error and exit (perror() alike)
+#
+
+__cdist_object_list()
+{
+   # FIXME: no local in posix
+   local basedir="$1"; shift
+
+   # Use subshell to prevent changing cwd in program
+   (
+      cd "${basedir}"
+
+      find . -name "$__cdist_name_dot_cdist" |    \
+         sed -e 's;^./;;' -e "s;/${__cdist_name_dot_cdist}\$;;"
+   )
+}

core/__cdist_object_manifest_run

@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# 2010 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# Run the manifest for the given object.
+#
+
+
+__cdist_object_manifest_run()
+{
+   [ $# -eq 1 ] || __cdist_usage "<object>"
+
+   __cdist_object_self="$1"; shift
+
+   # FIXME: rename to __cdist_object_dir (everywhere!)
+   __cdist_cur_object_dir="$(__cdist_object_dir "$__cdist_object_self")"
+   __cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")"
+
+   __cdist_echo info "Checking manifest "
+
+   __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")"
+   __cdist_manifest="$(__cdist_type_manifest "$__cdist_type")"
+
+   if [ -f "$__cdist_manifest" ]; then
+      if [ -x "$__cdist_manifest" ]; then
+         # Make __cdist_manifest available for cdist-type-emulator
+         export __cdist_manifest
+
+         __cdist_echo info "Executing manifest "
+         export $__cdist_name_var_object="$__cdist_cur_object_dir"
+         export $__cdist_name_var_object_id="$__cdist_object_id"
+         export $__cdist_name_var_type="$(__cdist_type_dir "$__cdist_type")"
+
+         __cdist_manifest_run "$__cdist_manifest"
+
+         # Tell cdist-object-run-all that there may be new objects
+         touch "$__cdist_objects_created"
+      else
+         __cdist_exit_err "${__cdist_manifest} needs to be executable."
+      fi
+   fi
+}

core/__cdist_object_prepare

@@ -0,0 +1,47 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# For the given object:
+#  - run type explorers
+#  - run type manifest
+#
+
+__cdist_object_prepare()
+{
+   [ $# -eq 1 ] || __cdist_usage "<object>"
+
+   __cdist_object_self="$1"; shift
+   __cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")"
+   [ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined"
+
+   # Export to non-core for use in manifest and gencode scripts
+   export $__cdist_name_var_self=$__cdist_object_self
+
+   __cdist_object_prepared="$(__cdist_object_prepared "$__cdist_object_self")"
+   if [ ! -f "$__cdist_object_prepared" ]; then
+      __cdist_echo info "Preparing object"
+      __cdist_object_explorer_run "$__cdist_object_self"
+      __cdist_object_manifest_run "$__cdist_object_self"
+
+      # Mark this object as prepared
+      touch "$__cdist_object_prepared"
+   fi
+}

core/__cdist_object_run

@@ -0,0 +1,74 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# For the given object:
+#  - run type explorers
+#  - run type manifest
+#  - generate code
+#  - copy object to target
+#  - execute code on target
+#
+
+__cdist_object_run()
+{
+   [ $# -eq 1 ] || __cdist_usage "<object>"
+
+   __cdist_object_self="$1"; shift
+   __cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")"
+   [ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined"
+
+   # Export to non-core for use in manifest and gencode scripts
+   export $__cdist_name_var_self=$__cdist_object_self
+
+   # FIXME: BUG: should be named differently!
+   # FIXME: BUG: I can be called recursively! -> variables are probably already set / overwritten!
+   __cdist_object_finished="$(__cdist_object_finished "$__cdist_object_self")"
+   if [ ! -f "$__cdist_object_finished" ]; then
+      # Resolve dependencies, if any
+      __cdist_object_require="$(__cdist_object_require "$__cdist_object_self")"
+      if [ -f "$__cdist_object_require" ]; then
+         # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP
+         while read __cdist_requirement; do
+            set -- "$@" "$__cdist_requirement"
+         done < "$__cdist_object_require"
+
+         while [ $# -gt 0 ]; do
+            __cdist_requirement="$1"; shift
+            __cdist_echo info "Resolving requirement $__cdist_requirement"
+            # FIXME: BUG: at this point, the other __cdist_object_run may have
+            # overwritten all our variables!
+            __cdist_object_run "$__cdist_requirement"
+         done
+      fi
+
+      __cdist_echo debug "Before gencode"
+      __cdist_object_gencode_run "$__cdist_object_self"
+      __cdist_echo debug "Before push"
+      __cdist_dir push "$(__cdist_object_dir "$__cdist_object_self")" \
+                "$(__cdist_remote_object_dir "$__cdist_object_self")"
+      __cdist_echo debug "Before run"
+      __cdist_object_code_run "$__cdist_object_self"
+      __cdist_echo debug "Object run done"
+
+      # Mark this object as done
+      touch "$__cdist_object_finished"
+   fi
+}

core/__cdist_run

@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Exit if an error occurs running something
+#
+
+__cdist_run()
+{
+   "$@" || __cdist_echo error  "$1 exited non-zero, aborting."
+}

core/__cdist_run_remote

@@ -21,13 +21,12 @@
 # Run a cdist binary on the remote side
 #
 
-. cdist-config
+__cdist_run_remote()
-[ $# -ge 2 ] || __cdist_usage "<target_host> <binary> [opts]"
+{
-set -ue
+   [ $# -ge 1 ] || __cdist_usage "<binary> [opts]"
 
-__cdist_target_host="$1"; shift
+   ssh "${__cdist_remote_user}@${__cdist_target_host}"   \
-
-ssh "${__cdist_remote_user}@${__cdist_target_host}"   \
       "export PATH=\"${__cdist_remote_bin_dir}:\$PATH\";" \
       "export __cdist_out_object_dir=\"$__cdist_remote_out_object_dir\";" \
       "$@"
+}

core/__cdist_run_shell

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Exit if an error occurs when running a shell script
+#
+
+__cdist_run_shell()
+{
+   # Prepend our path, so all cdist tools come before other tools
+   PATH="${__cdist_out_type_bin_dir}:$PATH" sh -e "$@"
+   if [ "$?" -ne 0 ]; then
+      __cdist_echo error  "$1 exited non-zero"
+      __cdist_echo warn "Faulty code:"
+      cat "$1"
+      __cdist_exit_err "Aborting due to non-zero exit code."
+   fi  
+}

core/__cdist_tmp_removal

@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Remove tmp dir
+#
+
+__cdist_tmp_removal()
+{
+   rm -rf "${__cdist_tmp_dir}"
+}

core/__cdist_type_build_emulation

@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# Build pseudo binaries for type emulation
+#
+
+__cdist_type_build_emulation()
+{
+   [ $# -eq 0 ] || __cdist_usage "No arguments"
+
+   [ -f "${__cdist_out_type_bin_dir}/.marker" ] && return 0
+
+   __cdist_type_emulator="$__cdist_abs_mydir/cdist-type-emulator"
+
+   if [ ! -d "${__cdist_type_dir}" ]; then
+      __cdist_exit_err "$__cdist_type_dir must exist and contain available types"
+   fi
+
+   # Get Types
+   (
+      cd "${__cdist_type_dir}"
+      ls -1 > "${__cdist_tmp_file}"
+   )
+
+   # Create binaries
+   mkdir -p "${__cdist_out_type_bin_dir}"
+   while read __cdist_type_build_emulation_type; do
+      ln -sf "${__cdist_type_emulator}" \
+         "${__cdist_out_type_bin_dir}/${__cdist_type_build_emulation_type}"
+   done < "${__cdist_tmp_file}"
+
+   touch "${__cdist_out_type_bin_dir}/.marker"
+}

core/__cdist_usage

@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Print error and exit (perror() alike)
+#
+
+__cdist_usage()
+{
+   __cdist_exit_err "$__cdist_myname: $@"
+}  

doc/changelog

@@ -1,5 +1,6 @@
 next:
 	* New Type: __package_rubygem (Chase Allen James)
+	* __self replaced by __object_fq (or so)
 
 1.7.1: 2011-07-26
 	* Documentation: Add explorers to reference

doc/dev/todo/niconext

@@ -1,26 +1,5 @@
-Catch broken instances in cdist-mass-deploy -p and report broken deployements at the end!
+- rewrite in python?
-
+   - also do with cdist-type-emulator, which had quirks applied from outside to run
-
+- support non-ssh access?
---------------------------------------------------------------------------------
-Bug with requirements when indirect requires is scheduled late:
-
-__package/collectd: Generating local code 
-__package/collectd: Generating remote code 
-__package/collectd: Transferring object
-__package/collectd: Checking code-local
-__package/collectd: Checking code-remote
-__file/etc/collectd/collectd.conf: Generating local code 
-__file/etc/collectd/collectd.conf: Generating remote code 
-__file/etc/collectd/collectd.conf: Transferring object
-__file/etc/collectd/collectd.conf: Checking code-local
-__file/etc/collectd/collectd.conf: Executing code-local
-scp: /etc/collectd/collectd.conf: No such file or directory
-__file/etc/collectd/collectd.conf: Error: /tmp/cdist.PYKFWmj9QknE/out/object/__file/etc/collectd/collectd.conf/.cdist/code-local exited non-zero
-__file/etc/collectd/collectd.conf: Warning: Faulty code:
-#
-# The following code was generated by /home/users/nico/oeffentlich/rechner/projekte/cdist-nutzung/conf/type/__file/gencode-local
-#
-
-scp /home/users/nico/oeffentlich/rechner/projekte/cdist-nutzung/conf/type/__ethz_collectd/files/collectd.conf.client root@shrek08:/etc/collectd/collectd.conf
-__file/etc/collectd/collectd.conf: Error: Aborting due to non-zero exit code.
 
+- Bug: os.path.join() may be wrong for the remote side!

doc/man/cdist-reference.text.sh

@@ -164,19 +164,19 @@ __manifest::
    Available for: initial manifest
 __global::
    Directory that contains generic output like explorer.
-   Available for: initial manifest, type manifest, type explorer, type codegen
+   Available for: initial manifest, type manifest, type gencode
 __object::
    Directory that contains the current object.
-   Available for: type manifest, type explorer, type codegen
+   Available for: type manifest, type explorer, type gencode
 __object_id::
    The type unique object id.
-   Available for: type manifest, type explorer, type codegen
+   Available for: type manifest, type explorer, type gencode
 __self::
    The full qualified name of the current object.
-   Available for: type manifest, type explorer, type codegen
+   Available for: type manifest, type explorer, type gencode
 __target_host::
    The host we are deploying to.
-   Available for: initial manifest, type manifest, type codegen
+   Available for: initial manifest, type manifest, type gencode
 __target_user::
    User to use for authentication on remote host.
    Currently static in core.