Merge remote-tracking branch 'cdist/install' into install

bin/cdist

@@ -25,6 +25,7 @@ import datetime
 import logging
 import multiprocessing
 import os
+import re
 import subprocess
 import shutil
 import stat
@@ -55,12 +56,12 @@ REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer")
 
 CODE_HEADER                = "#!/bin/sh -e\n"
 DOT_CDIST                  = ".cdist"
+TYPE_PREFIX                = "__"
 VERSION                    = "2.0.0"
 
 logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
 log = logging.getLogger()
 
-
 class TypeEmulator:
    def __init__(self, name):
       self.name = name
@@ -252,7 +253,7 @@ class Cdist:
 
    def list_type_explorers(self, type):
       """Return list of available explorers for a specific type"""
-      dir = self.type_explorer_dir(type)
+      dir = self.type_dir(type, "explorer")
       if os.path.isdir(dir):
          list = os.listdir(dir)
       else:
@@ -321,22 +322,9 @@ class Cdist:
 
       return objects
 
-   def type_dir(self, type):
+   def type_dir(self, type, *args):
       """Return directory the type"""
-      return os.path.join(self.type_base_dir, type)
+      return os.path.join(self.type_base_dir, type, *args)
-
-   def type_explorer_dir(self, type):
-      """Return directory that holds the explorers of a type"""
-      return os.path.join(self.type_dir(type), "explorer")
-
-   def type_gencode_paths(self, type):
-      """Return paths to gencode scripts of type"""
-      return [os.path.join(self.type_base_dir, type, "gencode-local"),
-              os.path.join(self.type_base_dir, type, "gencode-remote")]
-
-   def type_manifest_path(self, type):
-      """Return path to manifest of type"""
-      return os.path.join(self.type_base_dir, type, "manifest")
 
    def remote_type_explorer_dir(self, type):
       """Return remote directory that holds the explorers of a type"""
@@ -365,7 +353,7 @@ class Cdist:
          # Do not retransfer
          self.type_explorers_transferred[type] = 1
 
-      src = self.type_explorer_dir(type)
+      src = self.type_dir(type, "explorer")
       remote_base = os.path.join(REMOTE_TYPE_DIR, type)
       dst = self.remote_type_explorer_dir(type)
 
@@ -445,7 +433,7 @@ class Cdist:
    def run_type_manifest(self, cdist_object):
       """Run manifest for a specific object"""
       type = self.get_type_from_object(cdist_object)
-      manifest = self.type_manifest_path(type)
+      manifest = self.type_dir(type, "manifest")
       
       log.debug("%s: Running %s", cdist_object, manifest)
       if os.path.exists(manifest):
@@ -499,6 +487,7 @@ class Cdist:
       """Run gencode or code for an object"""
       log.debug("Running %s from %s", mode, cdist_object)
       requirements = self.list_object_requirements(cdist_object)
+      type = self.get_type_from_object(cdist_object)
          
       for requirement in requirements:
          log.debug("Object %s requires %s", cdist_object, requirement)
@@ -513,9 +502,13 @@ class Cdist:
       env["__object"]      = self.object_dir(cdist_object)
       env["__object_id"]   = self.get_object_id_from_object(cdist_object)
       env["__object_fq"]   = cdist_object
+      env["__type"]        = self.type_dir(type)
 
       if mode == "gencode":
-         paths = self.type_gencode_paths(self.get_type_from_object(cdist_object))
+         paths = [
+            self.type_dir(type, "gencode-local"),
+            self.type_dir(type, "gencode-remote")
+         ]
          for bin in paths:
             if os.path.isfile(bin):
                # omit "gen" from gencode and 
@@ -555,10 +548,8 @@ class Cdist:
             self.transfer_file(local_remote_code, remote_remote_code)
             self.run_or_fail([remote_remote_code], remote=True)
             
-   def deploy_to(self):
+   def stage_prepare(self):
-      """Mimic the old deploy to: Deploy to one host"""
+      """Do everything for a deploy, minus the actual code stage"""
-      log.info("Deploying to " + self.target_host)
-      time_start = datetime.datetime.now()
       self.init_deploy()
       self.run_global_explores()
       self.run_initial_manifest()
@@ -580,14 +571,24 @@ class Cdist:
                self.objects_prepared.append(cdist_object)
 
          objects = self.list_objects()
-      
+
+   def stage_run(self):
+      """The final (and real) step of deployment"""
       log.debug("Actual run objects")
       # Now do the final steps over the existing objects
-      for cdist_object in objects:
+      for cdist_object in self.list_objects():
          log.debug("Run object: %s", cdist_object)
          self.object_run(cdist_object, mode="gencode")
          self.object_run(cdist_object, mode="code")
 
+   def deploy_to(self):
+      """Mimic the old deploy to: Deploy to one host"""
+      log.info("Deploying to " + self.target_host)
+      time_start = datetime.datetime.now()
+
+      self.stage_prepare()
+      self.stage_run()
+
       time_end = datetime.datetime.now()
       duration = time_end - time_start
       log.info("Finished run of %s in %s seconds", 
@@ -628,7 +629,16 @@ def config(args):
    log.info("Total processing time for %s host(s): %s", len(args.host),
             (time_end - time_start).total_seconds())
 
-if __name__ == "__main__":
+def install(args):
+   """Install remote system"""
+   process = {}
+
+def emulator():
+   """Emulate type commands (i.e. __file and co)"""
+   type = os.path.basename(sys.argv[0])
+
+def commandline():
+   """Parse command line"""
    # Construct parser others can reuse
    parser = {}
    # Options _all_ parsers have in common
@@ -648,25 +658,33 @@ if __name__ == "__main__":
       add_help=False)
    parser['banner'].set_defaults(func=banner)
 
-   # Config
+   # Config and install (common stuff)
-   parser['config'] = parser['sub'].add_parser('config',
+   parser['configinstall'] = argparse.ArgumentParser(add_help=False)
-      parents=[parser['most']])
+   parser['configinstall'].add_argument('host', nargs='+',
-   parser['config'].add_argument('host', nargs='+',
       help='one or more hosts to operate on')
-   parser['config'].add_argument('-c', '--cdist-home',
+   parser['configinstall'].add_argument('-c', '--cdist-home',
        help='Change cdist home (default: .. from bin directory)',
        action='store')
-   parser['config'].add_argument('-i', '--initial-manifest', 
+   parser['configinstall'].add_argument('-i', '--initial-manifest', 
        help='Path to a cdist manifest',
        dest='manifest', required=False)
-   parser['config'].add_argument('-p', '--parallel',
+   parser['configinstall'].add_argument('-p', '--parallel',
        help='Operate on multiple hosts in parallel',
        action='store_true', dest='parallel')
-   parser['config'].add_argument('-s', '--sequential',
+   parser['configinstall'].add_argument('-s', '--sequential',
        help='Operate on multiple hosts sequentially (default)',
        action='store_false', dest='parallel')
+
+   # Config
+   parser['config'] = parser['sub'].add_parser('config',
+      parents=[parser['most'], parser['configinstall']])
    parser['config'].set_defaults(func=config)
 
+   # Install
+   parser['install'] = parser['sub'].add_parser('install',
+      parents=[parser['most'], parser['configinstall']])
+   parser['install'].set_defaults(func=install)
+
    for p in parser:
       parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
 
@@ -682,3 +700,9 @@ if __name__ == "__main__":
       args.func(args)
    except KeyboardInterrupt:
        sys.exit(0)
+
+if __name__ == "__main__":
+   if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])):
+      emulator()
+   else:
+      commandline()

bin/cdist-type-template

@@ -1,83 +0,0 @@
-#!/bin/sh
-#
-# 2011 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-# Create a new type from scratch
-#
-
-. cdist-config
-[ $# -eq 1 ] || __cdist_usage "<type>"
-set -eu
-
-__cdist_type="$1"; shift
-__cdist_my_type_dir="$(__cdist_type_dir "$__cdist_type")"
-
-if [ -d "$__cdist_my_type_dir" ]; then
-   __cdist_usage "Type $__cdist_type already exists"
-fi
-
-echo "Creating type $__cdist_type in $__cdist_my_type_dir ..."
-# Base
-mkdir -p "$__cdist_my_type_dir"
-
-# Parameter
-mkdir -p "$(__cdist_type_parameter_dir "$__cdist_type")"
-touch "$(__cdist_type_parameter_dir "$__cdist_type")/${__cdist_name_parameter_required}"
-touch "$(__cdist_type_parameter_dir "$__cdist_type")/${__cdist_name_parameter_optional}"
-
-# Manifest
-cat "$__cdist_abs_mydir/../doc/dev/header" - << eof > "$__cdist_my_type_dir/${__cdist_name_manifest}"
-
-#
-# This is the manifest, which can be used to create other objects like this:
-# __file /path/to/destination --source /from/where/
-#
-# To tell cdist to make use of it, you need to make it executable (chmod +x)
-#
-#
-
-eof
-
-# Gencode remote
-cat "$__cdist_abs_mydir/../doc/dev/header" - << eof > "$(__cdist_type_dir "$__cdist_type")/${__cdist_name_gencode}-${__cdist_name_gencode_remote}"
-
-#
-# This file should generate code on stdout, which will be collected by cdist
-# and run on the target.
-#
-# To tell cdist to make use of it, you need to make it executable (chmod +x)
-#
-#
-
-eof
-
-cat "$__cdist_abs_mydir/../doc/dev/header" - << eof > "$(__cdist_type_dir "$__cdist_type")/${__cdist_name_gencode}-${__cdist_name_gencode_local}"
-
-#
-# This file should generate code on stdout, which will be collected by cdist
-# and run on the same machine cdist-deploy-to is executed.
-#
-# To tell cdist to make use of it, you need to make it executable (chmod +x)
-#
-#
-
-eof
-
-# Explorer
-mkdir -p "$__cdist_my_type_dir/${__cdist_name_explorer}"

doc/dev/todo/niconext

@@ -1,9 +1,20 @@
+- Initial install support
+   - setup $__install = "" for
+      manifest(s)
+
+   - run standard manifest (?)
+      - creates initial objects
+         - only those having the installer flag?
+            - requires changegs to cdist-type-emulator!
+               - Goto Rewrite cdist-type-emulator
+
+   - run all other manifests
+      - creates all objects
+      - what about type explorer?
+
 - Support parallel execution
    - and maximum number of parallel runs (-p X)
    - error handling / report failed hosts
-- Support different home instead of ../
-- Bug: os.path.join() may be wrong for the remote side!
-   -> does not matter for now!
 
 - Rewrite cdist-type-emulator
    - Remove legacy code in cdist
@@ -11,8 +22,6 @@
    - Remove man1/cdist-type-emulator.text
    - Remove the PATH=... part from the README
 
-- Replace bin/cdist-type-template 
-
 - Allow manifest to be read from stdin
 - Create new video for cdist 2.0.0
    http://www.youtube.com/watch?v=PRMjzy48eTI

doc/man/cdist-reference.text.sh

@@ -182,7 +182,7 @@ __target_user::
    Currently static in core.
 __type::
    Path to the current type.
-   Available for: type manifest
+   Available for: type manifest, type gencode
 __type_explorer::
    Directory that contains the type explorers.
    Available for: type explorer

doc/man/man1/cdist-type-template.text

@@ -1,30 +0,0 @@
-cdist-type-template(1)
-======================
-Nico Schottelius <nico-cdist--@--schottelius.org>
-
-
-NAME
-----
-cdist-type-template - Create a new type
-
-
-SYNOPSIS
---------
-cdist-type-template NAME
-
-
-DESCRIPTION
------------
-cdist-type-template creates a new type and adds the usual files to it.
-It is thought to be helpful when writing new types.
-
-
-SEE ALSO
---------
-cdist(7)
-
-
-COPYING
--------
-Copyright \(C) 2011 Nico Schottelius. Free use of this software is
-granted under the terms of the GNU General Public License version 3 (GPLv3).