implement the big code shuffle

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>

bin/cdist-config

@@ -57,6 +57,7 @@ __cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname"
 : ${__cdist_name_object_finished:=done}
 : ${__cdist_name_object_id:=object_id}
 : ${__cdist_name_object_source:=source}
+: ${__cdist_name_objects_created:=.objects_created}
 : ${__cdist_name_out_dir:=out}
 : ${__cdist_name_parameter:=parameter}
 : ${__cdist_name_parameter_required:=required}
@@ -68,6 +69,7 @@ __cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname"
 : ${__cdist_name_type:=type}
 : ${__cdist_name_type_bin:=type_bin}
 : ${__cdist_name_type_explorer:=type_explorer}
+: ${__cdist_name_type_explorer_pushed:=.explorer_pushed}
 
 # Used for IDs: Allow everything not starting with - and .
 : ${__cdist_sane_regexp:=[^-\.].*}
@@ -115,6 +117,7 @@ __cdist_tmp_file=$(mktemp "$__cdist_tmp_dir/cdist.XXXXXXXXXXXX")
 : ${__cdist_out_dir:=$__cdist_local_base_dir/$__cdist_name_out_dir}
 : ${__cdist_out_explorer_dir:=$__cdist_out_dir/$__cdist_name_explorer}
 : ${__cdist_out_object_dir:=$__cdist_out_dir/$__cdist_name_object}
+: ${__cdist_out_type_dir:=$__cdist_out_dir/$__cdist_name_type}
 : ${__cdist_out_type_bin_dir:=$__cdist_out_dir/$__cdist_name_type_bin}
 
 ################################################################################
@@ -134,6 +137,11 @@ __cdist_tmp_file=$(mktemp "$__cdist_tmp_dir/cdist.XXXXXXXXXXXX")
 : ${__cdist_remote_out_explorer_dir:=$__cdist_remote_out_dir/$__cdist_name_explorer}
 : ${__cdist_remote_out_object_dir:=$__cdist_remote_out_dir/$__cdist_name_object}
 
+################################################################################
+# Global internal variables
+#
+: ${__cdist_objects_created:=$__cdist_out_object_dir/$__cdist_name_objects_created}
+
 ################################################################################
 # Internal functions
 #
@@ -329,6 +337,18 @@ __cdist_type_has_explorer()
    fi
 }
 
+__cdist_type_explorer_pushed()
+{
+   [ -f "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" ] \
+      && grep -q "$1" "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}"
+}
+
+__cdist_type_explorer_pushed_add()
+{
+   [ -d "$__cdist_out_type_dir" ] || mkdir "$__cdist_out_type_dir"
+   echo "$1" >> "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}"
+}
+
 __cdist_type_gencode()
 {
    echo "${__cdist_type_dir}/$1/${__cdist_name_gencode}"

bin/cdist-object-explorer-run

@@ -32,11 +32,19 @@ __object="$1"; shift
 __object_id="$(__cdist_object_id_from_object "$__object")"
 __cdist_type="$(__cdist_type_from_object "$__object")"
 
-# Check if type of object has >= 1 explorer
-__cdist_has_explorer="$(__cdist_type___cdist_has_explorer "$__cdist_type")"
 
-# If so, run explorers on remote side
+# Check if type of object has >= 1 explorer
+__cdist_has_explorer="$(__cdist_type_has_explorer "$__cdist_type")"
+# Run the type explorers for the current object if any
 if [ "$__cdist_has_explorer" ]; then
+   if ! __cdist_type_explorer_pushed "$__cdist_type"; then
+      src_dir="$(__cdist_type_explorer_dir "$__cdist_type")"
+      dst_dir="$(__cdist_remote_type_explorer_dir "$__cdist_type")"
+      echo "Transfering explorers for $__cdist_type ..."
+      cdist-dir push "$__cdist_target_host" "$src_dir" "$dst_dir"
+      __cdist_type_explorer_pushed_add "$__cdist_type"
+   fi
+
    echo "Running explorers for $__object ..."
    # Copy object parameters
    cdist-dir push "$__cdist_target_host"                       \

bin/cdist-object-manifest-run

@@ -52,8 +52,8 @@ if [ -f "$__cdist_manifest" ]; then
       export $__cdist_name_var_type="$(__cdist_type_dir "$__cdist_type")"
 
       cdist-manifest-run "$__cdist_target_host" "$__cdist_manifest"
-      FIXME: Tell cdist that there may be new objects - WHO? Mama?
+      # Tell cdist-object-run-all that there may be new objects
-      touch "$__cdist_new_objects_created"
+      touch "$__cdist_objects_created"
    else
       __cdist_exit_err "${__cdist_manifest} needs to be executable."
    fi

bin/cdist-object-run

@@ -40,23 +40,26 @@ touch "$__cdist_types_pushed"
 
 __cdist_object_finished="$(__cdist_object_finished "$__cdist_object")"
 if [ ! -f "$__cdist_object_finished" ]; then
-   echo
-   echo "Running object $__cdist_object"
 
-   # Check if type of object has >= 1 explorer
+   # Resolve dependencies if any
-   __cdist_has_explorer="$(__cdist_type_has_explorer "$__cdist_type")"
+   __cdist_object_require="$(__cdist_object_require "$__cdist_object")"
+   if [ -f "$__cdist_object_require" ]; then
+      echo 
+      # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP
+      while read __cdist_requirement; do
+         set -- "$@" "$__cdist_requirement"
+      done < "$__cdist_object_require"
 
-   FIXME: put into cdist-object-explorer-run
+      while [ $# -gt 0 ]; do
-   # Run the type explorers for the current object if any
+         __cdist_requirement="$1"; shift
-   if [ "$__cdist_has_explorer" ]; then
+         echo "Resolving dependency $__cdist_requirement for $__cdist_object  ..."
-      if ! grep -q "$__cdist_type" "$__cdist_types_pushed"; then
+         cdist-object-run "$__cdist_target_host" "$__cdist_requirement"
-         cdist-type-explorer-push "$__cdist_target_host" "$__cdist_type"
+      done
-         echo "$__cdist_type" >> "$__cdist_types_pushed" 
-      fi 
-
-      cdist-object-explorer-run "$__cdist_target_host" "$__cdist_object"
    fi
 
+   echo
+   echo "Running object $__cdist_object"
+   cdist-object-explorer-run  "$__cdist_target_host" "$__cdist_object"
    cdist-object-manifest-run  "$__cdist_target_host" "$__cdist_object"
    cdist-object-gencode-run   "$__cdist_target_host" "$__cdist_object"
    cdist-object-push          "$__cdist_target_host" "$__cdist_object"

bin/cdist-object-run-all

@@ -30,15 +30,12 @@ __cdist_target_host="$1"; shift
 
 __cdist_objects="$__cdist_tmp_dir/objects"
 
-FIXME: reuse in subscripts, save in objects_base_dir
-export __cdist_objects_created="$__cdist_tmp_dir/objects_created"
-
 # Loop until we do not create new objects anymore
 # which is equal to all objects have been run
-touch "$__cdist_new_objects_created"
+touch "$__cdist_objects_created"
-while [ -f "$__cdist_new_objects_created" ]; do
+while [ -f "$__cdist_objects_created" ]; do
    # Assume we're done after this run
-   rm "$__cdist_new_objects_created"
+   rm "$__cdist_objects_created"
 
    # Get listing of objects
    __cdist_object_list "$__cdist_out_object_dir" > "$__cdist_objects"
@@ -50,18 +47,6 @@ while [ -f "$__cdist_new_objects_created" ]; do
 
    while [ $# -gt 0 ]; do
       __cdist_object="$1"; shift
-
-      FIXME: migrate into cdist-object-run
-      FIXME: take care of SSH foo after migration in while loop
-      __cdist_object_require="$(__cdist_object_require "$__cdist_object")"
-      if [ -f "$__cdist_object_require" ]; then
-         echo 
-         while read __cdist_requirement; do
-            echo "Resolving dependency $__cdist_requirement for $__cdist_object  ..."
-            cdist-object-run "$__cdist_target_host" "$__cdist_requirement"
-         done < "$__cdist_object_require"
-      fi
-
       # Process the object
       cdist-object-run "$__cdist_target_host" "$__cdist_object"
    done

bin/cdist-type-explorer-push

@@ -1,40 +0,0 @@
-#!/bin/sh
-#
-# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
-# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-# 
-# Push the given types explorers to the target host
-#
-
-. cdist-config
-[ $# -eq 2 ] || __cdist_usage "<target host> <type>"
-
-set -eu
-
-__cdist_target_host="$1"; shift
-__cdist_type="$1"; shift
-
-src_dir="$(__cdist_type_explorer_dir "$__cdist_type")"
-dst_dir="$(__cdist_remote_type_explorer_dir "$__cdist_type")"
-
-if [ -d "$src_dir" ]; then
-   echo "Transfering explorers for $__cdist_type ..."
-   cdist-dir push "$__cdist_target_host" "$src_dir" "$dst_dir"
-fi
-

doc/dev/todo/steven-from-nico

@@ -31,28 +31,29 @@ x cdist-object-manifest-run:
 
 x general: cdist-object-run-all looks like a good idea!
 
-- export $__cdist_name_var_self=$__cdist_object_self -> non core
+x cdist-type-explorer-push: wherefore if [ -d "$src_dir" ];?
-
-- bin/cdist-object-run: type_explorer stuff: probably put into own binary
-   - sounds like cdist-object-explorer-run is already the right executable to
-      place such stuff into
-
-- cdist-type-explorer-push: wherefore if [ -d "$src_dir" ];?
    -> does this not even hide bugs?
    -> not sure whether covering cdist-dir in its own script makes
       sense, as cdist-dir push is only a one liner
       -> if cdist-dir does too less, enhance it
    => merge into cdist-object-explorer-run
 
-- marker for type transferred / pushed goes into out/type/
+x bin/cdist-object-run: type_explorer stuff: probably put into own binary
+   - sounds like cdist-object-explorer-run is already the right executable to
+      place such stuff into
+
+x marker for type transferred / pushed goes into out/type/
    -> new variables for out/type
    -> new variables for out/type/.MARKERFOONAMEMEGOOD
 
-- new function: __cdist_type_explorer_created $name
+x new function: __cdist_type_explorer_created $name
    if ! -d foo -> mkdir foo, echo $name >> foo/$NEW_FANCY_VAR
 
+- export $__cdist_name_var_self=$__cdist_object_self -> non core
+
 
 - align messages (already in todo for steven, but makes life much easier, thus repeated)
    -> prefix all object stuff with $__self
    -> __cdist_echo object string
       -> $__cdist_object_self :-)
+

doc/man/cdist-stages.text

@@ -51,8 +51,11 @@ Related manpages:
    - cdist-manifest-run-init(1)
 
 
-STAGE 3: EXECUTION OF TYPES
+STAGE 3: EXECUTION OF OBJECTS
----------------------------
+-----------------------------
+Each object in the cconfig database is run through the following stages:
+- 
+
 Every object is checked whether its type has a manifest file. If the type has
 a manifest file and it is executable, it will be executed. The manifest script
 may generate and change the created objects. In other words, one type can reuse