Merge branch 'master' into changerunningorder

Conflicts:
	bin/cdist-deploy-to

Signed-off-by: Nico Schottelius <nico@kr.ethz.ch>

.gitignore

@@ -2,13 +2,8 @@
 .*.swp
 
 # Ignore generated manpages
-doc/man/*.[1-9]
 doc/man/.marker
 doc/man/man*/
 
-doc/html
+# Ignore cache for version control
-
+cache/
-conf/type/*/*.7
-conf/type/*/man.html
-
-

README

@@ -52,22 +52,28 @@ You can [browse the documentation for the latest version online](man) as well.
 
 ### Features
 
-Stuff that should probably be included in every configuration management,
+ * Elegant code and clean design
-but is not. Or: Why I began to write cdist:
+  * Type and core cleanly seperated
-
+  * Small codebase in core
- * Speed
- * Elegant code
- * Clean design
  * Good documentation (man pages)
- * Meaningful error messages
- * The no surprise factor
  * Consistency in behaviour, naming and documentation
+ * Meaningful error messages
+  * Either standard error messages from tools or added description for clearification
+ * The no surprise factor
+  * No magic guessing of what the user wants
+ * Simple and well-known DSL
+  * Posix shell
  * Easy integration into bare metal installations
- * Simple and well-known DSL: posix shell
+  * requires only ssh + sh
- * It must be very easy to extend and debug cdist
+ * Easy upgrade
- * Focus on reuse of existing functionality (like sh, ssh, find, rm, ...)
+  * ***There is no need to update cdist on target hosts!***
- * Easy upgrade: ***There is no need to update cdist on target hosts!***
+  * cdist only needs to be updated on the master server(s)
-  * cdist only needs to be update on the master server
+ * Very easy to extend
+  * Can be done via types, which can be stacked on top of others
+ * Reuse of existing functionality
+  * sh, ssh, find, rm, mv, ...
+ * Very easy to debug
+  * Just add set -x in the scripts
 
 ### OS support
 
@@ -77,6 +83,7 @@ cdist was tested or is know to run on at least
  * [Debian](http://www.debian.org/)
  * [Gentoo](http://www.gentoo.org/)
  * [Mac OS X](http://www.apple.com/macosx/)
+ * [OpenBSD](http://www.openbsd.org)
  * [Redhat](http://www.redhat.com/)
  * [Ubuntu](http://www.ubuntu.com/)
 
@@ -108,7 +115,7 @@ To install cdist, execute the following commands:
     export PATH=$PATH:$(pwd -P)/bin
 
     # If you want the manpages (requires gmake and asciidoc to be installed)
-    make man
+    ./build.sh man
     export MANPATH=$MANPATH:$(pwd -P)/doc/man
 
 
@@ -150,7 +157,7 @@ To upgrade cdist in the current branch use
     git pull
 
     # Also update the manpages
-    make man
+    ./build.sh man
     export MANPATH=$MANPATH:$(pwd -P)/doc/man
 
 If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break.

bin/cdist-cache

@@ -0,0 +1,37 @@
+#!/bin/sh
+#
+# 2010 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+# 
+# Let's build a cconfig tree from a configuration
+# And save it into the cache tree
+#
+
+. cdist-config
+[ $# -eq 1 ] || __cdist_usage "<target host>"
+set -u
+
+__cdist_target_host="$1"; shift
+
+mkdir -p "${__cdist_local_base_cache_dir}"
+__cdist_abs_cache_dir="$(cd "$__cdist_local_base_cache_dir" && pwd -P)"
+__cdist_abs_ddir="$__cdist_abs_cache_dir/$__cdist_target_host"
+
+echo "Saving cache to $__cdist_abs_ddir ..."
+rm -rf "$__cdist_abs_ddir"
+mv "$__cdist_local_base_dir" "$__cdist_abs_ddir"

bin/cdist-config

@@ -19,7 +19,7 @@
 #
 #
 
-__cdist_version="1.4.1"
+__cdist_version="1.5.0pre"
 
 # Fail if something bogus is going on
 set -u
@@ -41,6 +41,7 @@ __cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname"
 #
 
 : ${__cdist_name_bin:=bin}
+: ${__cdist_name_cache:=cache}
 : ${__cdist_name_code:=code}
 : ${__cdist_name_code_finished:=codedone}
 : ${__cdist_name_conf_dir:=conf}
@@ -100,6 +101,7 @@ __cdist_tmp_file=$(mktemp "$__cdist_tmp_dir/cdist.XXXXXXXXXXXX")
 # Local Base
 # 
 : ${__cdist_local_base_dir:=$__cdist_tmp_dir}
+: ${__cdist_local_base_cache_dir:=$__cdist_abs_mydir/../$__cdist_name_cache}
 
 : ${__cdist_conf_dir:="$(cd "$__cdist_abs_mydir/../conf" && pwd -P)"}
 

bin/cdist-deploy-to

@@ -51,5 +51,6 @@ cdist-explorer-run-global     "$__cdist_target_host"
 cdist-manifest-run-init       "$__cdist_target_host"
 cdist-manifest-run-all        "$__cdist_target_host"
 cdist-object-run-all          "$__cdist_target_host"
+cdist-cache                   "$__cdist_target_host"
 
 echo "cdist $__cdist_version: Successfully finished run on $__cdist_target_host"

bin/cdist-type-emulator

@@ -155,7 +155,7 @@ if [ -e "${__cdist_object_dir}" ]; then
    # Allow diff to fail
    set +e
    diff -ru "${__cdist_new_object_dir}/${__cdist_name_parameter}" \
-      "${__cdist_object_dir}/${__cdist_name_parameter}" \
+      "${__cdist_object_dir}/${__cdist_name_parameter}"           \
       > "$__cdist_tmp_file"; ret=$?
    set -e
    

build.sh

@@ -41,6 +41,7 @@ case "$1" in
    man)
       set -e
       "$0" mandirs
+      "$0" mangen
       "$0" mantype
       "$0" man1
       "$0" man7
@@ -50,9 +51,10 @@ case "$1" in
    manbuild)
       for src in ${MAN1DSTDIR}/*.text ${MAN7DSTDIR}/*.text; do
          echo "Compiling manpage and html for $src"
-         $A2XM "$src"
+         $A2XM "$src" &
-         $A2XH "$src"
+         $A2XH "$src" &
       done
+      wait
    ;;
 
    mandirs)
@@ -61,7 +63,7 @@ case "$1" in
    ;;
 
    mantype)
-	   for mansrc in ${MAN7TYPESRC}; do
+	   for mansrc in conf/type/*/man.text; do
          dst="$(echo $mansrc | sed -e 's;conf/;cdist-;'  -e 's;/;;' -e 's;/man;;' -e 's;^;doc/man/man7/;')"
          ln -sf "../../../$mansrc" "$dst"
       done
@@ -83,9 +85,9 @@ case "$1" in
    ;;
 
    man7)
-      for man in cdist.text cdist-best-practise.text cdist-hacker.text         \
+      for man in cdist.text cdist-best-practice.text cdist-hacker.text         \
       cdist-quickstart.text cdist-reference.text cdist-stages.text             \
-      cdist-type.text
+      cdist-type.text cdist-cache.text
          do
          ln -sf ../$man ${MAN7DSTDIR}
       done

conf/type/__process/explorer/runs

@@ -0,0 +1,30 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Check whether a process runs
+#
+
+if [ -f "$__object/parameter/name" ]; then
+   name="$(cat "$__object/parameter/name")"
+else
+   name="$__object_id"
+fi
+
+pgrep -x -f "$name" || true

conf/type/__process/gencode-remote

@@ -0,0 +1,59 @@
+#!/bin/sh
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# __file is a very basic type, which will probably be reused quite often
+#
+
+if [ -f "$__object/parameter/name" ]; then
+   name="$(cat "$__object/parameter/name")"
+else
+   name="$__object_id"
+fi
+
+runs="$(cat "$__object/explorer/runs")"
+state="$(cat "$__object/parameter/state")"
+
+case "$state" in
+   running)
+      # Does not run, start it!
+      if [ -z "$runs" ]; then
+         if [ -f "$__object/parameter/start" ]; then
+            cat "$__object/parameter/start"
+         else
+            echo "$name"
+         fi
+      fi
+   ;;
+   stopped)
+      # Runs, kill it!
+      if [ "$runs" ]; then
+         if [ -f "$__object/parameter/stop" ]; then
+            cat "$__object/parameter/stop"
+         else
+            echo kill "${runs}"
+         fi
+      fi
+   ;;
+   *)
+      echo "Unknown state: $state" >&2
+      exit 1
+   ;;
+
+esac

conf/type/__process/man.text

@@ -0,0 +1,64 @@
+cdist-type__process(7)
+======================
+Nico Schottelius <nico-cdist--@--schottelius.org>
+
+
+NAME
+----
+cdist-type__process - Start or stop process
+
+
+DESCRIPTION
+-----------
+This cdist type allows you to define the state of a process.
+
+
+REQUIRED PARAMETERS
+-------------------
+state::
+   State of the process: Either stopped or running.
+
+
+OPTIONAL PARAMETERS
+-------------------
+name::
+   Process name to match on when using pgrep -f -x.
+
+stop::
+   Executable to use for stopping the process.
+
+start::
+   Executable to use for starting the process.
+
+
+EXAMPLES
+--------
+
+--------------------------------------------------------------------------------
+# Start /usr/sbin/sshd if not running
+__process /usr/sbin/sshd --state running
+
+# Start /usr/sbin/sshd if not running with a different binary
+__process /usr/sbin/sshd --state running --start "/etc/rc.d/sshd start"
+
+# Stop the process using kill (the type default)
+__process /usr/sbin/sshd --state stopped
+
+# Stop the process using /etc/rc.d/sshd stop
+__process /usr/sbin/sshd --state stopped --stop "/etc/rc.d/sshd stop"
+
+# Ensure cups is running, which runs with -C ...:
+__process cups --start "/etc/rc.d/cups start" --state running \
+   --name "/usr/sbin/cupsd -C /etc/cups/cupsd.conf"
+--------------------------------------------------------------------------------
+
+
+SEE ALSO
+--------
+- cdist-type(7)
+
+
+COPYING
+-------
+Copyright \(C) 2011 Nico Schottelius. Free use of this software is
+granted under the terms of the GNU General Public License version 3 (GPLv3).

conf/type/__process/parameter/optional

@@ -0,0 +1,3 @@
+name
+stop
+start

conf/type/__process/parameter/required

@@ -0,0 +1 @@
+state

doc/changelog

@@ -1,3 +1,8 @@
+1.5.0:
+	* Add cache functionality
+	* New type __process
+	* Restructured execution: Run whole object at once (REPHRASE) (Steven Armstrong)
+
 1.4.1: 2011-03-25
 	* New type __key_value (Steven Armstrong)
 	* New type __apt_ppa (Steven Armstrong)
@@ -41,7 +46,6 @@
 	* Document cdist-object-gencode-all
 	* Document cdist-manifest-run
 
-
 1.1.0: 2011-03-16
 	* Replace type __file with __file, __directory, __link
 	* Document cdist-env

doc/dev/logs/2011-03-27.pgrep

@@ -0,0 +1,12 @@
+Some pgrep fun when grepping for -f /usr/lib/postfix/master:
+
+[23:08] kr:cdist% cat cache/localhost/out/object/__process/usr/lib/postfix/master/.cdist/explorer/runs | grep -e 2529 -e 2537 -e 2538 -e 2539
+nico      2529  0.0  0.0  14848  1816 pts/45   S+   23:08   0:00 /bin/sh /home/users/nico/oeffentlich/rechner/projekte/cdist/bin/cdist-run-remote localhost __object="/var/lib/cdist/out/object/__process/usr/lib/postfix/master" __object_id="usr/lib/postfix/master" cdist-remote-explorer-run __type_explorer /var/lib/cdist/conf/type/__process/explorer /var/lib/cdist/out/object/__process/usr/lib/postfix/master/explorer
+nico      2537  0.0  0.0  41976  2324 pts/45   S+   23:08   0:00 ssh root@localhost export PATH="/var/lib/cdist/bin:$PATH"; __object="/var/lib/cdist/out/object/__process/usr/lib/postfix/master" __object_id="usr/lib/postfix/master" cdist-remote-explorer-run __type_explorer /var/lib/cdist/conf/type/__process/explorer /var/lib/cdist/out/object/__process/usr/lib/postfix/master/explorer
+root      2538  0.0  0.0  11440  1264 ?        Ss   23:08   0:00 bash -c export PATH="/var/lib/cdist/bin:$PATH"; __object="/var/lib/cdist/out/object/__process/usr/lib/postfix/master" __object_id="usr/lib/postfix/master" cdist-remote-explorer-run __type_explorer /var/lib/cdist/conf/type/__process/explorer /var/lib/cdist/out/object/__process/usr/lib/postfix/master/explorer
+root      2539  0.0  0.0  11440  1524 ?        S    23:08   0:00 /bin/sh /var/lib/cdist/bin/cdist-remote-explorer-run __type_explorer /var/lib/cdist/conf/type/__process/explorer /var/lib/cdist/out/object/__process/usr/lib/postfix/master/explorer
+2529
+2537
+2538
+2539
+

doc/dev/todo/TAKEME

@@ -5,7 +5,8 @@ Feel free to pick one!
 
 CORE
 ----
-Only build manpages if necessary for types as well as for the core!
+- Only build manpages if necessary for types as well as for the core!
+- support $__self = relative_type/object_id
 
 TYPES
 ------
@@ -17,3 +18,4 @@ Types to be written/extended:
          -> aka sed.
    - __cron
 
+

doc/dev/todo/niconext

@@ -1,23 +1,9 @@
-Core:
-   - support $__self = relative_type/object_id
-   - cache
-
 Cache:
-   Assume you want to configure stuff one host ("monitor node"),
+   - add example how to use 
-   depending on the configuration of other hosts ("cluster nodes").
+   - export variable $__cache
+      -> for current host
+      -> add function to cdist-config, import from cdist-cache
 
-   For instance, the monitor host would like to know,
-   which hosts are configured with the provider
-   "apache" and option --start true.
 
-   This requires the monitor node to be able to
+remove --preseed from package_apt and add debconf_set_selection or similar
-   query all other configured nodes. It can't
+   -> much cleaner!
-   ask for all hosts, because cdist does not
-   know which hosts are configured or may exist.
-
-   Example implementation
-
-   If cdist keeps ("caches") the configuration of every
-   node it configures, each new node can query the
-   cache for existing nodes that acquired the given
-   configuration.

doc/man/cdist-best-practice.text

@@ -1,10 +1,10 @@
-cdist-best-practise(7)
+cdist-best-practice(7)
 ======================
 Nico Schottelius <nico-cdist--@--schottelius.org>
 
 NAME
 ----
-cdist-best-practise - Practises used in real environments
+cdist-best-practice - Practices used in real environments
 
 
 PASSWORDLESS CONNECTIONS
@@ -61,7 +61,6 @@ machine-b % git clone git://your-git-server/cdist
 --------------------------------------------------------------------------------
 
 
-
 SEE ALSO
 --------
 - cdist(7)

doc/man/cdist-cache.text

@@ -0,0 +1,31 @@
+cdist-cache(1)
+==============
+Nico Schottelius <nico-cdist--@--schottelius.org>
+
+
+NAME
+----
+cdist-cache - Cache output of last run
+
+
+SYNOPSIS
+--------
+cdist-cache TARGET_HOST
+
+
+DESCRIPTION
+-----------
+cdist-cache moves away the objects created during last run so the
+next run can use the previous information and compare them with
+the current status.
+
+
+SEE ALSO
+--------
+cdist(7)
+
+
+COPYING
+-------
+Copyright \(C) 2011 Nico Schottelius. Free use of this software is
+granted under the terms of the GNU General Public License version 3 (GPLv3).

doc/man/cdist.text

@@ -10,8 +10,13 @@ cdist - Configuration management system
 
 DESCRIPTION
 -----------
-Cdist is a very simple, elegant, cleanly designed configuration management
+Cdist is a usable configuration management system.
-system. The easiest way to get started with cdist is to run cdist-quickstart.
+The easiest way to get started with cdist is to initialise
+the environment and run cdist-quickstart:
+
+--------------------------------------------------------------------------------
+eval `./bin/cdist-env` && cdist-quickstart
+--------------------------------------------------------------------------------
 
 Cdist configurations are written in the shell scripting language.
 The mapping of configurations to hosts is defined in so called manifests,