Merge branch 'master' into preos

Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>

Conflicts:
	docs/changelog

cdist/conf/type/__file/gencode-local

@@ -30,6 +30,7 @@ create_file=
 if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then
    if [ ! -f "$__object/parameter/source" ]; then
       create_file=1
+      echo create >> "$__messages_out"
    else
       source="$(cat "$__object/parameter/source")"
       if [ "$source" = "-" ]; then
@@ -64,6 +65,7 @@ if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then
 destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")"
 DONE
       if [ "$upload_file" ]; then
+         echo upload >> "$__messages_out"
          cat << DONE
 $__remote_copy $source ${__target_host}:\$destination_upload
 DONE

cdist/conf/type/__file/gencode-remote

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
 # 2013 Steven Armstrong (steven-cdist armstrong.cc)
 #
 # This file is part of cdist.
@@ -26,58 +26,69 @@ stat_file="$__object/explorer/stat"
 
 
 get_current_value() {
-   if [ -s "$stat_file" ]; then
+    if [ -s "$stat_file" ]; then
-      _name="$1"
+        _name="$1"
-      _value="$2"
+        _value="$2"
-      case "$_value" in
+        case "$_value" in
-         [0-9]*)
+            [0-9]*)
-            _index=2
+                _index=2
-         ;;
+            ;;
-         *)
+            *)
-            _index=3
+                _index=3
-         ;;
+            ;;
-      esac
+        esac
-      awk '/'"$_name"':/ { print $'$_index' }' "$stat_file"
+        awk '/'"$_name"':/ { print $'$_index' }' "$stat_file"
-      unset _name _value _index
+        unset _name _value _index
-   fi
+    fi
 }
 
 set_group() {
-   echo chgrp \"$1\" \"$destination\"
+    echo chgrp \"$1\" \"$destination\"
+    echo chgrp $1 >> "$__messages_out"
 }
 
 set_owner() {
-   echo chown \"$1\" \"$destination\"
+    echo chown \"$1\" \"$destination\"
+    echo chown $1 >> "$__messages_out"
 }
 
 set_mode() {
    echo chmod \"$1\" \"$destination\"
+   echo chmod $1 >> "$__messages_out"
 }
 
 set_attributes=
 case "$state_should" in
-   present|exists)
+    present|exists)
-      # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
+    # Note: Mode - needs to happen last as a chown/chgrp can alter mode by
-      #  clearing S_ISUID and S_ISGID bits (see chown(2))
+    #  clearing S_ISUID and S_ISGID bits (see chown(2))
-      for attribute in group owner mode; do
+    for attribute in group owner mode; do
-         if [ -f "$__object/parameter/$attribute" ]; then
+        if [ -f "$__object/parameter/$attribute" ]; then
             value_should="$(cat "$__object/parameter/$attribute")"
+
+            # change 0xxx format to xxx format => same as stat returns
+            if [ "$attribute" = mode ]; then
+                value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')"
+            fi
+            
             value_is="$(get_current_value "$attribute" "$value_should")"
             if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then
-               "set_$attribute" "$value_should"
+                "set_$attribute" "$value_should"
             fi
-         fi
+        fi
-      done
+    done
-   ;;
 
-   absent)
+    ;;
-      if [ "$type" = "file" ]; then
-         echo rm -f \"$destination\"
-      fi
-   ;;
 
-   *)
+    absent)
-      echo "Unknown state: $state_should" >&2
+        if [ "$type" = "file" ]; then
-      exit 1
+            echo rm -f \"$destination\"
-   ;;
+            echo remove >> "$__messages_out"
+        fi
+    ;;
+
+    *)
+        echo "Unknown state: $state_should" >&2
+        exit 1
+    ;;
 esac

cdist/conf/type/__file/man.text

@@ -50,6 +50,21 @@ source::
    If not supplied, an empty file or directory will be created.
    If source is '-' (dash), take what was written to stdin as the file content.
 
+MESSAGES
+--------
+chgrp <group>::
+    Changed group membership
+chown <owner>::
+    Changed owner
+chmod <mode>::
+    Changed mode
+create::
+    Empty file was created (no --source specified)
+remove::
+    File exists, but state is absent, file will be removed by generated code.
+upload::
+    File was uploaded
+
 
 EXAMPLES
 --------
@@ -90,5 +105,5 @@ SEE ALSO
 
 COPYING
 -------
-Copyright \(C) 2011-2012 Nico Schottelius. Free use of this software is
+Copyright \(C) 2011-2013 Nico Schottelius. Free use of this software is
 granted under the terms of the GNU General Public License version 3 (GPLv3).

cdist/conf/type/__iptables_apply/gencode-remote

@@ -1,2 +1,3 @@
-# Rebuild rules - FIXME: do conditionally as soon as cdist supports it
+if grep -q "^__file/etc/iptables.d/" "$__messages_in"; then
-echo /etc/init.d/iptables restart
+    echo /etc/init.d/iptables restart
+fi

cdist/conf/type/__iptables_rule/manifest

@@ -21,12 +21,7 @@
 base_dir=/etc/iptables.d
 
 name="$__object_id"
-
+state="$(cat "$__object/parameter/state")"
-if [ -f "$__object/parameter/state" ]; then
-    state="$(cat "$__object/parameter/state")"
-else
-    state="present"
-fi
 
 ################################################################################
 # Basic setup

cdist/conf/type/__iptables_rule/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__package_zypper/explorer/pkg_version

@@ -27,4 +27,4 @@ else
    name="$__object_id"
 fi
 
-rpm -q --whatprovides "$name" 2>/dev/null || true
+rpm -q --whatprovides "$name"  | grep -v 'no package provides' || true

cdist/conf/type/__package_zypper/gencode-remote

@@ -39,15 +39,22 @@ else
    state_should="present"
 fi
 
+pkg_version="$(cat "$__object/explorer/pkg_version")"
+if [ -z "$pkg_version" ]; then
+    state_is="absent"
+else
+    state_is="present"
+fi
+
 # Exit if nothing is needed to be done
 [ "$state_is" = "$state_should" ] && exit 0
 
 case "$state_should" in
    present)
-         echo zypper "$globalopts" install --auto-agree-with-licenses \"$name\"
+         echo zypper $globalopts install --auto-agree-with-licenses \"$name\" ">/dev/null"
    ;;
    absent)
-         echo pacman "$globalopts" remove \"$name\"
+         echo zypper $globalopts remove \"$name\" ">/dev/null"
    ;;
    *)
       echo "Unknown state: $state_should" >&2

cdist/core/code.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -104,7 +104,8 @@ class Code(object):
                 '__object_id': cdist_object.object_id,
                 '__object_name': cdist_object.name,
             })
-            return self.local.run_script(script, env=env, return_output=True)
+            message_prefix=cdist_object.name
+            return self.local.run_script(script, env=env, return_output=True, message_prefix=message_prefix)
 
     def run_gencode_local(self, cdist_object):
         """Run the gencode-local script for the given cdist object."""
@@ -119,9 +120,6 @@ class Code(object):
         source = os.path.join(self.local.object_path, cdist_object.code_remote_path)
         destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path)
         # FIXME: BUG: do not create destination, but top level of destination!
-        # FIXME: BUG2: we are called AFTER the code-remote has been transferred already:
-        # mkdir: cannot create directory `/var/lib/cdist/object/__directory/etc/acpi/actions/.cdist/code-remote': File exists
-        # OR: this is from previous run -> cleanup missing!
         self.remote.mkdir(destination)
         self.remote.transfer(source, destination)
 

cdist/core/manifest.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -122,7 +122,8 @@ class Manifest(object):
         if not os.path.isfile(initial_manifest):
             raise NoInitialManifestError(initial_manifest, user_supplied)
 
-        self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest))
+        message_prefix="initialmanifest"
+        self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix)
 
     def env_type_manifest(self, cdist_object):
         type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
@@ -141,5 +142,6 @@ class Manifest(object):
 
     def run_type_manifest(self, cdist_object):
         type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
+        message_prefix = cdist_object.name
         if os.path.isfile(type_manifest):
            self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object))

cdist/exec/local.py

@@ -30,6 +30,7 @@ import logging
 import tempfile
 
 import cdist
+import cdist.message
 from cdist import core
 
 class Local(object):
@@ -92,6 +93,7 @@ class Local(object):
         self.conf_path = os.path.join(self.base_path, "conf")
         self.global_explorer_out_path = os.path.join(self.base_path, "explorer")
         self.object_path = os.path.join(self.base_path, "object")
+        self.messages_path = os.path.join(self.base_path, "messages")
 
         # Depending on conf_path
         self.global_explorer_path = os.path.join(self.conf_path, "explorer")
@@ -128,6 +130,7 @@ class Local(object):
     def create_files_dirs(self):
         self._init_directories()
         self._create_conf_path_and_link_conf_dirs()
+        self._create_messages()
         self._link_types_for_emulator()
 
 
@@ -150,7 +153,7 @@ class Local(object):
         self.log.debug("Local mkdir: %s", path)
         os.makedirs(path, exist_ok=True)
 
-    def run(self, command, env=None, return_output=False):
+    def run(self, command, env=None, return_output=False, message_prefix=None):
         """Run the given command with the given environment.
         Return the output as a string.
 
@@ -163,6 +166,10 @@ class Local(object):
         # Export __target_host for use in __remote_{copy,exec} scripts
         env['__target_host'] = self.target_host
 
+        if message_prefix:
+            message = cdist.message.Message(message_prefix, self.messages_path)
+            env.update(message.env)
+
         try:
             if return_output:
                 return subprocess.check_output(command, env=env).decode()
@@ -172,8 +179,11 @@ class Local(object):
             raise cdist.Error("Command failed: " + " ".join(command))
         except OSError as error:
             raise cdist.Error(" ".join(*args) + ": " + error.args[1])
+        finally:
+            if message_prefix:
+                message.merge_messages()
 
-    def run_script(self, script, env=None, return_output=False):
+    def run_script(self, script, env=None, return_output=False, message_prefix=None):
         """Run the given script with the given environment.
         Return the output as a string.
 
@@ -181,7 +191,7 @@ class Local(object):
         command = ["/bin/sh", "-e"]
         command.append(script)
 
-        return self.run(command, env, return_output)
+        return self.run(command=command, env=env, return_output=return_output, message_prefix=message_prefix)
 
     def save_cache(self):
         destination = os.path.join(self.cache_path, self.target_host)
@@ -195,6 +205,11 @@ class Local(object):
 
         shutil.move(self.base_path, destination)
 
+    def _create_messages(self):
+        """Create empty messages"""
+        with open(self.messages_path, "w"):
+            pass
+
     def _create_conf_path_and_link_conf_dirs(self):
         # Link destination directories
         for sub_dir in [ "explorer", "manifest", "type" ]:

cdist/message.py

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+import os
+import shutil
+import tempfile
+
+import cdist
+
+log = logging.getLogger(__name__)
+
+
+class Message(object):
+    """Support messaging between types
+
+    """
+    def __init__(self, prefix, messages):
+        self.prefix = prefix
+        self.global_messages = messages
+
+        self.messages_in  = tempfile.mkstemp(suffix='.cdist_message_in')[1]
+        self.messages_out = tempfile.mkstemp(suffix='.cdist_message_out')[1]
+
+        self._copy_messages()
+
+
+    @property
+    def env(self):
+        env = {}
+        env['__messages_in']  = self.messages_in
+        env['__messages_out'] = self.messages_out
+
+        return env
+
+    def _copy_messages(self):
+        """Copy global contents into our copy"""
+        shutil.copyfile(self.global_messages, self.messages_in)
+
+    def _cleanup(self):
+        """remove temporary files"""
+        if os.path.exists(self.messages_in):
+            os.remove(self.messages_in)
+        if os.path.exists(self.messages_out):
+            os.remove(self.messages_out)
+
+    def _merge_messages(self):
+        """merge newly written lines into global file"""
+        with open(self.messages_out) as fd:
+            content = fd.readlines()
+
+        with open(self.global_messages, 'a') as fd:
+            for line in content:
+                fd.write("%s:%s" % (self.prefix, line))
+
+    def merge_messages(self):
+        self._merge_messages()
+        self._cleanup()

cdist/test/message/__init__.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+#
+# 2013 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import tempfile
+
+from cdist import test
+import cdist.message
+
+class MessageTestCase(test.CdistTestCase):
+
+    def setUp(self):
+        self.prefix="cdist-test"
+        self.content = "A very short story"
+        self.tempfile = tempfile.mkstemp()[1]
+        self.message = cdist.message.Message(prefix=self.prefix,
+            messages=self.tempfile)
+
+    def tearDown(self):
+        os.remove(self.tempfile)
+        self.message._cleanup()
+
+    def test_env(self):
+        """
+        Ensure environment is correct
+        """
+
+        env = self.message.env
+
+        self.assertIn('__messages_in', env)
+        self.assertIn('__messages_out', env)
+
+
+    def test_copy_content(self):
+        """
+        Ensure content copying is working
+        """
+
+        with open(self.tempfile, "w") as fd:
+            fd.write(self.content)
+
+        self.message._copy_messages()
+
+        with open(self.tempfile, "r") as fd:
+            testcontent = fd.read()
+
+        self.assertEqual(self.content, testcontent)
+
+    def test_message_merge_prefix(self):
+        """Ensure messages are merged and are prefixed"""
+
+        expectedcontent = "%s:%s" % (self.prefix, self.content)
+
+        out = self.message.env['__messages_out']
+
+        with open(out, "w") as fd:
+            fd.write(self.content)
+
+        self.message._merge_messages()
+
+        with open(self.tempfile, "r") as fd:
+            testcontent = fd.read()
+
+        self.assertEqual(expectedcontent, testcontent)

docs/changelog

@@ -4,12 +4,19 @@ Changelog
 	* Changes are always commented with their author in (braces)
 	* Exception: No braces means author == Nico Schottelius
 
-2.3.7:
+
+3.0.0:
+	* Core: Messaging support added
+	* Type: __iptables_rule: Use default parameter
+	* Type __file: Do not generate code if mode is 0xxx
+
+2.3.7: 2013-12-02
 	* Type __file: Secure the file transfer by using mktemp (Steven Armstrong)
 	* Type __file: Only remove file when state is absent (Steven Armstrong)
 	* Type __link: Only remove link when state is absent (Steven Armstrong)
 	* Type __directory: Only remove directory when state is absent (Steven Armstrong)
 	* Type __directory: Fix newly introduced quoting issue
+	* Type __package_zypper: Fix explorer and parameter issue (Daniel Heule)
 	* Core: Fix backtrace when cache cannot be deleted
 
 2.3.6: 2013-11-25

docs/dev/logs/2013-01-20.notifications

@@ -0,0 +1,20 @@
+Allow cross-type communication
+
+Sending notifications is possible from
+
+    - manifest
+    - gencode-local
+    - gencode-remote
+
+Sending a notification from an object means writing to the file "notifications" into
+its object:
+
+    echo mytest >> "$__object/notifications" # a type reports something
+
+Reading / Reacting on notifications works by accessing the file
+referred to be "$__notifications". All notifications are prefixed with
+the object name ($__object_name) and are appended into this file.
+
+To find out, whether a file was copied, run:
+
+    grep __file/etc/passwd:copy "$__notifications"

docs/dev/logs/2013-01-20.triggers

@@ -0,0 +1,49 @@
+An alternative / complementary approach to notifications: triggers (or actions?)
+
+A type may support various actions by creating files in its subdirectory
+"actions". Other types can trigger an action of a different type or object
+by calling them (indirectly?):
+
+
+if grep "__file/etc/nginx/conf.d/.*:copy" "$__notifications"; then
+
+    # Call action from a type
+    cdist trigger __nginx/reload
+fi
+
+
+Not sure whether this approach (calling "actions" of other types) is sane,
+as nginx should probably better know if it should be restarted "itself".
+
+
+--------------------------------------------------------------------------------
+
+Alternate approach:
+
+__nginx_vhost www.some-domain.ch --custom << eof
+some custom code for __nginx_vhost inclusion
+eof
+
+__nginx_vhost:
+
+    manifest:
+        # __nginx_vhost requires __nginx: creates directories
+
+        require"$__object_name" __nginx --require-only
+        
+        # Do WE or __file ... depend on nginx?
+        cdist require __nginx
+
+        # Create file that contains the giving code
+        __file /etc/nginx/conf.d/www.some-domain.ch
+
+        require="__nginx" __file /etc/nginx/conf.d/www.some-domain.ch
+
+__nginx:
+    manifest:
+        __package nginx --state present
+
+        __file some-custom-files
+
+    gencode-remote:
+        if first_install or file changed:

docs/dev/logs/2013-11-25.notifications

@@ -0,0 +1,50 @@
+Follow up from 2013-01-20:
+
+    - (re-)create message file per object?
+        - yes, but do not necessarily save in object space
+        - save $anywhere
+
+    - object_run
+        - current notifications are imported into a file available at $__messages_in
+        - after object run, everything that has been written to $__messages_out is merged into the $__messages file
+
+        - functions:
+            self.explorer.run_global_explorers(self.local.global_explorer_out_path)
+            self.manifest.run_initial_manifest(self.local.initial_manifest)
+                self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest))
+            self.explorer.run_type_explorers(cdist_object)
+            self.manifest.run_type_manifest(cdist_object)
+                self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object))
+            self.code.run_gencode_local(cdist_object)
+                self.local.run_script(script, env=env, return_output=True)
+            self.code.run_gencode_remote(cdist_object)
+                self.local.run_script(script, env=env, return_output=True)
+
+
+    - message support in ...
+        - initialmanifest - yes
+        - explorer - no
+        - only locally - yes
+        
+    - how to use notification / messaging in cdist
+        - can be used in all local scripts:
+            - initial manifest
+            - type manifest
+            - type gencode-*
+        - order of object exeution is random or as you requested using require=""
+
+    - example use:
+
+__file/gencode-local:
+         if [ "$local_cksum" != "$remote_cksum" ]; then
+            echo "$__remote_copy" "$source" "${__target_host}:${destination}"
+            echo "copy" >> "$__messages_out"
+         fi
+ 
+__nginx/manifest:
+    __file /etc/nginx/sites-enabled/myfile --source "$__type/files/nginx-config"
+
+__nginx/gencode-remote:
+    if grep -q "__file/etc/nginx/sites-enabled/myfile:copy" "$__messages_in"; then
+        echo /etc/init.d/nginx restart
+    fi

docs/man/cdist-reference.text.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org)
+# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -191,6 +191,12 @@ __manifest::
 __global::
     Directory that contains generic output like explorer.
     Available for: initial manifest, type manifest, type gencode, shell
+__messages_in::
+    File to read messages from
+    Available for: initial manifest, type manifest, type gencode
+__messages_out::
+    File to write messages
+    Available for: initial manifest, type manifest, type gencode
 __object::
     Directory that contains the current object.
     Available for: type manifest, type explorer, type gencode

docs/man/man7/cdist-messaging.text

@@ -0,0 +1,72 @@
+cdist-messaging(7)
+==================
+Nico Schottelius <nico-cdist--@--schottelius.org>
+
+NAME
+----
+cdist-messaging - How the initial manifest and types can communication
+
+
+DESCRIPTION
+-----------
+cdist has a simple but powerful way of allowing communication between
+the initial manifest and types as well as types and types.
+
+Whenever execution is passed from cdist to one of the
+scripts described below, cdist generate 2 new temporary files
+and exports the environment variables __messages_in and
+__messages_out to point to them. 
+
+Before handing over the control, the content of the global message 
+file is copied into the file referenced by $__messages_in.
+
+After cdist gained control back, the content of the file referenced
+by $__messages_out is appended to the global message file.
+
+This way overwriting any of the two files by accident does not 
+interfere with other types.
+
+The order of execution is not defined unless you create dependencies 
+between the different objects (see cdist-manifest(7)) and thus you
+can only react reliably on messages by objects that you depend on.
+
+
+AVAILABILITY
+------------
+Messaging is possible between all **local** scripts:
+
+- initial manifest
+- type/manifest
+- type/gencode-local
+- type/gencode-remote
+
+
+EXAMPLES
+--------
+When you want to emit a message use:
+
+--------------------------------------------------------------------------------
+echo "something" >> "$__messages_out"
+--------------------------------------------------------------------------------
+
+When you want to react on a message use:
+
+--------------------------------------------------------------------------------
+if grep -q "^__your_type/object/id:something" "$__messages_in"; then
+    echo "I do something else"
+fi
+--------------------------------------------------------------------------------
+
+
+SEE ALSO
+--------
+- cdist(1)
+- cdist-manifest(7)
+- cdist-reference(7)
+- cdist-type(7)
+
+
+COPYING
+-------
+Copyright \(C) 2013 Nico Schottelius. Free use of this software is
+granted under the terms of the GNU General Public License version 3 (GPLv3).