Sync with master. Add changelog.

cdist/conf/explorer/init

@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+# 2016 Daniel Heule (hda at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Returns the process name of pid 1 ( normaly the init system )
+# for example at linux this value is "init" or "systemd" in most cases
+#
+
+uname_s="$(uname -s)"
+
+case "$uname_s" in
+    Linux|FreeBSD)
+        ps -o comm= -p 1 || true
+    ;;
+    *)
+        # return a empty string as unknown value
+        echo ""
+    ;;
+esac

cdist/conf/type/__apt_ppa/manifest

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011-2016 Steven Armstrong (steven-cdist at armstrong.cc)
 #
 # This file is part of cdist.
 #
@@ -21,9 +21,8 @@
 name="$__object_id"
 
 __package software-properties-common
-__package python-software-properties
 
-require="__package/software-properties-common __package/python-software-properties" \
+require="__package/software-properties-common" \
    __file /usr/local/bin/remove-apt-repository \
    --source "$__type/files/remove-apt-repository" \
    --mode 0755

cdist/conf/type/__consul/files/versions/0.4.1/source

@@ -1 +1 @@
-https://releases.hashicorp.com/consul/0.4.1/consul_0.4.1_linux_amd64.zip
+https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip

cdist/conf/type/__consul/files/versions/0.5.0/source

@@ -1 +1 @@
-https://releases.hashicorp.com/consul/0.5.0/consul_0.5.0_linux_amd64.zip
+https://dl.bintray.com/mitchellh/consul/0.5.0_linux_amd64.zip

cdist/conf/type/__consul/files/versions/0.5.1/source

@@ -1 +1 @@
-https://releases.hashicorp.com/consul/0.5.1/consul_0.5.1_linux_amd64.zip
+https://dl.bintray.com/mitchellh/consul/0.5.1_linux_amd64.zip

cdist/conf/type/__consul/files/versions/0.6.4/cksum

@@ -0,0 +1 @@
+3832641574 23002736 consul

cdist/conf/type/__consul/files/versions/0.6.4/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/0.6.4/consul_0.6.4_linux_amd64.zip

cdist/conf/type/__consul/parameter/default/version

@@ -1 +1 @@
-0.5.0
+0.6.4

cdist/conf/type/__firewalld_rule/explorer/rule

@@ -1,3 +1,4 @@
+#!/bin/sh
 #
 # 2015 Nico Schottelius (nico-cdist at schottelius.org)
 #

cdist/conf/type/__firewalld_rule/gencode-remote

@@ -1,3 +1,4 @@
+#!/bin/sh
 #
 # 2015 Nico Schottelius (nico-cdist at schottelius.org)
 #

cdist/conf/type/__iptables_apply/gencode-remote

@@ -1,3 +1,5 @@
+#!/bin/sh
+
 if grep -q "^__file/etc/iptables.d/" "$__messages_in"; then
     echo /etc/init.d/iptables restart
 fi

cdist/conf/type/__iptables_apply/manifest

@@ -1,3 +1,4 @@
+#!/bin/sh
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
 #

cdist/conf/type/__iptables_rule/manifest

@@ -1,3 +1,4 @@
+#!/bin/sh
 #
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
 #

cdist/conf/type/__key_value/files/remote_script.sh

@@ -1,3 +1,5 @@
+#!/bin/sh
+
 export key="$(cat "$__object/parameter/key" 2>/dev/null \
    || echo "$__object_id")"
 export state="$(cat "$__object/parameter/state")"

cdist/conf/type/__motd/gencode-remote

@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 # 2013 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.

cdist/conf/type/__package_zypper/explorer/pkg_version

@@ -42,7 +42,7 @@ case "$ptype" in
         zypper search --match-exact --installed-only --type "$ptype" "$name" | grep -E '^i' | cut -d " " -f 3 || true
     ;;
     *)
-        echo "unknown ptype in __package_zypper explorer" &>2
+        echo "unknown ptype in __package_zypper explorer" >&2
         exit 1
     ;;
 esac

cdist/conf/type/__pacman_conf/manifest

@@ -42,11 +42,10 @@ contains_element() {
     MATCH=0
 
     target=$1
-    keys="${@:2}"
+    shift
 
-
+    for key in "$@"; do
-    for key in ${keys}; do
+        if [ "${key}" = "${target}" ]; then
-        if [ "${key}" == "${target}" ]; then
             MATCH=1
             return 0
         fi  
@@ -58,14 +57,14 @@ if [ "${file}" ]; then
     __file "${sec_path}/plain_file_${file}"\
         --state exists --mode 666
 
-    if [ "${state}" == "present" ]; then
+    if [ "${state}" = "present" ]; then
 
         require="__file/${sec_path}/plain_file_${file}" __key_value ${file}_${key}\
             --file ${sec_path}/plain_file_${file} --key ${key} --value ${value} --delimiter ' = '
 
         exit 0
 
-    elif [ "${state}" == "absent" ]; then
+    elif [ "${state}" = "absent" ]; then
         require="__file/${sec_path}/plain_file_${file}" __key_value ${file}_${key}\
                 --state absent
         exit 0
@@ -76,7 +75,7 @@ if [ "${file}" ]; then
     fi
 fi  
 
-if [ "${section}" == "options" ]; then
+if [ "${section}" = "options" ]; then
 
     __file "${sec_path}/${section}"\
             --state exists --mode 666 --source - << eof
@@ -87,10 +86,10 @@ eof
     contains_element "${key}" "${boolean_option_keys}"
 
     if [ "${MATCH}" -eq 1 ]; then
-        if [ "${value}" == "on" ]; then
+        if [ "${value}" = "on" ]; then
             require="__file/${sec_path}/${section}" __line ${key}_${value}\
                 --file ${sec_path}/${section} --line ${key}
-        elif [ "${value}" == "off" ]; then
+        elif [ "${value}" = "off" ]; then
             require="__file/${sec_path}/${section}" __line ${key}_${value}\
                 --file ${sec_path}/${section} --line ${key} --state absent
         fi
@@ -111,7 +110,7 @@ else
             --state exists --mode 666 --source - << eof
 [${section}]
 eof
-    if [ "${state}" == "present" ]; then
+    if [ "${state}" = "present" ]; then
 
         #check if key is valid
         contains_element "${key}" "${allowed_repo_keys}"
@@ -122,7 +121,7 @@ eof
         require="__file/${sec_path}/repo_${section}" __key_value ${section}_${key}\
                 --file ${sec_path}/repo_${section} --key ${key} --value ${value} --delimiter ' = '
 
-    elif [ "${state}" == "absent" ]; then
+    elif [ "${state}" = "absent" ]; then
 
         require="__file/${sec_path}/repo_${section}" __key_value ${section}_${key}\
             --state absent

cdist/conf/type/__pacman_conf_integrate/manifest

@@ -22,7 +22,7 @@ state=$(cat $__object/parameter/state 2>/dev/null)
 
 path="/etc/"
 
-if [ "${state}" == "present" ]; then
+if [ "${state}" = "present" ]; then
     __file /etc/pacman.conf\
         --owner root --group root --mode 644 --source $__type/files/pacman.conf.cdist
 
@@ -35,7 +35,7 @@ if [ "${state}" == "present" ]; then
     __file /etc/pacman.d/plain_file_empty_placeholder\
         --owner root --group root --mode 644
 
-elif [ "${state}" == "absent" ]; then
+elif [ "${state}" = "absent" ]; then
 
     __file /etc/pacman.conf\
         --owner root --group root --mode 644 --source $__type/files/pacman.conf.pacman

cdist/conf/type/__qemu_img/gencode-remote

@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 ################################################################################
 # State: absent is handled by manifest - we need only to do stuff if image is
 # not existing and state != absent

cdist/conf/type/__qemu_img/manifest

@@ -1,3 +1,5 @@
+#!/bin/sh
+#
 ################################################################################
 # Default settings
 #

cdist/conf/type/__start_on_boot/explorer/state

@@ -24,47 +24,51 @@
 
 os=$("$__explorer/os")
 runlevel=$("$__explorer/runlevel")
+init=$("$__explorer/init")
 target_runlevel="$(cat "$__object/parameter/target_runlevel")"
 name="$__object_id"
 
-case "$os" in
+if [ "$init" = 'systemd' ]; then
-    archlinux)
+    # this handles ALL linux distros with systemd
-        state=$(systemctl is-enabled "$name" >/dev/null 2>&1 \
+    # e.g. archlinux, gentoo, new RHEL and SLES versions
-            && echo present \
+    state=$(systemctl is-enabled "$name" >/dev/null 2>&1 \
-            || echo absent)
+        && echo present \
-    ;;
+        || echo absent)
 
-    debian|openwrt)
+else
-        state="present"
+    case "$os" in
-        [ -f "/etc/rc$runlevel.d/S"??"$name" ] || state="absent"
+        debian|openwrt)
-    ;;
+            state="present"
-    ubuntu)
+            [ -f "/etc/rc$runlevel.d/S"??"$name" ] || state="absent"
-        state="absent"
+        ;;
-        [ -f "/etc/rc$runlevel.d/S"??"$name" ] && state="present"
+        ubuntu)
-        [ -f "/etc/init/${name}.conf" ] && state="present"
+            state="absent"
-    ;;
+            [ -f "/etc/rc$runlevel.d/S"??"$name" ] && state="present"
+            [ -f "/etc/init/${name}.conf" ] && state="present"
+        ;;
 
-    amazon|centos|fedora|owl|redhat)
+        amazon|centos|fedora|owl|redhat)
-        state=$(chkconfig --level "$runlevel" "$name" || echo absent)
+            state=$(chkconfig --level "$runlevel" "$name" || echo absent)
-        [ "$state" ] || state="present"
+            [ "$state" ] || state="present"
-    ;;
+        ;;
-    suse)
+        suse)
-        # check for target if set, usable for boot. services in runlevel B
+            # check for target if set, usable for boot. services in runlevel B
-        if [ "$target_runlevel" != 'default' ]; then
+            if [ "$target_runlevel" != 'default' ]; then
-            runlevel="$target_runlevel"
+                runlevel="$target_runlevel"
-        fi
+            fi
-        # suses chkconfig has the same name, but works different ...
+            # suses chkconfig has the same name, but works different ...
-        state=$(chkconfig --check "$name" "$runlevel" || echo absent)
+            state=$(chkconfig --check "$name" "$runlevel" || echo absent)
-        [ "$state" ] || state="present"
+            [ "$state" ] || state="present"
-    ;;
+        ;;
-    gentoo)
+        gentoo)
-        state="present"
+            state="present"
-        [ -f "/etc/runlevels/${target_runlevel}/${name}" ] || state="absent"
+            [ -f "/etc/runlevels/${target_runlevel}/${name}" ] || state="absent"
-    ;;
+        ;;
-    *)
+        *)
-       echo "Unsupported os: $os" >&2
+           echo "Unsupported os: $os" >&2
-       exit 1
+           exit 1
-    ;;
+        ;;
-esac
+    esac
+fi
 
 echo $state

cdist/conf/type/__start_on_boot/gencode-remote

@@ -1,7 +1,7 @@
 #!/bin/sh
 #
 # 2012-2013 Nico Schottelius (nico-cdist at schottelius.org)
-# 2013 Daniel Heule (hda at sfs.biz)
+# 2016 Daniel Heule (hda at sfs.biz)
 #
 # This file is part of cdist.
 #
@@ -22,6 +22,7 @@
 
 state_should="$(cat "$__object/parameter/state")"
 state_is=$(cat "$__object/explorer/state")
+init=$(cat "$__global/explorer/init")
 target_runlevel="$(cat "$__object/parameter/target_runlevel")"
 
 # Short circuit if nothing is to be done
@@ -33,78 +34,86 @@ name="$__object_id"
 
 case "$state_should" in
     present)
-        case "$os" in
+        if [ "$init" = 'systemd' ]; then
-            archlinux)
+            # this handles ALL linux distros with systemd
-                echo "systemctl enable \"$name\""
+            # e.g. archlinux, gentoo in some cases, new RHEL and SLES versions
-            ;;
+            echo "systemctl -q enable \"$name\""
-            debian)
+        else
-                case "$os_version" in
+            case "$os" in
-                    [1-7]*)
+                debian)
-                        echo "update-rc.d \"$name\" defaults >/dev/null"
+                    case "$os_version" in
-                    ;;
+                        [1-7]*)
-                    8*)
+                            echo "update-rc.d \"$name\" defaults >/dev/null"
-                        echo "systemctl enable \"$name\""
+                        ;;
-                    ;;
+                        8*)
-                    *)
+                            echo "systemctl enable \"$name\""
-                        echo "Unsupported version $os_version of $os" >&2
+                        ;;
-                        exit 1
+                        *)
-                    ;;
+                            echo "Unsupported version $os_version of $os" >&2
-                esac
+                            exit 1
-            ;;
+                        ;;
+                    esac
+                ;;
 
-            gentoo)
+                gentoo)
-                echo rc-update add \"$name\" \"$target_runlevel\"
+                    echo rc-update add \"$name\" \"$target_runlevel\"
-            ;;
+                ;;
 
-            amazon|centos|fedora|owl|redhat|suse)
+                amazon|centos|fedora|owl|redhat|suse)
-                echo chkconfig \"$name\" on
+                    echo chkconfig \"$name\" on
-            ;;
+                            echo "Unsupported version $os_version of $os" >&2
+                            exit 1
+                ;;
 
-            openwrt)
+                openwrt)
-                # 'enable' can be successful and still return a non-zero exit
+                    # 'enable' can be successful and still return a non-zero exit
-                # code, deal with it by checking for success ourselves in that
+                    # code, deal with it by checking for success ourselves in that
-                # case (the || ... part).
+                    # case (the || ... part).
-                echo "/etc/init.d/\"$name\" enable || [ -f /etc/rc.d/S??\"$name\" ]"
+                    echo "/etc/init.d/\"$name\" enable || [ -f /etc/rc.d/S??\"$name\" ]"
-            ;;
+                ;;
 
-            ubuntu)
+                ubuntu)
-                echo "update-rc.d \"$name\" defaults >/dev/null"
+                    echo "update-rc.d \"$name\" defaults >/dev/null"
-            ;;
+                ;;
 
-            *)
+                *)
-               echo "Unsupported os: $os" >&2
+                   echo "Unsupported os: $os" >&2
-               exit 1
+                   exit 1
-            ;;
+                ;;
-        esac
+            esac
+        fi
     ;;
 
     absent)
-        case "$os" in
+        if [ "$init" = 'systemd' ]; then
-            archlinux)
+            # this handles ALL linux distros with systemd
-                echo "systemctl disable \"$name\""
+            # e.g. archlinux, gentoo in some cases, new RHEL and SLES versions
-            ;;
+            echo "systemctl -q disable \"$name\""
-            debian|ubuntu)
-                echo update-rc.d -f \"$name\" remove
-            ;;
 
-            gentoo)
+        else
-                echo rc-update del \"$name\"  \"$target_runlevel\"
+            case "$os" in
-            ;;
+                debian|ubuntu)
+                    echo update-rc.d -f \"$name\" remove
+                ;;
 
-            centos|fedora|owl|redhat|suse)
+                gentoo)
-                echo chkconfig \"$name\" off
+                    echo rc-update del \"$name\"  \"$target_runlevel\"
-            ;;
+                ;;
 
-            openwrt)
+                centos|fedora|owl|redhat|suse)
-                echo "\"/etc/init.d/$name\" disable"
+                    echo chkconfig \"$name\" off
-            ;;
+                ;;
 
-            *)
+                openwrt)
-               echo "Unsupported os: $os" >&2
+                    echo "\"/etc/init.d/$name\" disable"
-               exit 1
+                ;;
-            ;;
+
-        esac
+                *)
- 
+                   echo "Unsupported os: $os" >&2
+                   exit 1
+                ;;
+            esac
+        fi
     ;;
 
     *)

cdist/conf/type/__zypper_service/explorer/service_uri

@@ -25,6 +25,4 @@ if [ -f "$__object/parameter/uri" ]; then
 else
    uri="/$__object_id"
 fi
-# simpler command which works only on SLES11 SP3 or newer:
+echo $(zypper ls -u | awk 'BEGIN { FS = "[ ]+\\|[ ]+" } ; $4 == "Yes" && $NF == "'$uri'" {print $NF}')
-# echo $(zypper ls -u -E | grep -E "\<$uri\>" | cut -d'|' -f 7)
-echo $(zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | grep -E "\<$uri\>" | cut -d'|' -f 7 )

cdist/exec/local.py

@@ -195,7 +195,7 @@ class Local(object):
         except subprocess.CalledProcessError:
             raise cdist.Error("Command failed: " + " ".join(command))
         except OSError as error:
-            raise cdist.Error(" ".join(*args) + ": " + error.args[1])
+            raise cdist.Error(" ".join(command) + ": " + error.args[1])
         finally:
             if message_prefix:
                 message.merge_messages()

cdist/exec/remote.py

@@ -130,12 +130,29 @@ class Remote(object):
         # FIXME: replace this by -o SendEnv name -o SendEnv name ... to ssh?
         # can't pass environment to remote side, so prepend command with
         # variable declarations
+
+        # cdist command prepended with variable assignments expects
+        # posix shell (bourne, bash) at the remote as user default shell.
+        # If remote user shell isn't poxis shell, but for e.g. csh/tcsh
+        # then these var assignments are not var assignments for this
+        # remote shell, it tries to execute it as a command and fails.
+        # So really do this by default:
+        # /bin/sh -c 'export <var assignments>; command'
+        # so that constructed remote command isn't dependent on remote
+        # shell. Do this only if env is not None. env breaks this.
+        # Explicitly use /bin/sh, because var assignments assume poxis
+        # shell already.
+        # This leaves the posibility to write script that needs to be run
+        # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g.
+        # /bin/csh will execute this script in the right way.
         if env:
-            remote_env = ["%s=%s" % item for item in env.items()]
+            cmd.append("/bin/sh")
-            cmd.extend(remote_env)
+            cmd.append("-c")
-
+            remote_env = [" export %s=%s;" % item for item in env.items()]
-        cmd.extend(command)
+            string_cmd = " ".join(remote_env) + " ".join(command)
-
+            cmd.append(string_cmd)
+        else:
+            cmd.extend(command)
         return self._run_command(cmd, env=env, return_output=return_output)
 
     def _run_command(self, command, env=None, return_output=False):

cdist/test/exec/remote.py

@@ -39,7 +39,7 @@ class RemoteTestCase(test.CdistTestCase):
         user = getpass.getuser()
         remote_exec = "ssh -o User=%s -q" % user
         remote_copy = "scp -o User=%s -q" % user
-        self.remote = remote.Remote(self.target_host, self.base_path, remote_exec, remote_copy)
+        self.remote = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy)
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
@@ -125,7 +125,7 @@ class RemoteTestCase(test.CdistTestCase):
         os.chmod(remote_exec_path, 0o755)
         remote_exec = remote_exec_path
         remote_copy = "echo"
-        r = remote.Remote(self.target_host, self.base_path, remote_exec, remote_copy)
+        r = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy)
         self.assertEqual(r.run('/bin/true', return_output=True), "%s\n" % self.target_host)
 
     def test_run_script_target_host_in_env(self):
@@ -135,8 +135,33 @@ class RemoteTestCase(test.CdistTestCase):
         os.chmod(remote_exec_path, 0o755)
         remote_exec = remote_exec_path
         remote_copy = "echo"
-        r = remote.Remote(self.target_host, self.base_path, remote_exec, remote_copy)
+        r = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy)
         handle, script = self.mkstemp(dir=self.temp_dir)
         with os.fdopen(handle, "w") as fd:
             fd.writelines(["#!/bin/sh\n", "/bin/true"])
         self.assertEqual(r.run_script(script, return_output=True), "%s\n" % self.target_host)
+
+    def test_run_script_with_env_target_host_in_env(self):
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.writelines(["#!/bin/sh\n", 'if [ "$__object" ]; then echo $__object; else echo no_env; fi\n'])
+        os.chmod(script, 0o755)
+        handle, remote_exec_path = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, 'w') as fd:
+            fd.writelines(["#!/bin/sh\n", 'shift; cmd=$1; shift; $cmd "$@"\n'])
+        os.chmod(remote_exec_path, 0o755)
+        remote_exec = remote_exec_path
+        remote_copy = "echo"
+        r = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy)
+        output = r.run_script(script, return_output=True)
+        self.assertEqual(output, "no_env\n")
+        env = {
+            '__object': 'test_object',
+        }
+        output = r.run_script(script, env=env, return_output=True)
+        self.assertEqual(output, "test_object\n")
+
+if __name__ == '__main__':
+    import unittest
+
+    unittest.main()

docs/changelog

@@ -2,6 +2,10 @@ Changelog
 ---------
 
 next:
+	* Core: Add CDIST_REMOTE_COPY/EXEC env variables and multiplexing options for default scp/ssh (Darko Poljak)
+	* Types: Remove bashisms in scripts (Darko Poljak)
+	* Core: Fix bug in remote command with environment (Darko Poljak)
+	* Core: Fix bug in local code execution (Darko Poljak)
 	* Documentation: Fix spelling in manual pages (Dmitry Bogatov)
 	* New type: __pacman_conf: Manage pacman.conf (Dominique Roux)
 	* New type: __pacman_conf_integrate: cdist compatible pacman.conf (Dominique Roux)
@@ -16,6 +20,7 @@ next:
 	* Type __file: Add NetBSD support (Jonathan A. Kollasch)
 	* Type __group: Add NetBSD support (Jonathan A. Kollasch)
 	* Type __consul: Add new consul versions (Nico Schottelius)
+	* Type __apt_ppa: Do not install legacy package python-software-properties (Steven Armstrong)
 
 
 3.1.13: 2015-05-16