Log changes related to transparent shebang support which is now available for all contexts : all scripts (including generated code) destined for local or remote hosts

bin/cdist-build-helper

@@ -534,8 +534,7 @@ eof
     ;;
 
     version)
-        target_version="$(git describe | sed 's/-/.dev/; s/-/+/g')"
+        printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py
-        printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py
     ;;
 
     target-version)

cdist/conf/type/__apt_pin/man.rst

@@ -23,7 +23,7 @@ package
    Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used.
 
 priority
-   The priority value to assign to matching packages. Defaults to 500. (To match the default target distro's priority)
+   The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority)
 
 state
    Will be passed to underlying `__file` type; see there for valid values and defaults.

cdist/conf/type/__package_luarocks/manifest

@@ -19,5 +19,5 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-__package luarocks --state present
+__package luarocks --state installed
-__package make --state present
+__package make --state installed

cdist/conf/type/__timezone/gencode-remote

@@ -34,12 +34,3 @@ case "$os" in
       echo "echo \"$timezone_should\" > /etc/timezone"
    ;;
 esac
-
-case "$os" in
-    openwrt)
-        cat <<EOF
-			uci set system.@system[0].timezone="$timezone_should"
-			uci commit
-EOF
-   ;;
-esac

cdist/conf/type/__timezone/manifest

@@ -53,10 +53,7 @@ case "$os" in
                 --file /etc/sysconfig/clock \
                 --delimiter '=' \
                 --value "\"$timezone\""
-        ;;
+    ;;
-    openwrt)
-        : # Uses gencode-remote
-        ;;
     *)
         echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
         echo "Please contribute an implementation for it if you can." >&2

cdist/conf/type/__user_groups/explorer/group

@@ -20,4 +20,4 @@
 
 user="$(cat "$__object/parameter/user" 2>/dev/null || echo "$__object_id")"
 
-(id -G -n "$user" | tr ' ' '\n') 2>/dev/null || true
+(id -G -n "$user" | tr ' ' '\n' | sort) 2>/dev/null || true

cdist/conf/type/__user_groups/gencode-remote

@@ -26,15 +26,13 @@ os=$(cat "$__global/explorer/os")
 mkdir "$__object/files"
 # file has to be sorted for comparison with `comm`
 sort "$__object/parameter/group" > "$__object/files/group.sorted"
-# Use local sort for remote groups
-sort "$__object/explorer/group" > "$__object/files/group-remote.sorted"
 
 case "$state_should" in
    present)
-      changed_groups="$(comm -13 "$__object/files/group-remote.sorted" "$__object/files/group.sorted")"
+      changed_groups="$(comm -13 "$__object/explorer/group" "$__object/files/group.sorted")"
    ;;
    absent)
-      changed_groups="$(comm -12 "$__object/files/group-remote.sorted" "$__object/files/group.sorted")"
+      changed_groups="$(comm -12 "$__object/explorer/group" "$__object/files/group.sorted")"
    ;;
 esac
 

cdist/core/cdist_object.py

@@ -239,9 +239,9 @@ class CdistObject:
             lambda obj: os.path.join(obj.absolute_path, "state"))
     source = fsproperty.FileListProperty(
             lambda obj: os.path.join(obj.absolute_path, "source"))
-    code_local = fsproperty.FileStringProperty(
+    code_local = fsproperty.FileScriptProperty(
             lambda obj: os.path.join(obj.base_path, obj.code_local_path))
-    code_remote = fsproperty.FileStringProperty(
+    code_remote = fsproperty.FileScriptProperty(
             lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
     typeorder = fsproperty.FileListProperty(
             lambda obj: os.path.join(obj.absolute_path, 'typeorder'))

cdist/exec/remote.py

@@ -216,18 +216,46 @@ class Remote:
             _wrap_addr(self.target_host[0]), destination)])
         self._run_command(command)
 
+    def check_if_executable(self, path):
+        """Check if the given remote resource is "executable",
+        (i.e. has its execute bit set).
+        Return True/False
+
+        """
+        self.log.trace("Remote check if executable : %s", path)
+
+        # XXX: Too bad we can't just check the returned status.
+        # Hence the dance with "echo TRUE/FALSE"
+        chk = " ".join([
+                'if [ -f "{path}" ] && [ -x "{path}" ] ',
+                '; then echo TRUE ; else echo FALSE ; fi']).format(path=path)
+
+        out = self.run(['/bin/sh', '-c', "'" + chk + "'"],
+                    env=None, return_output=True) or ""
+        return out.strip() == 'TRUE'
+
     def run_script(self, script, env=None, return_output=False, stdout=None,
                    stderr=None):
         """Run the given script with the given environment on the remote side.
         Return the output as a string.
 
         """
+        command = [script]
 
-        command = [
+        if self.check_if_executable(script):
-            self.configuration.get('remote_shell', "/bin/sh"),
+            # Allow transparent shebang support for "executable" scripts
-            "-e"
+            self.log.debug(
-        ]
+                '%-70s : Remote script is executable, '
-        command.append(script)
+                        + 'running it directly', script)
+        else:
+            # FIXME: Who knows what "-e" means for an arbitrary remote_shell ?
+            # Keeping the old behavior (of always adding "-e") for the moment.
+            shell = [self.configuration.get('remote_shell', "/bin/sh"), "-e"]
+
+            command = shell + command
+            self.log.debug(
+                '%-70s : Remote script is NOT executable, '
+                        + 'running it with %s', script, " ".join(shell))
 
         return self.run(command, env=env, return_output=return_output,
                         stdout=stdout, stderr=stderr)

cdist/test/code/__init__.py

@@ -24,6 +24,7 @@ import getpass
 import os
 import shutil
 import logging
+import re
 
 import cdist
 from cdist import core
@@ -37,123 +38,191 @@ my_dir = op.abspath(op.dirname(__file__))
 fixtures = op.join(my_dir, 'fixtures')
 conf_dir = op.join(fixtures, 'conf')
 
+class Burried:
+    '''This a bogus -alas needed- wrapper class whose sole
+    purpose is to hide the inner classes from `unittest`
+    that would otherwise try to run them, even though
+    they have now become abstractish base classes.
+    '''
 
-class CodeTestCase(test.CdistTestCase):
+    class CodeTestCase(test.CdistTestCase):
+        '''
+        This is now a base class (which should not be invoked by unittest).
 
+        A couple tests had to be refactored, without any behavioral changes,
+        so as to suit POLYGLOT testing.
+        '''
+
+        def setUp(self):
+            self.setup_for_type('__dump_environment')
+
+        def setup_for_type(self, tested_type_name):
+            self.local_dir = self.mkdtemp()
+            self.hostdir = cdist.str_hash(self.target_host[0])
+            self.host_base_path = os.path.join(self.local_dir, self.hostdir)
+
+            self.local = local.Local(
+                target_host=self.target_host,
+                target_host_tags=self.target_host_tags,
+                base_root_path=self.host_base_path,
+                host_dir_name=self.hostdir,
+                exec_path=cdist.test.cdist_exec_path,
+                add_conf_dirs=[conf_dir])
+            self.local.create_files_dirs()
+
+            self.remote_dir = self.mkdtemp()
+            remote_exec = self.remote_exec
+            remote_copy = self.remote_copy
+            self.remote = remote.Remote(
+                target_host=self.target_host,
+                remote_exec=remote_exec,
+                remote_copy=remote_copy,
+                base_path=self.remote_dir,
+                stdout_base_path=self.local.stdout_base_path,
+                stderr_base_path=self.local.stderr_base_path)
+            self.remote.create_files_dirs()
+
+            self.code = code.Code(self.target_host, self.local, self.remote)
+
+            self.cdist_type = core.CdistType(self.local.type_path,
+                                            tested_type_name)
+            self.cdist_object = core.CdistObject(
+                    self.cdist_type, self.local.object_path, 'whatever',
+                    self.local.object_marker_name)
+            self.cdist_object.create()
+
+        def tearDown(self):
+            shutil.rmtree(self.local_dir)
+            shutil.rmtree(self.remote_dir)
+
+        def _expected_environment(self):
+            expected = {
+                '__target_host' : self.local.target_host[0],
+                '__target_hostname' :  self.local.target_host[1],
+                '__target_fqdn' : self.local.target_host[2],
+                '__global' : self.local.base_path,
+                '__type' : self.cdist_type.absolute_path,
+                '__object' : self.cdist_object.absolute_path,
+                '__object_id' : self.cdist_object.object_id,
+                '__object_name' : self.cdist_object.name,
+                '__files' : self.local.files_path,
+                '__target_host_tags' : self.local.target_host_tags,
+                '__cdist_log_level': str(logging.WARNING),
+                '__cdist_log_level_name' : 'WARNING'
+            }
+            return expected
+
+        # Keeping around older simplex parsing logic for a while
+        def _parse_env_from_generated_code(self, script_text):
+            output_dict = {}
+            for line in script_text.split('\n'):
+                if line:
+                    junk, value = line.split(': ')
+                    key = junk.split(' ')[1]
+                    output_dict[key] = value
+            return output_dict
+
+        def test_run_gencode_local_environment(self):
+            script_text = self.code.run_gencode_local(self.cdist_object)
+
+            self.assertDictContainsSubset(
+                self._expected_environment(),
+                self._parse_env_from_generated_code(script_text)
+            )
+
+        def test_run_gencode_remote_environment(self):
+            script_text = self.code.run_gencode_remote(self.cdist_object)
+
+            self.assertDictContainsSubset(
+                self._expected_environment(),
+                self._parse_env_from_generated_code(script_text)
+            )
+
+        def test_transfer_code_remote(self):
+            self.cdist_object.code_remote = self.code.run_gencode_remote(
+                    self.cdist_object)
+            self.code.transfer_code_remote(self.cdist_object)
+            destination = os.path.join(self.remote.object_path,
+                                    self.cdist_object.code_remote_path)
+            self.assertTrue(os.path.isfile(destination))
+
+        def test_run_code_local(self):
+            self.cdist_object.code_local = self.code.run_gencode_local(
+                    self.cdist_object)
+            self.code.run_code_local(self.cdist_object)
+
+        def test_run_code_remote_environment(self):
+            self.cdist_object.code_remote = self.code.run_gencode_remote(
+                    self.cdist_object)
+            self.code.transfer_code_remote(self.cdist_object)
+            self.code.run_code_remote(self.cdist_object)
+
+    class CodeTestCasePolyglot(CodeTestCase):
+        '''Base class for testing generetaed polyglot code
+
+           The assumption here is that, if cdist works for Perl code,
+           it should work for any script language, as long as the
+           corresponding interpretor (given on the shebang line)
+           is available.
+        '''
+        def _parse_env_from_generated_code(self, script_text):
+            '''
+            New parsing logic allows for some simple generated perlish code
+            (as well as basic shell code).
+
+            This implementation can actually be moved up the parent class
+            as a drop in replacement of the older parser.
+
+            NOTE that this kind of parsing is necesarily fragile and sloppy.
+            It has been factored out and can easily be overriden, if needed.
+            '''
+            output_dict = {}
+            for line in script_text.split('\n'):
+                line = line.strip()
+
+                if not line:
+                    continue    # Ignore blank lines
+
+                if  bool(re.match('^#', line)):
+                    continue    # Ignore lines consisting of only comments
+
+                if not bool(re.search(': ', line)):
+                    continue      # Ignore lines that do not contain our splitter pattern.
+
+                junk, value = line.split(': ')
+                key = junk.split(' ')[1]
+
+                # Remove leading quotes (single or double)
+                key = re.sub('^\s*["\']+\s*', '', key)
+
+                # Remove trailing quotes
+                # and any eventual whitespace escape sequences just before
+                # XXX: Why do we need to double escape the backslash (4 in all)
+                value = re.sub('\s*([\\\\]+[trn])*\s*["\']*\s*[;]*\s*$', '', value)
+
+                output_dict[key] = value
+
+            return output_dict
+
+# Actual Test Case classes (visible to unittest)
+class CodeTestCase(Burried.CodeTestCase):
+    ''' Older tests that cover the monoglot scenario (shell generates shell)
+        gencode-* is typically written for /bin/sh (which, in turn, must generate shell code).
+    '''
     def setUp(self):
-        self.local_dir = self.mkdtemp()
+        self.setup_for_type('__dump_environment')
-        self.hostdir = cdist.str_hash(self.target_host[0])
-        self.host_base_path = os.path.join(self.local_dir, self.hostdir)
 
-        self.local = local.Local(
+class CodeTestCase_polyglot_perl_generates_perl(Burried.CodeTestCasePolyglot):
-            target_host=self.target_host,
+    def setUp(self):
-            target_host_tags=self.target_host_tags,
+        self.setup_for_type('__dump_environment_polyglot_perl_generates_perl')
-            base_root_path=self.host_base_path,
-            host_dir_name=self.hostdir,
-            exec_path=cdist.test.cdist_exec_path,
-            add_conf_dirs=[conf_dir])
-        self.local.create_files_dirs()
 
-        self.remote_dir = self.mkdtemp()
+class CodeTestCase_polyglot_perl_generates_shell(Burried.CodeTestCasePolyglot):
-        remote_exec = self.remote_exec
+    def setUp(self):
-        remote_copy = self.remote_copy
+        self.setup_for_type('__dump_environment_polyglot_perl_generates_shell')
-        self.remote = remote.Remote(
-            target_host=self.target_host,
-            remote_exec=remote_exec,
-            remote_copy=remote_copy,
-            base_path=self.remote_dir,
-            stdout_base_path=self.local.stdout_base_path,
-            stderr_base_path=self.local.stderr_base_path)
-        self.remote.create_files_dirs()
-
-        self.code = code.Code(self.target_host, self.local, self.remote)
-
-        self.cdist_type = core.CdistType(self.local.type_path,
-                                         '__dump_environment')
-        self.cdist_object = core.CdistObject(
-                self.cdist_type, self.local.object_path, 'whatever',
-                self.local.object_marker_name)
-        self.cdist_object.create()
-
-    def tearDown(self):
-        shutil.rmtree(self.local_dir)
-        shutil.rmtree(self.remote_dir)
-
-    def test_run_gencode_local_environment(self):
-        output_string = self.code.run_gencode_local(self.cdist_object)
-        output_dict = {}
-        for line in output_string.split('\n'):
-            if line:
-                junk, value = line.split(': ')
-                key = junk.split(' ')[1]
-                output_dict[key] = value
-        self.assertEqual(output_dict['__target_host'],
-                         self.local.target_host[0])
-        self.assertEqual(output_dict['__target_hostname'],
-                         self.local.target_host[1])
-        self.assertEqual(output_dict['__target_fqdn'],
-                         self.local.target_host[2])
-        self.assertEqual(output_dict['__global'], self.local.base_path)
-        self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
-        self.assertEqual(output_dict['__object'],
-                         self.cdist_object.absolute_path)
-        self.assertEqual(output_dict['__object_id'],
-                         self.cdist_object.object_id)
-        self.assertEqual(output_dict['__object_name'], self.cdist_object.name)
-        self.assertEqual(output_dict['__files'], self.local.files_path)
-        self.assertEqual(output_dict['__target_host_tags'],
-                         self.local.target_host_tags)
-        self.assertEqual(output_dict['__cdist_log_level'],
-                         str(logging.WARNING))
-        self.assertEqual(output_dict['__cdist_log_level_name'], 'WARNING')
-
-    def test_run_gencode_remote_environment(self):
-        output_string = self.code.run_gencode_remote(self.cdist_object)
-        output_dict = {}
-        for line in output_string.split('\n'):
-            if line:
-                junk, value = line.split(': ')
-                key = junk.split(' ')[1]
-                output_dict[key] = value
-        self.assertEqual(output_dict['__target_host'],
-                         self.local.target_host[0])
-        self.assertEqual(output_dict['__target_hostname'],
-                         self.local.target_host[1])
-        self.assertEqual(output_dict['__target_fqdn'],
-                         self.local.target_host[2])
-        self.assertEqual(output_dict['__global'], self.local.base_path)
-        self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
-        self.assertEqual(output_dict['__object'],
-                         self.cdist_object.absolute_path)
-        self.assertEqual(output_dict['__object_id'],
-                         self.cdist_object.object_id)
-        self.assertEqual(output_dict['__object_name'], self.cdist_object.name)
-        self.assertEqual(output_dict['__files'], self.local.files_path)
-        self.assertEqual(output_dict['__target_host_tags'],
-                         self.local.target_host_tags)
-        self.assertEqual(output_dict['__cdist_log_level'],
-                         str(logging.WARNING))
-        self.assertEqual(output_dict['__cdist_log_level_name'], 'WARNING')
-
-    def test_transfer_code_remote(self):
-        self.cdist_object.code_remote = self.code.run_gencode_remote(
-                self.cdist_object)
-        self.code.transfer_code_remote(self.cdist_object)
-        destination = os.path.join(self.remote.object_path,
-                                   self.cdist_object.code_remote_path)
-        self.assertTrue(os.path.isfile(destination))
-
-    def test_run_code_local(self):
-        self.cdist_object.code_local = self.code.run_gencode_local(
-                self.cdist_object)
-        self.code.run_code_local(self.cdist_object)
-
-    def test_run_code_remote_environment(self):
-        self.cdist_object.code_remote = self.code.run_gencode_remote(
-                self.cdist_object)
-        self.code.transfer_code_remote(self.cdist_object)
-        self.code.run_code_remote(self.cdist_object)
 
+class CodeTestCase_polyglot_shell_generates_perl(Burried.CodeTestCasePolyglot):
+    def setUp(self):
+        self.setup_for_type('__dump_environment_polyglot_shell_generates_perl')
 
 if __name__ == '__main__':
     import unittest

cdist/test/code/fixtures/conf/type/__dump_environment_polyglot_perl_generates_perl/gencode-local

@@ -0,0 +1,23 @@
+#!/usr/bin/env perl
+# [POLYGLOT]: A perl script that generates perl code
+
+use strict;
+
+print <<"EOT";
+#!/usr/bin/env perl
+use strict;
+
+print "__target_host: $ENV{__target_host}\n";
+print "__target_hostname: $ENV{__target_hostname}\n";
+print "__target_fqdn: $ENV{__target_fqdn}\n";
+print "__global: $ENV{__global}\n";
+print "__type: $ENV{__type}\n";
+print "__object: $ENV{__object}\n";
+print "__object_id: $ENV{__object_id}\n";
+print "__object_name: $ENV{__object_name}\n";
+print "__files: $ENV{__files}\n";
+print "__target_host_tags: $ENV{__target_host_tags}\n";
+print "__cdist_log_level: $ENV{__cdist_log_level}\n";
+print "__cdist_log_level_name: $ENV{__cdist_log_level_name}\n";
+
+EOT

cdist/test/code/fixtures/conf/type/__dump_environment_polyglot_perl_generates_perl/gencode-remote

@@ -0,0 +1 @@
+gencode-local

cdist/test/code/fixtures/conf/type/__dump_environment_polyglot_perl_generates_shell/gencode-local

@@ -0,0 +1,19 @@
+#!/usr/bin/env perl
+# [POLYGLOT]: A perl script that generates shell code
+
+use strict;
+
+print <<"EOT";
+echo __target_host: $ENV{__target_host}
+echo __target_hostname: $ENV{__target_hostname}
+echo __target_fqdn: $ENV{__target_fqdn}
+echo __global: $ENV{__global}
+echo __type: $ENV{__type}
+echo __object: $ENV{__object}
+echo __object_id: $ENV{__object_id}
+echo __object_name: $ENV{__object_name}
+echo __files: $ENV{__files}
+echo __target_host_tags: $ENV{__target_host_tags}
+echo __cdist_log_level: $ENV{__cdist_log_level}
+echo __cdist_log_level_name: $ENV{__cdist_log_level_name}
+EOT

cdist/test/code/fixtures/conf/type/__dump_environment_polyglot_perl_generates_shell/gencode-remote

@@ -0,0 +1 @@
+gencode-local

cdist/test/code/fixtures/conf/type/__dump_environment_polyglot_shell_generates_perl/gencode-local

@@ -0,0 +1,20 @@
+#!/bin/sh
+# [POLYGLOT]: A shell script that generates perl code
+
+cat <<-EOT
+#!/usr/bin/env perl
+use strict;
+
+print "__target_host: ${__target_host}\n";
+print "__target_hostname: ${__target_hostname}\n";
+print "__target_fqdn: ${__target_fqdn}\n";
+print "__global: ${__global}\n";
+print "__type: ${__type}\n";
+print "__object: ${__object}\n";
+print "__object_id: ${__object_id}\n";
+print "__object_name: ${__object_name}\n";
+print "__files: ${__files}\n";
+print "__target_host_tags: ${__target_host_tags}\n";
+print "__cdist_log_level: ${__cdist_log_level}\n";
+print "__cdist_log_level_name: ${__cdist_log_level_name}\n";
+EOT

cdist/test/code/fixtures/conf/type/__dump_environment_polyglot_shell_generates_perl/gencode-remote

@@ -0,0 +1 @@
+gencode-local

cdist/test/exec/local.py

@@ -29,6 +29,8 @@ import random
 import time
 import datetime
 import argparse
+import stat
+import unittest
 
 import cdist
 import cdist.configuration as cc
@@ -278,6 +280,85 @@ class LocalTestCase(test.CdistTestCase):
         for fmt, expected, actual in cases:
             self.assertEqual(expected, actual)
 
+    #------------------------------------------------------------------------
+    # POLYGLOT tests:
+    #   Ensure cdist is truely language-agnostic
+    #   with proper support of shebang for "executable" scripts
+    #------------------------------------------------------------------------
+    @staticmethod
+    def _mark_as_executable(script):
+        # grant execute permission to everyone
+        os.chmod(script,
+                (os.stat(script).st_mode & 0o777)
+            | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+    def test_polyglot_run_shell_script_with_exec_permisions(self):
+        xc = self.local
+        xc.create_files_dirs()
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.writelines(["#!/bin/sh\n", "[ 1 == 1 ] && echo OK" ])
+
+        self._mark_as_executable(script)
+
+        xc.run_script(script)
+
+    def test_polyglot_run_shell_script_without_exec_permissions(self):
+        xc = self.local
+        xc.create_files_dirs()
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.writelines(["#!/bin/sh\n", "[ 1 == 1 ] && echo OK" ])
+
+        xc.run_script(script)
+
+    def test_polyglot_run_perl_script_with_exec_permissions(self):
+        xc = self.local
+        xc.create_files_dirs()
+        try:
+            xc.run(["/usr/bin/env", "perl", "-v"])
+        except:
+            raise unittest.SkipTest(
+                'perl interpreter or env program is not available.')
+
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.write(
+                """#!/usr/bin/env perl
+                use strict;
+                print 'OK';
+                """)
+
+        self._mark_as_executable(script)
+
+        self.assertEqual(xc.run_script(script, return_output=True), "OK")
+
+    def test_polyglot_run_perl_script_without_exec_permissions_and_fail(self):
+        xc = self.local
+        xc.create_files_dirs()
+        try:
+            xc.run(["/usr/bin/env", "perl", "-v"])
+        except:
+            raise unittest.SkipTest(
+                'perl interpreter or env program is not available.')
+
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.write(
+                """#!/usr/bin/env perl
+                use strict;
+                print 'OK';
+                """)
+
+        # NOTE that we deliberately abstain from setting execute permissions
+        # on the script, so that it ends up being fed into /bin/sh
+        # by the executor, which in turn should cause an error.
+        failed = False
+        try:
+            xc.run_script(script)
+        except:
+            failed = True
+        self.assertTrue(failed)
 
 if __name__ == "__main__":
     import unittest

cdist/test/exec/remote.py

@@ -23,6 +23,8 @@ import os
 import getpass
 import shutil
 import multiprocessing
+import stat
+import unittest
 
 import cdist
 from cdist import test
@@ -220,6 +222,86 @@ class RemoteTestCase(test.CdistTestCase):
         self.assertEqual(output, "test_object\n")
 
 
+    #------------------------------------------------------------------------
+    # POLYGLOT tests:
+    #   Ensure cdist is truely language-agnostic
+    #   with proper support of shebang for "executable" scripts
+    #------------------------------------------------------------------------
+    @staticmethod
+    def _mark_as_executable(script):
+        # grant execute permission to everyone
+        os.chmod(script,
+                (os.stat(script).st_mode & 0o777)
+            | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+    def test_polyglot_run_shell_script_with_exec_permisions(self):
+        xc = self.remote
+        xc.create_files_dirs()
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.writelines(["#!/bin/sh\n", "[ 1 == 1 ] && echo OK" ])
+
+        self._mark_as_executable(script)
+
+        xc.run_script(script)
+
+    def test_polyglot_run_shell_script_without_exec_permissions(self):
+        xc = self.remote
+        xc.create_files_dirs()
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.writelines(["#!/bin/sh\n", "[ 1 == 1 ] && echo OK" ])
+
+        xc.run_script(script)
+
+    def test_polyglot_run_perl_script_with_exec_permissions(self):
+        xc = self.remote
+        xc.create_files_dirs()
+        try:
+            xc.run(["/usr/bin/env", "perl", "-v"])
+        except:
+            raise unittest.SkipTest(
+                'perl interpreter or env program is not available.')
+
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.write(
+                """#!/usr/bin/env perl
+                use strict;
+                print 'OK';
+                """)
+
+        self._mark_as_executable(script)
+
+        self.assertEqual(xc.run_script(script, return_output=True), "OK")
+
+    def test_polyglot_run_perl_script_without_exec_permissions_and_fail(self):
+        xc = self.remote
+        xc.create_files_dirs()
+        try:
+            xc.run(["/usr/bin/env", "perl", "-v"])
+        except:
+            raise unittest.SkipTest(
+                'perl interpreter or env program is not available.')
+
+        handle, script = self.mkstemp(dir=self.temp_dir)
+        with os.fdopen(handle, "w") as fd:
+            fd.write(
+                """#!/usr/bin/env perl
+                use strict;
+                print 'OK';
+                """)
+
+        # NOTE that we deliberately abstain from setting execute permissions
+        # on the script, so that it ends up being fed into /bin/sh
+        # by the executor, which in turn should cause an error.
+        failed = False
+        try:
+            xc.run_script(script)
+        except:
+            failed = True
+        self.assertTrue(failed)
+
 if __name__ == '__main__':
     import unittest
 

cdist/test/explorer/fixtures/conf/explorer/polyglot_perl

@@ -0,0 +1,4 @@
+#!/usr/bin/env perl
+use strict;
+
+print "Polyglot - perl\n"

cdist/util/filesystem.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+#
+# 2023 Tabulon (dev-cdist at tabulon.net)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import stat
+
+def ensure_file_is_executable_by_all(path):
+    """Ensure (and if needed, add) execute permissions
+       for everyone (user, group, others) on the given file
+       Similar to : chmod a+x <path>
+    """
+    ensure_file_permissions(path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+def ensure_file_permissions(path, permissions):
+    """Ensure (and if needed, add) the given permissions
+       for the given filesystem object.
+       Similar to using '+' with chmod
+    """
+    perm = os.stat(path).st_mode & 0o777 # only the last 3 bits relate to permissions
+
+    # If desired permissions were already set, don't meddle.
+    if ( (perm & permissions ) != permissions ):
+        os.chmod(path, perm | permissions)
+
+    # return a mask of desired permissions that are/were actually set
+    return os.stat(path).st_mode & 0o777 & permissions
+
+def file_has_shebang(path):
+    """Does the given file start with a shebang ?
+    """
+    return read_from_file(path, size=2) == '#!'
+
+def read_from_file(path, size=-1, ignore=None):
+    """Read and return a number of bytes from the given file.
+       If size is '-1' (the default) the entire contents are returned.
+    """
+    value = ""
+    try:
+        with open(path, "r") as fd:
+            value = fd.read(size)
+    except ignore:
+        pass
+    finally:
+        fd.close()
+    return value
+
+def slurp_file(path, ignore=None):
+    """Read and return the entire contents of a given file
+    """
+    return read_from_file(path, size=-1, ignore=ignore)

cdist/util/fsproperty.py

@@ -23,7 +23,7 @@ import os
 import collections
 
 import cdist
-
+import cdist.util.filesystem as fs
 
 class AbsolutePathRequiredError(cdist.Error):
     def __init__(self, path):
@@ -319,3 +319,26 @@ class FileStringProperty(FileBasedProperty):
                 os.remove(path)
             except EnvironmentError:
                 pass
+
+class FileScriptProperty(FileStringProperty):
+    """A property specially tailored for script text,
+       which stores its value in a file.
+    """
+    # Descriptor Protocol
+    def __set__(self, instance, value):
+        super().__set__(instance, value)
+
+        # -------------------------------------------------------------------
+        # NOTE [tabulon@2023-03-31]: If the file starts with a shebang (#!),
+        # mark it as an executable (chmod a+x), so that exec.(local|remote)
+        # can decide to invoke it directly (instead of feeding it to /bin/sh)
+        # -------------------------------------------------------------------
+        # NOTE that this enables cdist to become completely language-agnostic,
+        # even with regard to code generated via (gencode-*) that end up being
+        # stored as a `FileScriptProperty`; since most Unix/Linux systems are
+        # able to detect the **shebang**
+        # -------------------------------------------------------------------
+        if value:
+            path = self._get_path(instance)
+            if fs.file_has_shebang(path):
+                fs.ensure_file_is_executable_by_all(path)

docs/changelog

@@ -1,10 +1,10 @@
 Changelog
 ---------
 
-next:
-	* Type __timezone: Add support for OpenWRT (Nico Schottelius)
-
 7.0.1:
+  * Documentation: Add a new page and several mentions for polyglot scripting (Tabulon)
+  * Core: Automatically set execute permissions on generated scripts containing a shebang (Tabulon)
+  * Remote: Run executable scripts directly, enabling transparent shebang support (Tabulon)
 	* Core: Remove double definition of scan parser (Nico Schottelius)
 	* Type __apt_mark: Narrow down grep for hold packages (marcoduif)
 	* Type __apt_source: Set required options variable (Mark Verboom)

docs/src/cdist-explorer.rst

@@ -3,12 +3,29 @@ Explorer
 
 Description
 -----------
-Explorers are small shell scripts, which will be executed on the target
+Explorers are small scripts, typically written in POSIX shell,
-host. The aim of each explorer is to give hints to types on how to act on the
+which will be executed on the target host.
+The aim of each explorer is to give hints to types on how to act on the
 target system. An explorer outputs the result to stdout, which is usually
 a one liner, but may be empty or multi line especially in the case of
 type explorers.
 
+.. tip::
+    An :program:`explorer` can be written in **any scripting language**,
+    provided it is executable and has a proper **shebang**.
+
+    Nevertheless, for explorers, it is usually best to stick with the
+    **POSIX shell** in order to minimize
+    requirements on target hosts where they would need to be executed.
+
+    For executable shell code, the recommended shebang is :code:`#!/bin/sh -e`.
+
+    If an :program:`explorer` lacks `execute` permissions,
+    :program:`cdist` assumes it to be written in **shell** and executes it using
+    `$CDIST_REMOTE_SHELL`, which defaults to :code:`/bin/sh -e`.
+
+    For more details and examples, see :doc:`cdist-polyglot`.
+
 There are general explorers, which are run in an early stage, and
 type explorers. Both work almost exactly the same way, with the difference
 that the values of the general explorers are stored in a general location and
@@ -32,9 +49,14 @@ error message on stderr, which will cause cdist to abort.
 You can also use stderr for debugging purposes while developing a new
 explorer.
 
+
 Examples
 --------
-A very simple explorer may look like this::
+A very simple explorer may look like this:
+
+.. code-block:: sh
+
+    #!/bin/sh -e
 
     hostname
 
@@ -44,6 +66,8 @@ A type explorer, which could check for the status of a package may look like thi
 
 .. code-block:: sh
 
+    #!/bin/sh -e
+
     if [ -f "$__object/parameter/name" ]; then
        name="$(cat "$__object/parameter/name")"
     else

docs/src/cdist-features.rst

@@ -22,8 +22,14 @@ Fast development
     Focus on straightforwardness of type creation is a main development objective
     Batteries included: A lot of requirements can be solved using standard types
 
-Modern Programming Language
+Modern Programming Language (for cdist itself)
-    cdist is written in Python
+    cdist itself is written in Python
+
+Language-agnostic / Polyglot (for the rest)
+    Although cdist itself is written in Python, it can be configured
+    and extended with any scripting language available.
+
+    (The `POSIX shell <https://en.wikipedia.org/wiki/Unix_shell>`_ is recommended, especially for any code destined to run on target hosts)
 
 Requirements, Scalability
     No central server needed, cdist operates in push mode and can be run from any computer
@@ -44,5 +50,6 @@ UNIX, familiar environment, documentation
     Is available as manpages and HTML
 
 UNIX, simplicity, familiar environment
-    cdist is configured in POSIX shell
+    The ubiquitious `POSIX shell <https://en.wikipedia.org/wiki/Unix_shell>`_ is the recommended language for configuring and extending cdist.
 
+    The :program:`Cdist API` is based on simple and familiar UNIX constructs: environment variables, standard I/O, and files/directories

docs/src/cdist-manifest.rst

@@ -3,7 +3,9 @@ Manifest
 
 Description
 -----------
-Manifests are used to define which objects to create.
+Manifests are scripts that are executed *locally* (on master)
+for the purpose of defining which objects to create.
+
 Objects are instances of **types**, like in object oriented programming languages.
 An object is represented by the combination of
 **type + slash + object name**: **\__file/etc/cdist-configured** is an
@@ -24,10 +26,10 @@ at an example::
     # Same with the __directory type
     __directory /tmp/cdist --state present
 
-These two lines create objects, which will later be used to realise the 
+These two lines create objects, which will later be used to realise the
 configuration on the target host.
 
-Manifests are executed locally as a shell script using **/bin/sh -e**.
+Manifests are executed *locally* (on master).
 The resulting objects are stored in an internal database.
 
 The same object can be redefined in multiple different manifests as long as
@@ -36,6 +38,20 @@ the parameters are exactly the same.
 In general, manifests are used to define which types are used depending
 on given conditions.
 
+.. tip::
+
+    A manifest can be written in **any scripting language**,
+    provided that the script is executable and has a proper **shebang**.
+
+    For executable shell code, the recommended shebang is :code:`#!/bin/sh -e`.
+
+    If :program:`manifest` lacks `execute` permissions,  :program:`cdist` assumes
+    it to be written in **shell** and executes it using
+    `$CDIST_LOCAL_SHELL`, which defaults to :code:`/bin/sh -e`.
+
+    For more details and examples, see :doc:`cdist-polyglot`.
+
+.. _cdist-manifest#initial-and-type-manifests:
 
 Initial and type manifests
 --------------------------
@@ -64,14 +80,14 @@ environment variable **__target_host** and/or **__target_hostname** and/or
        ;;
     esac
 
-This manifest says: Independent of the host, always use the type 
+This manifest says: Independent of the host, always use the type
 **__cdistmarker**, which creates the file **/etc/cdist-configured**,
 with the timestamp as content.
-The directory **/home/services/kvm-vm**, including all parent directories, 
+The directory **/home/services/kvm-vm**, including all parent directories,
 is only created on the host **localhost**.
 
 As you can see, there is no magic involved, the manifest is simple shell code that
-utilises cdist types. Every available type can be executed like a normal 
+utilises cdist types. Every available type can be executed like a normal
 command.
 
 
@@ -102,17 +118,17 @@ delimiters including space, tab and newline.
 
      1 # No dependency
      2 __file /etc/cdist-configured
-     3 
+     3
      4 # Require above object
      5 require="__file/etc/cdist-configured" __link /tmp/cdist-testfile \
      6    --source /etc/cdist-configured  --type symbolic
-     7 
+     7
      8 # Require two objects
      9 require="__file/etc/cdist-configured __link/tmp/cdist-testfile" \
     10    __file /tmp/cdist-another-testfile
 
 
-Above the "require" variable is only set for the command that is 
+Above the "require" variable is only set for the command that is
 immediately following it. Dependencies should always be declared that way.
 
 On line 4 you can see that the instantiation of a type "\__link" object needs
@@ -156,7 +172,7 @@ in `cdist execution stages <cdist-stages.html>`_ and of how types work in `cdist
 
 Create dependencies from execution order
 -----------------------------------------
-You can tell cdist to execute all types in the order in which they are created 
+You can tell cdist to execute all types in the order in which they are created
 in the manifest by setting up the variable CDIST_ORDER_DEPENDENCY.
 When cdist sees that this variable is setup, the current created object
 automatically depends on the previously created object.
@@ -288,14 +304,14 @@ and there are no other dependencies from this manifest.
 
 Overrides
 ---------
-In some special cases, you would like to create an already defined object 
+In some special cases, you would like to create an already defined object
 with different parameters. In normal situations this leads to an error in cdist.
 If you wish, you can setup the environment variable CDIST_OVERRIDE
-(any value or even empty is ok) to tell cdist, that this object override is 
+(any value or even empty is ok) to tell cdist, that this object override is
 wanted and should be accepted.
-ATTENTION: Only use this feature if you are 100% sure in which order 
+ATTENTION: Only use this feature if you are 100% sure in which order
 cdist encounters the affected objects, otherwise this results
-in an undefined situation. 
+in an undefined situation.
 
 If CDIST_OVERRIDE and CDIST_ORDER_DEPENDENCY are set for an object,
 CDIST_ORDER_DEPENDENCY will be ignored, because adding a dependency in case of
@@ -348,11 +364,11 @@ How to override objects:
     # (e.g. for example only sourced if a special application is on the target host)
 
     # this leads to an error ...
-    __user foobar --password 'some_other_hash' 
+    __user foobar --password 'some_other_hash'
 
     # this tells cdist, that you know that this is an override and should be accepted
     CDIST_OVERRIDE=yes __user foobar --password 'some_other_hash'
-    # it's only an override, means the parameter --home is not touched 
+    # it's only an override, means the parameter --home is not touched
     # and stays at the original value of /home/foobarexample
 
 Dependencies defined by execution order work as following:

docs/src/cdist-polyglot.rst

@@ -0,0 +1,443 @@
+Polyglot
+========
+
+Description
+-----------
+
+Although **cdist** itself is written in **Python**, it features a
+*language-agnostic* (and hence *polyglot*) extension system.
+
+As such, **cdist** can be extended with a mix-and-match of
+**any scripting language** in addition to the usual -and recommended-
+**POSIX shell** (`sh`): `bash`, `perl`, `python`, `ruby`, `node`, ... whatever.
+
+This is true for all extension mechanisms available for **cdist**, namely:
+
+.. list-table::
+
+    * - :doc:`manifests <cdist-manifest>`
+      - (including :ref:`manifest/init <cdist-manifest#initial-and-type-manifests>`
+        and :ref:`type manifests <cdist-type#manifest>`)
+
+    * - :doc:`explorers <cdist-explorer>`
+      - (both **global** and :ref:`type explorers <cdist-type#explorers>`)
+
+    * - :ref:`gencode-* scripts <cdist-type#gencode-scripts>`
+      - (both :program:`gencode-local` and :program:`gencode-remote`)
+
+    * - and even :ref:`generated code <cdist-type#gencode-scripts>`
+      - (i.e. the outputs from
+        :ref:`gencode-* scripts <cdist-type#gencode-scripts>`)
+
+
+.. raw:: html
+
+    <details>
+    <summary>
+        <a>You do not have to commit to any single language...</a>
+    </summary>
+
+.. container::
+
+    .. note::
+
+        It's indeed possible (though not necessarily recommended)
+        to **mix-and-match** different
+        languages when extending **cdist**, for example:
+
+        A **type** could, in principal, have a `manifest` and an **explorer** written
+        in **POSIX shell**, a `gencode-remote` in **Python**
+        (which could generate code in **POSIX shell**) and a `gencode-local`
+        in **Perl**  (which could generate code in **Perl**,
+        or some other language), while you are at it...
+
+        Just don't expect to submit such a hodge-podge as a candidate for being
+        distributed  with **cdist** itself, though... :-)
+        especially if it turns out to be something that can be acheieved with
+        reasonable effort in **POSIX shell**.
+
+        In practise, you would at least want to enforce some consistency, if anything for
+        code maintainibility and your own sanity, in addition to the
+        the `CAVEATS`_ mentioned down below.
+
+.. raw:: html
+
+    </details>
+    <br/>
+
+Needless to say, just because you *can* do something,
+doesn't mean you *should* be doing it, or it's even a *good idea* to do so.
+
+As a general rule of thumb, when extending **cdist**,
+there are many good reasons in favor of sticking with the **POSIX shell**
+wherever you can, and very few in favor of opting for some other
+scripting language.
+
+This is particularly true for any code that is meant to be run *remotely*
+on **target hosts** (such as **explorers**),
+where it is usually important to keep assumptions and requirements/dependencies
+to a bare minimum. See the  `CAVEATS`_ down below.
+
+That being said, **polyglot** capabilities of **cdist** can come
+quite handy for when you really need this sort of thing,
+provided that you are ready to bare the consequences,
+including the burden of extra dependecies
+--- which is usually not that hard for code run *locally* on **master**
+(`manifests`, `gencode-*` scripts, and code generated by `gencode-local`).
+
+In any case, the mere fact of knowing we *can* escape the POSIX hatch
+if we really have to, can be quite comforting for those of us suffering
+from POSIX claustrophobia... which *is* of course a real health hazard
+associated with high anxiety levels and all,
+in case you didn't already know... ;-)
+
+
+Writing polyglot extensions for **cdist**
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Whatever the kind of script (`manifest`, explorer, ...) you are writing,
+you need to ensure that all 3 conditions below are met:
+
+1.  your script starts with an appropriate **shebang** line, such as::
+
+      #!/usr/bin/env bash
+
+    .. comment: It would have been nice to make use of an extension
+        (such as `"sphinx_design"`) which provides a `.. dropdown::`
+        directive (for toggling visibility) which is the reason for
+        the ugly `.. raw:: html` stuff below...
+
+    .. raw:: html
+
+        <details>
+        <summary><a>It's usually preferable to rely on the <b>env</b> program...</a></summary>
+
+    .. container::
+
+        It's usually preferable to rely on the :program:`env` program,
+        like in the example above, to find the interpreter by searching the PATH.
+
+        The :program:`env` program is almost guaranteed to exist even on a rudimentary
+        UNIX/Linux system at a pretty stable location: `/usr/bin/env`
+
+        It is, of course, also possible to write down a **hard coded** path
+        for the interpreter, if you are certain that it will always be
+        located at that location, like so::
+
+            #!/bin/bash
+
+        This may sometimes be desirable, for example when you want to ascertain
+        using a specific version of an interpreter or when you are unsure about
+        what might get foundthrough the PATH.
+
+    .. raw:: html
+
+        </details>
+
+2.  your script has "*execute*" permissions set (in the Unix/Linux sense),
+    like so::
+
+        chmod a+x /path/to/your/script
+
+    This is essentially what matters to **cdist**, which it will take as a
+    clue for invoking your script *directly* (instead of passing it
+    to a shell as an argument).
+
+    For **generated code**, `cdist` will automatically take care of setting
+    *execute* permissions for you,
+    based on the presence of a leading **shebang** within the generated code.
+
+3.  the **interpreter** referenced by the **shebang** is available on any host(s)
+    where your code will run.
+
+.. raw:: html
+
+    <details>
+    <summary>
+        <a>
+        Even for the <b>POSIX shell</b>,
+        it is still recommended to <b>follow the same guidelines</b> outlined above.
+        </a>
+    </summary>
+
+.. note::
+
+    Even if you are just writing for the **POSIX shell**,
+    it is still recommended to follow the same guidelines outlined above.
+
+    At the very least, make sure your script has a proper **shebang**.
+
+    -   If you have been following the usual **cdist** advise:
+            you probably already have a proper **shebang** at the very beginning
+            of your POSIX shell scripts.
+
+
+    -   If (and *only* if), your POSIX shell script *does* contain a proper **shebang**:
+            you are also encouraged to also give it *"execute"* permissions,
+            so that your **shebang** will actually get honored.
+
+.. raw:: html
+
+    </details>
+    <br/>
+
+
+That's pretty much it... except...
+
+.. seealso:: The `CAVEATS`_ below.
+
+
+CAVEATS
+^^^^^^^^^^^^
+
+Shebang and execute permissions
+"""""""""""""""""""""""""""""""""
+In general, the first two conditions above are trivial to satisfy:
+Just make sure you put in a **shebang** and mark your script as *executable*.
+
+
+**Beware**, however, that:
+
+.. attention::
+
+    -   If your script lacks `execute` permissions (regardless of any **shebang**):
+            **cdist** will end up passing your script to `/bin/sh -e`
+            (or to `local_shell` / `remote_shell`,
+            if one is configured for the current context),
+            which may or may not be what you want.
+
+    -   If your script *does* have `execute` permissions but *lacks* a **shebang**:
+            you can no longer be sure which interpreter (if any) will end up running your script.
+
+            What is certain, on the other hand, is that there is a wide range of
+            different things that could happen in such a case, depending on the OS and the chain
+            of execution up to that point...
+
+            It is possible (but not certain) that, in such a case, your script may
+            end up getting fed into `/bin/sh` or the default shell
+            (whatever it happens to be for the current user).
+
+            There's even a legend according to which even `csh` may get a chance to feed
+            on your script, and then proceed to burning your barn...
+
+            So, don't do that.
+
+
+
+
+Interpreter availibility
+"""""""""""""""""""""""""""""""""
+
+For the last condition (interpreter availability),
+your mileage may vary for languages other than the **POSIX shell**.
+
+- For scripts meant to be run *locally* on the **master**, things remain relatively easy :
+    All you may need, if anything,
+    is a one time installation of stuff.
+
+    So, things should be realtively easy when it comes to: :file:`manifest` and :file:`gencode-*` scripts themselves, as well as any code generated by :file:`gencode-local`.
+
+
+- For scripts meant to be run *remotely* on **target hosts**, things might get quite tricky,
+    depending on how likely it is
+    for the desired **interpreter** to be installed by default
+    on the **target system**.
+
+    This is an important concern for :file:`explorer` scripts
+    and any code generated by :file:`gencode-remote`.
+
+    .. warning::
+
+        Apart from the POSIX shell (`/bin/sh`), there aren't many interpreters out
+        there that are likely to have a guaranteed presence on a pristine system.
+
+        At the very least, you would have to make sure that the required interpreter
+        (and any extra modules/libraries your script might depend on)
+        are indeed available on those host(s)
+        before your script is invoked...
+        which kind of goes against the near-zero-dependency philosphy embraced
+        by **cdist**.
+
+        Depending on the target host OS, you might get lucky with
+        `bash`, `perl`, or `python` being preinstalled.
+        Even then, those may not necessarily be the version you expect
+        or have the extra modules/libraries your script might require.
+
+        **You have been warned.**
+
+
+More details
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As mentioned earlier, **cdist** itself mostly cares about the script
+being marked as an *executable*, which it will take as a clue for invoking
+that script *directly* (instead of passing it to a shell as an argument).
+
+The **shebang** magic is handled by the usual process `exec` mechanisms
+of the host OS (where the script is invoked) that will take over from
+that point on.
+
+
+Here is a simplified summary :
+
++-------------+---------------+------------------------------+--------------+--------------------------------------------------------+
+| executable? | shebang       | invocation resembles         | interpreter  | remarks                                                |
++=============+===============+==============================+==============+========================================================+
+| yes         | `#!/bin/sh`   | `/path/to/script`            | `/bin/sh`    | shebang **honored** by OS                              |
++-------------+---------------+------------------------------+--------------+--------------------------------------------------------+
+| yes         | `#!/bin/bash` | `/path/to/script`            | `/bin/bash`  | shebang **honored** by OS                              |
++-------------+---------------+------------------------------+--------------+--------------------------------------------------------+
+| yes         |               | `/path/to/script`            | *uncertain*  | shebang **absent**                                     |
++-------------+---------------+------------------------------+--------------+--------------------------------------------------------+
+| no          | `#!/bin/sh`   | `/bin/sh -e /path/to/script` | `/bin/sh -e` | shebang **irrelevant** (as script is not "executable") |
++-------------+---------------+------------------------------+--------------+--------------------------------------------------------+
+| no          | `#!/bin/bash` | `/bin/sh -e /path/to/script` | `/bin/sh -e` | shebang **irrelevant** (as script is not "executable") |
++-------------+---------------+------------------------------+--------------+--------------------------------------------------------+
+| no          |               | `/bin/sh -e /path/to/script` | `/bin/sh -e` | shebang **irrelevant** (as script is not "executable") |
++-------------+---------------+------------------------------+--------------+--------------------------------------------------------+
+
+In fact, it's a little bit more involved than the above. Remember:
+
+- As a special case, for any **generated code** (output by `gencode-*` scripts),
+  **cdist** will solely rely on the presence (or absence) of a leading **shebang**,
+  and set the executable bits accordingly, for obvious reasons.
+
+- In the end, if a script is NOT marked as "executable",
+  it will simply be passed as an argument to the configured shell
+  that corresponds to the relevant context (i.e. `local_shell` or `remote_shell`),
+  if one is defined within the **cdist** configuration,
+  or else to `/bin/sh -e`, as a fallback in in both cases.
+
+Well, there are also some gory implementation details
+(related to how environment variables get propagated),
+but those should normally have no relevance to this discussion.
+
+
+The API between **cdist** and any polyglot extensions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Conceptually, the API, based on well-known UNIX constructs,
+remains exactly the same as it is for
+any extension written for the **POSIX shell**.
+
+Basically, you are all set as long as your scripting language is capable of:
+
+- accessing **environment variables**;
+- reading from and writing to the **filesystem** (files, directories, ...);
+- reading from :file:`STDIN` and writing to :file:`STDOUT` (and eventually to :file:`STDERR`)
+- **executing** other programs/commands;
+- **exiting** with an appropriate **status code** (where 0=>success).
+
+For all we know, no serious scripting language out there
+would be missing any such basics.
+
+The actual syntax and mechanisms will obviously be different,
+the shell idioms usually being much more concise for this sort of thing,
+as expected.
+
+See the below example entitled "`Interacting with the cdist API`_".
+
+
+Examples
+-------------------
+
+Interacting with the cdist API
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As an API example, here's an excerpt from a **cdist** `type manifest`,
+written for the POSIX shell, showing how one would get at the name
+of the kernel on the **target host**:::
+
+    kernel_name=$(cat "${__global}/explorer/kernel_name")
+
+    # ... do something with kernel_name ...
+
+
+In a nutshell, the above snippet gives the general idea about the cdist API:
+
+Basically, we are stuffing a shell variable with the contents of a file...
+which happens to contain the output from the `kernel_name` explorer...
+
+Before invoking our `manifest` script,  **cdist** would have, among other things,
+run all **global explorers** on the **target host**,
+collected and copied their outputs under a temporary directory on the **master**, and
+set a specific environment variable (`$__global`)
+to the path of a specifc subdirectory of that temporary working area.
+
+At this point, that file (which contains the kernel name) is sitting there,
+ready to be slurped... which can obviously be done from any language
+that can access environment variables and read files from the filesystem...
+
+Here's how you could do the same thing in **Python**:
+
+.. code-block:: python
+
+    #!/usr/bin/env python
+
+    import os
+
+    def read_file(path):
+        content = ""
+        try:
+            with open(path, "r") as fd:
+                content = fd.read().rstrip('\n')
+        except EnvironmentError:
+            pass
+        return content
+
+    kernel_name = read_file( os.environ['__global'] + '/explorer/kernel_name' )
+
+    # ... do something with kernel_name ...
+
+
+And in **Perl**, it could look like:
+
+.. code-block:: perl
+
+    #!/usr/bin/env perl
+
+    sub read_file {
+        my ($path) = @_;
+        return unless open( my $fh, $path );
+        local ($/);
+        <$fh>
+    }
+
+    my $kernel_name = read_file("$ENV{__global}/explorer/kernel_name");
+
+    # ... do something with kernel_name ...
+
+
+Incidently, this example also helps appreciate some aspects of programming
+for the shell... which were designed for this sort of thing in the first place...
+
+A polygot type explorer (in Perl)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Here's an imaginary type explorer written in **Perl**,
+that ouputs the version of the perl interpreter running on the target host:
+
+.. code-block:: perl
+
+    #!/usr/bin/env perl
+
+    use English;
+
+    print "${PERL_VERSION}\n";
+
+If the path to the intended interpreter can be ascertained, you can
+put that down directly on the **shebang**, like so::
+
+     #!/usr/bin/perl
+
+However, more often than not, you would want to rely
+on the `env` program (`/usr/bin/env`) to
+invoke the first interpreter with the given name (`perl`, in this case)
+found on the current PATH, like in the above example.
+
+Don't forget to set *execute* permissions on the script file:::
+
+    chmod a+x ...
+
+Or else **cdist** will feed it to a shell instance...
+which may burn your barn... :-)

docs/src/cdist-type.rst

@@ -79,9 +79,9 @@ then this content is printed as a deprecation messages, e.g.:
 
 .. code-block:: sh
 
-    $ ls -l deprecated 
+    $ ls -l deprecated
     -rw-r--r--  1 darko  darko  71 May 20 18:30 deprecated
-    $ cat deprecated 
+    $ cat deprecated
     This type is deprecated. It will be removed in the next minor release.
     $ echo '__foo foo' | ./bin/cdist config -i - 185.203.112.26
     WARNING: 185.203.112.26: Type __foo is deprecated: This type is deprecated. It will be removed in the next minor release.
@@ -90,7 +90,7 @@ If 'deprecated' marker has no content then general message is printed, e.g.:
 
 .. code-block:: sh
 
-    $ ls -l deprecated 
+    $ ls -l deprecated
     -rw-r--r--  1 darko  darko  0 May 20 18:36 deprecated
     $ echo '__bar foo' | ./bin/cdist config -i - 185.203.112.26
     WARNING: 185.203.112.26: Type __bar is deprecated.
@@ -98,41 +98,132 @@ If 'deprecated' marker has no content then general message is printed, e.g.:
 
 How to write a new type
 -----------------------
-A type consists of
-
-- parameter    (optional)
-- manifest     (optional)
-- singleton    (optional)
-- explorer     (optional)
-- gencode      (optional)
-- nonparallel  (optional)
 
 Types are stored below cdist/conf/type/. Their name should always be prefixed with
-two underscores (__) to prevent collisions with other executables in $PATH.
+two underscores (__) to prevent collisions with other executables in :code:`$PATH`.
 
-To implement a new type, create the directory **cdist/conf/type/__NAME**.
+To implement a new type, create the directory :file:`cdist/conf/type/{__NAME}`,
+either manually or using the helper script `cdist-new-type <man1/cdist-new-type.html>`_
+which will also create the basic skeleton for you.
 
-Type manifest and gencode can be written in any language. They just need to be
+A type consists of the following elements (all of which are currently *optional*):
-executable and have a proper shebang. If they are not executable then cdist assumes
-they are written in shell so they are executed using '/bin/sh -e' or 'CDIST_LOCAL_SHELL'.
 
-For executable shell code it is suggested that shebang is '#!/bin/sh -e'.
+* some **markers** in the form of **plain files** within the type's directory:
 
-For creating type skeleton you can use helper script
+    .. list-table::
-`cdist-new-type <man1/cdist-new-type.html>`_.
 
+        * - :file:`singleton`
+          - *(optional)*
+          - A type flagged as a :file:`singleton` may be used **only
+            once per host**, which is useful for types that can be used only once on a
+            system.
+
+            .. raw:: html
+
+                <br/>
+
+            Singleton types do not take an object name as argument.
+
+        * - :file:`nonparallel`
+          - (optional)
+          - Objects of a type flagged as :file:`nonparallel` cannot be run in parallel
+            when using :code:`-j` option.
+
+            .. raw:: html
+
+                <br/>
+
+
+            An example of such a type is :program:`__package_dpkg` type
+            where :program:`dpkg` itself prevents to be run in more than one instance.
+
+        * - :file:`install`
+          - *(optional)*
+          - A type flagged as :file:`install` is used only with :command:`install` command.
+            With other :program:`cdist` commands, i.e. :command:`config`, such types are skipped if used.
+
+        * - :file:`deprecated`
+          - *(optional)*
+          - A type flagged as :file:`deprecated` causes
+            :program:`cdist` to emit a **warning** whenever that type is used.
+
+            .. raw:: html
+
+                <br/>
+
+            If the file that corresponds to the `deprecated` marker has any content,
+            then this is used as a custom **deprecation message** for the warning.
+
+* some more **metadata**:
+
+    .. list-table::
+
+        * - :file:`parameter/\*`
+          - *(optional)*
+          - A type may have **parameters**. These must be declared following a simple convention described in `Defining parameters`_, which
+            permits specifying additional properties for each parameter:
+
+                * required or optional
+                * single-value or multi-value
+                * string or boolean
+
+            It is also possible to give a `default` value for each optional parameter.
+
+* and some **code** (scripts):
+
+    .. list-table::
+
+        * - :file:`manifest`
+          - *(optional)*
+          - :doc:`Type manifest <cdist-manifest>`
+
+        * - :file:`explorer/*`
+          - *(optional)*
+          - Any number of :doc:`type explorer <cdist-explorer>` scripts may exist under :file:`explorer` subdirectory.
+
+        * - :file:`gencode-local`
+          - *(optional)*
+          - A script that generates code to be executed *locally* (on master).
+
+        * - :file:`gencode-remote`
+          - *(optional)*
+          - A script that generates code to be executed *remotely* (on target host).
+
+
+.. tip::
+
+    Each of the above-mentioned scripts can be written in **any scripting language**,
+    provided that the script is executable and has a proper **shebang**.
+
+    For executable shell code, the recommended shebang is :code:`#!/bin/sh -e`.
+
+    If a script lacks `execute` permissions,  :program:`cdist` assumes
+    it to be written in **shell** and executes it using
+    `$CDIST_LOCAL_SHELL` or `$CDIST_REMOTE_SHELL`, if one is defined
+    for the current execution context (*local* or *remote*),
+    or else falling back to :code:`/bin/sh -e`.
+
+
+    For any code susceptible to run on remote target hosts
+    (i.e. **explorers** and any code generated by :code:`gencode-remote`),
+    it is recommended to stick to **POSIX shell**
+    in order to minimize requirements on target hosts where they would need to be executed.
+
+    For more details and examples, see :doc:`cdist-polyglot`.
+
+.. seealso:: `cdist execution stages <cdist-stages.html>`_
 
 Defining parameters
 -------------------
 Every type consists of required, optional and boolean parameters, which must
 each be declared in a newline separated file in **parameter/required**,
-**parameter/required_multiple**, **parameter/optional**, 
+**parameter/required_multiple**, **parameter/optional**,
 **parameter/optional_multiple** and **parameter/boolean**.
 Parameters which are allowed multiple times should be listed in
 required_multiple or optional_multiple respectively. All other parameters
 follow the standard unix behaviour "the last given wins".
 If either is missing, the type will have no required, no optional, no boolean
-or no parameters at all. 
+or no parameters at all.
 
 Default values for optional parameters can be predefined in
 **parameter/default/<name>**.
@@ -237,7 +328,7 @@ In the __file type, stdin is used as source for the file, if - is used for sourc
         source="$(cat "$__object/parameter/source")"
         if [ "$source" = "-" ]; then
             source="$__object/stdin"
-        fi  
+        fi
     ....
 
 
@@ -307,6 +398,7 @@ stdin from */dev/null*:
         done < "$__object/parameter/foo"
     fi
 
+.. _cdist-type#manifest:
 
 Writing the manifest
 --------------------
@@ -380,6 +472,7 @@ in your type directory:
 
 For example, package types are nonparallel types.
 
+.. _cdist-type#explorers:
 
 The type explorers
 ------------------
@@ -402,6 +495,7 @@ client, like this (shortened version from the type __file):
        md5sum < "$destination"
     fi
 
+.. _cdist-type#gencode-scripts:
 
 Writing the gencode script
 --------------------------

docs/src/cdist-why.rst

@@ -4,44 +4,65 @@ Why should I use cdist?
 There are several motivations to use cdist, these
 are probably the most popular ones.
 
-Known language
+No need to learn a new language
---------------
+-------------------------------
 
-Cdist is being configured in
+When adopting cdist, your staff does not need to learn a new
-`shell script <https://en.wikipedia.org/wiki/Shell_script>`_.
-Shell script is used by UNIX system engineers for decades.
-So when cdist is introduced, your staff does not need to learn a new
 `DSL <https://en.wikipedia.org/wiki/Domain-specific_language>`_
-or programming language.
+or programming language, as cdist can be configured
+and extended in **any scripting language**, the recommended one
+being `shell scripts <https://en.wikipedia.org/wiki/Shell_script>`_.
+
+Shell scripts enjoy ubiquity: they have been widely used by UNIX system engineers
+for decades, and a suitable interpreter (:code:`/bin/sh`) is all but
+guaranteed to be widely available on target hosts.
+
+
+Easy idempotance -- without having to give up control
+-----------------------------------------------------------------------------
+
+For the sake of `idempotence <https://en.wikipedia.org/wiki/Idempotence>`_, many **contemporary SCMs**  choose to ditch the power and versatality of general purpose programming languages, and adopt some form of
+declarative `DSL <https://en.wikipedia.org/wiki/Domain-specific_language>`_ for describing the desired end states on target systems.
+
+:program:`Cdist` takes a quite different approach, enabling *both* `idempotence <https://en.wikipedia.org/wiki/Idempotence>`_ *and* a decent level of programming power.
+
+Unlike other SCMs, :program:`cdist` allows you to use a general purpose scripting language (POSIX shell is recommended) for describing the desired end states on target systems, instead of some declarative `DSL <https://en.wikipedia.org/wiki/Domain-specific_language>`_.
+
+Unlike regular scripting, however, you are not left on your own for ensuring `idempotence <https://en.wikipedia.org/wiki/Idempotence>`_. :program:`Cdist` makes this really easy.
+
+It does not matter how many times you "invoke" **cdist types** and in which order: :program:`cdist` will ensure that the actual code associated with each type will be executed only once (in dependency order) which, in turn, may effectively end up becoming a no-op, if the actual state is already the same as the desired one.
+
+.. TODO: It would be great if there were an "architectural overview" page which could be referenced from here.
+
 
 Powerful language
------------------
+--------------------
 
-Not only is shell scripting widely known by system engineers,
+Compared to a typical `DSL <https://en.wikipedia.org/wiki/Domain-specific_language>`_,
-but it is also a very powerful language. Here are some features
+shell scripts feature a much more powerful language.
-which make daily work easy:
+Here are some features which make daily work easy:
 
- * Configuration can react dynamically on explored values
+ * Ability to dynamically adapt configuration based on information
+   *explored* from target hosts;
  * High level string manipulation (using sed, awk, grep)
  * Conditional support (**if, case**)
  * Loop support (**for, while**)
- * Support for dependencies between cdist types
+ * Variable expansion
+ * Support for dependencies between cdist types and objects
+
+If and when needed, it's always possible to simply
+make use of **any other scripting language** at your disposal
+*(albeit at the expense of adding a dependency on the corresponding interpreter
+and libraries)*.
 
-More than shell scripting
--------------------------
 
-If you compare regular shell scripting with cdist, there is one major
-difference: When using cdist types,
-the results are
-`idempotent <https://en.wikipedia.org/wiki/Idempotence>`_.
-In practise that means it does not matter in which order you
-call cdist types, the result is always the same.
 
 Zero dependency configuration management
-----------------------------------------
+-----------------------------------------
 
 Cdist requires very little on a target system. Even better,
-in almost all cases all dependencies are usually fulfilled.
+in almost all cases all dependencies are usually already
+fulfilled.
 Cdist does not require an agent or high level programming
 languages on the target host: it will run on any host that
 has a **ssh server running** and a POSIX compatible shell

docs/src/index.rst

@@ -30,6 +30,7 @@ It natively supports IPv6 since the first release.
    cdist-type
    cdist-types
    cdist-explorer
+   cdist-polyglot
    cdist-messaging
    cdist-parallelization
    cdist-inventory