Implement python types

cdist/conf/type/__file_py/__init__.py

@@ -0,0 +1,103 @@
+import os
+import re
+import sys
+from cdist.core import PythonType
+
+
+class FileType(PythonType):
+    def get_attribute(self, stat_file, attribute, value_should):
+        if os.path.exists(stat_file):
+            if re.match('[0-9]', value_should):
+                index = 1
+            else:
+                index = 2
+            with open(stat_file, 'r') as f:
+                for line in f:
+                    if re.match(attribute + ":", line):
+                        fields = line.split()
+                        return fields[index]
+            return None
+
+    def set_attribute(self, attribute, value_should, destination):
+        cmd = {
+            'group': 'chgrp',
+            'owner': 'chown',
+            'mode': 'chmod',
+        }
+        self.send_message("{} '{}'".format(cmd[attribute], value_should))
+        return "{} '{}' '{}'".format(cmd[attribute], value_should, destination)
+
+    def type_manifest(self):
+        yield from ()
+
+    def type_gencode(self):
+        typeis = self.get_explorer('type')
+        state_should = self.get_parameter('state')
+
+        if state_should == 'exists' and typeis == 'file':
+            return
+
+        source = self.get_parameter('source')
+        if source == '-':
+            source = self.stdin_path
+        destination = '/' + self.object_id
+        if state_should == 'pre-exists':
+            if source is not None:
+                self.die('--source cannot be used with --state pre-exists')
+            if typeis == 'file':
+                return None
+            else:
+                self.die('File {} does not exist'.format(destination))
+
+        create_file = False
+        upload_file = False
+        set_attributes = False
+        code = []
+        if state_should == 'present' or state_should == 'exists':
+            if source is None:
+                remote_stat = self.get_explorer('stat')
+                if not remote_stat:
+                    create_file = True
+            else:
+                if os.path.exists(source):
+                    if typeis == 'file':
+                        local_cksum = self.run_local(['cksum', source, ])
+                        local_cksum = local_cksum.split()[0]
+                        remote_cksum = self.get_explorer('cksum')
+                        remote_cksum = remote_cksum.split()[0]
+                        upload_file = local_cksum != remote_cksum
+                    else:
+                        upload_file = True
+                else:
+                    self.die('Source {} does not exist'.format(source))
+            if create_file or upload_file:
+                set_attributes = True
+                tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination)
+                destination_upload = self.run_remote(
+                    ["mktemp", tempfile_template, ])
+                if upload_file:
+                    self.transfer(source, destination_upload)
+                code.append('rm -rf {}'.format(destination))
+                code.append('mv {} {}'.format(destination_upload, destination))
+
+        if state_should in ('present', 'exists', 'pre-exists', ):
+            for attribute in ('group', 'owner', 'mode', ):
+                if attribute in self.parameters:
+                    value_should = self.get_parameter(attribute)
+                    if attribute == 'mode':
+                        value_should = re.sub('^0', '', value_should)
+                    stat_file = self.get_explorer_file('stat')
+                    value_is = self.get_attribute(stat_file, attribute,
+                                                  value_should)
+                    if set_attributes or value_should != value_is:
+                        code.append(self.set_attribute(attribute,
+                                                       value_should,
+                                                       destination))
+        elif state_should == 'absent':
+            if typeis == 'file':
+                code.append('rm -f {}'.format(destination))
+                self.send_message('remove')
+        else:
+            self.die('Unknown state {}'.format(state_should))
+
+        return "\n".join(code)

cdist/conf/type/__file_py/explorer/cksum

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Retrieve the md5sum of a file to be created, if it is already existing.
+#
+
+destination="/$__object_id"
+
+if [ -e "$destination" ]; then
+    if [ -f  "$destination" ]; then
+        cksum < "$destination"
+    else
+        echo "NO REGULAR FILE"
+    fi
+else
+    echo "NO FILE FOUND, NO CHECKSUM CALCULATED."
+fi

cdist/conf/type/__file_py/explorer/stat

@@ -0,0 +1,56 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+destination="/$__object_id"
+
+# nothing to work with, nothing we could do
+[ -e "$destination" ] || exit 0
+
+os=$("$__explorer/os")
+case "$os" in
+   "freebsd"|"netbsd"|"openbsd")
+      # FIXME: should be something like this based on man page, but can not test
+      stat -f "type: %ST
+owner: %Du %Su
+group: %Dg %Sg
+mode: %Op %Sp
+size: %Dz
+links: %Dl
+" "$destination"
+   ;;
+   "macosx")
+     stat -f "type: %HT
+owner: %Du %Su
+group: %Dg %Sg
+mode: %Lp %Sp
+size: %Dz
+links: %Dl
+" "$destination"
+   ;;
+   *)
+      stat --printf="type: %F
+owner: %u %U
+group: %g %G
+mode: %a %A
+size: %s
+links: %h
+" "$destination"
+   ;;
+esac

cdist/conf/type/__file_py/explorer/type

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+destination="/$__object_id"
+
+if [ ! -e "$destination" ]; then
+   echo none
+elif [ -h "$destination" ]; then
+   echo symlink
+elif [ -f "$destination" ]; then
+   echo file
+elif [ -d "$destination" ]; then
+   echo directory
+else
+   echo unknown
+fi

cdist/conf/type/__file_py/parameter/default/state

@@ -0,0 +1 @@
+present

cdist/conf/type/__file_py/parameter/optional

@@ -0,0 +1,5 @@
+state
+group
+mode
+owner
+source

cdist/config.py

@@ -90,13 +90,15 @@ class Config(object):
                 shutil.rmtree(path)
 
     def __init__(self, local, remote, dry_run=False, jobs=None,
-                 cleanup_cmds=None, remove_remote_files_dirs=False):
+                 cleanup_cmds=None, remove_remote_files_dirs=False,
+                 timestamp=False):
 
         self.local = local
         self.remote = remote
         self._open_logger()
         self.dry_run = dry_run
         self.jobs = jobs
+        self.timestamp = timestamp
         if cleanup_cmds:
             self.cleanup_cmds = cleanup_cmds
         else:
@@ -424,7 +426,8 @@ class Config(object):
                 cleanup_cmds.append(cleanup_cmd)
             c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs,
                     cleanup_cmds=cleanup_cmds,
-                    remove_remote_files_dirs=remove_remote_files_dirs)
+                    remove_remote_files_dirs=remove_remote_files_dirs,
+                    timestamp=args.timestamp)
             c.run()
             cls._remove_paths()
 
@@ -755,6 +758,21 @@ class Config(object):
                     ("The requirements of the following objects could not be "
                      "resolved:\n%s") % ("\n".join(info_string)))
 
+    def _timeit(self, func, msg_prefix):
+        def wrapper_func(*args, **kwargs):
+            loglevel = self.log.getEffectiveLevel()
+            if loglevel >= logging.VERBOSE and self.timestamp:
+                start_time = time.time()
+                rv = func(*args, **kwargs)
+                end_time = time.time()
+                duration = end_time - start_time
+                self.log.verbose("%s duration: %.6f seconds",
+                                 msg_prefix, duration)
+            else:
+                rv = func(*args, **kwargs)
+            return rv
+        return wrapper_func
+
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
         self.log.verbose("Preparing object {}".format(cdist_object.name))
@@ -762,11 +780,28 @@ class Config(object):
             "Running manifest and explorers for " + cdist_object.name)
         self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
-            self.manifest.run_type_manifest(cdist_object)
+            self.log.verbose("Preparing object {}".format(cdist_object.name))
+            self.log.verbose(
+                    "Running manifest and explorers for " + cdist_object.name)
+            self.explorer.run_type_explorers(cdist_object,
+                                             transfer_type_explorers)
+            if self.is_py_type(cdist_object):
+                self._timeit(self.manifest.run_py_type_manifest,
+                             "Python type manifest for {}".format(
+                                 cdist_object.name))(cdist_object)
+            else:
+                self._timeit(self.manifest.run_type_manifest,
+                             "Type manifest for {}".format(
+                                 cdist_object.name))(cdist_object)
             cdist_object.state = core.CdistObject.STATE_PREPARED
         except cdist.Error as e:
             raise cdist.CdistObjectError(cdist_object, e)
 
+    def is_py_type(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        init_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+        return os.path.exists(init_path)
+
     def object_run(self, cdist_object):
         """Run gencode and code for an object"""
         try:
@@ -777,9 +812,20 @@ class Config(object):
 
             # Generate
             self.log.debug("Generating code for %s" % (cdist_object.name))
-            cdist_object.code_local = self.code.run_gencode_local(cdist_object)
+            if self.is_py_type(cdist_object):
-            cdist_object.code_remote = self.code.run_gencode_remote(
+                cdist_object.code_local = ''
-                cdist_object)
+                cdist_object.code_remote = self._timeit(self.code.run_py,
+                             "Python type generate code for {}".format(
+                                 cdist_object.name))(cdist_object)
+            else:
+                cdist_object.code_local = self._timeit(
+                    self.code.run_gencode_local,
+                             "Type generate code local for {}".format(
+                                 cdist_object.name))(cdist_object)
+                cdist_object.code_remote = self._timeit(
+                    self.code.run_gencode_remote,
+                             "Type generate code remote for {}".format(
+                                 cdist_object.name))(cdist_object)
             if cdist_object.code_local or cdist_object.code_remote:
                 cdist_object.changed = True
 
@@ -790,12 +836,16 @@ class Config(object):
                 if cdist_object.code_local:
                     self.log.trace("Executing local code for %s"
                                    % (cdist_object.name))
-                    self.code.run_code_local(cdist_object)
+                    self._timeit(self.code.run_code_local,
+                                "Type run code local for {}".format(
+                                    cdist_object.name))(cdist_object)
                 if cdist_object.code_remote:
                     self.log.trace("Executing remote code for %s"
                                    % (cdist_object.name))
                     self.code.transfer_code_remote(cdist_object)
-                    self.code.run_code_remote(cdist_object)
+                    self._timeit(self.code.run_code_remote,
+                                "Type run code remote for {}".format(
+                                    cdist_object.name))(cdist_object)
 
             # Mark this object as done
             self.log.trace("Finishing run of " + cdist_object.name)

cdist/core/__init__.py

@@ -21,6 +21,7 @@
 #
 
 from cdist.core.cdist_type import CdistType
+from cdist.core.python_type import PythonType, ManifestEntry
 from cdist.core.cdist_type import InvalidTypeError
 from cdist.core.cdist_object import CdistObject
 from cdist.core.cdist_object import IllegalObjectIdError

cdist/core/code.py

@@ -22,6 +22,10 @@
 #
 
 import os
+import importlib.util
+import inspect
+import cdist
+from cdist.core import PythonType
 from . import util
 
 
@@ -113,6 +117,44 @@ class Code(object):
                 local.log),
         }
 
+    def run_py(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        module_name = cdist_type.name
+        file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+
+        if os.path.isfile(file_path):
+            spec = importlib.util.spec_from_file_location(module_name,
+                                                          file_path)
+            m = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(m)
+            classes = inspect.getmembers(m, inspect.isclass)
+            type_class = None
+            for _, cl in classes:
+                if cl != PythonType and issubclass(cl, PythonType):
+                    if type_class:
+                        raise cdist.Error("Only one python type class is "
+                                          "supported, but at least two "
+                                          "found: {}".format((type_class,
+                                                              cl, )))
+                    else:
+                        type_class = cl
+            env = os.environ.copy()
+            env.update(self.env)
+            message_prefix = cdist_object.name
+            type_obj = type_class(env=env, cdist_object=cdist_object,
+                                  local=self.local, remote=self.remote,
+                                  message_prefix=message_prefix)
+            if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run):
+                if self.local.save_output_streams:
+                    which = 'gencode-py'
+                    stderr_path = os.path.join(cdist_object.stderr_path, which)
+                    stdout_path = os.path.join(cdist_object.stdout_path, which)
+                    with open(stderr_path, 'a+') as stderr, \
+                            open(stdout_path, 'a+') as stdout:
+                        return type_obj.run(stdout=stdout, stderr=stderr)
+                else:
+                    return type_obj.run()
+
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
         script = os.path.join(self.local.type_path,

cdist/core/manifest.py

@@ -22,9 +22,13 @@
 
 import logging
 import os
-
+import importlib.util
+import inspect
 import cdist
+import cdist.emulator
 from . import util
+from cdist.core import PythonType, ManifestEntry
+
 
 '''
 common:
@@ -209,3 +213,73 @@ class Manifest(object):
                     type_manifest,
                     env=self.env_type_manifest(cdist_object),
                     message_prefix=message_prefix)
+
+    def env_py_type_manifest(self, cdist_object):
+        env = os.environ.copy()
+        env.update(self.env)
+        env.update({
+            '__cdist_object_marker': self.local.object_marker_name,
+            '__cdist_manifest': cdist_object.cdist_type,
+            '__manifest': self.local.manifest_path,
+            '__object': cdist_object.absolute_path,
+            '__object_id': cdist_object.object_id,
+            '__object_name': cdist_object.name,
+            '__type': cdist_object.cdist_type.absolute_path,
+        })
+
+        return env
+
+    def run_py_type_manifest(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        module_name = cdist_type.name
+        file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+        message_prefix = cdist_object.name
+        if os.path.isfile(file_path):
+            self.log.verbose("Running python type manifest for object %s",
+                             cdist_object.name)
+            spec = importlib.util.spec_from_file_location(module_name,
+                                                          file_path)
+            m = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(m)
+            classes = inspect.getmembers(m, inspect.isclass)
+            type_class = None
+            for _, cl in classes:
+                if cl != PythonType and issubclass(cl, PythonType):
+                    if type_class:
+                        raise cdist.Error("Only one python type class is "
+                                          "supported, but at least two "
+                                          "found: {}".format((type_class,
+                                                              cl, )))
+                    else:
+                        type_class = cl
+            env = self.env_py_type_manifest(cdist_object)
+            type_obj = type_class(env=env, cdist_object=cdist_object,
+                                  local=self.local, remote=None,
+                                  message_prefix=message_prefix)
+            if self.local.save_output_streams:
+                which = 'manifest'
+                stderr_path = os.path.join(cdist_object.stderr_path, which)
+                stdout_path = os.path.join(cdist_object.stdout_path, which)
+                with open(stderr_path, 'a+') as stderr, \
+                        open(stdout_path, 'a+') as stdout:
+                    self._process_py_type_manifest_entries(
+                        type_obj, env, stdout=stdout, stderr=stderr)
+            else:
+                self._process_py_type_manifest_entries(type_obj, env)
+
+    def _process_py_type_manifest_entries(self, type_obj, env, stdout=None,
+                                          stderr=None):
+        if hasattr(type_obj, 'manifest') and \
+                inspect.ismethod(type_obj.manifest):
+            for entry in type_obj.manifest(stdout=stdout, stderr=stderr):
+                if not isinstance(entry, ManifestEntry):
+                    raise TypeError("Manifest entry must be of "
+                                    "type ManifestEntry")
+                kwargs = {
+                    'argv': entry.cmd_line(),
+                    'env': env,
+                }
+                if entry.stdin:
+                    kwargs['stdin'] = entry.stdin
+                emulator = cdist.emulator.Emulator(**kwargs)
+                emulator.run()

cdist/core/python_type.py

@@ -0,0 +1,160 @@
+import logging
+import os
+import io
+import sys
+import re
+from cdist import message, Error
+
+
+class PythonType:
+    def __init__(self, env, cdist_object, local, remote, message_prefix=None):
+        self.env = env
+        self.cdist_object = cdist_object
+        self.object_id = cdist_object.object_id
+        self.object_name = cdist_object.name
+        self.cdist_type = cdist_object.cdist_type
+        self.local = local
+        self.remote = remote
+        self.object_path = cdist_object.absolute_path
+        self.type_path = cdist_object.cdist_type.absolute_path
+        self.explorer_path = os.path.join(self.object_path, 'explorer')
+        self.parameters = cdist_object.parameters
+        self.stdin_path = os.path.join(self.object_path, 'stdin')
+        self.log = logging.getLogger(
+            self.local.target_host[0] + ':' + self.object_name)
+
+        self.message_prefix = message_prefix
+        self.message = None
+
+    def get_parameter(self, name):
+        return self.parameters.get(name)
+
+    def get_explorer_file(self, name):
+        path = os.path.join(self.explorer_path, name)
+        return path
+
+    def get_explorer(self, name):
+        path = self.get_explorer_file(name)
+        with open(path, 'r') as f:
+            value = f.read()
+            if value:
+                value = value.strip()
+            return value
+
+    def run_local(self, command, env=None):
+        rv = self.local.run(command, env=env, return_output=True)
+        if rv:
+            rv = rv.rstrip('\n')
+        return rv
+
+    def run_remote(self, command, env=None):
+        rv = self.remote.run(command, env=env, return_output=True)
+        if rv:
+            rv = rv.rstrip('\n')
+        return rv
+
+    def transfer(self, source, destination):
+        self.remote.transfer(source, destination)
+
+    def die(self, msg):
+        raise Error("{}: {}".format(self.cdist_object, msg))
+
+    def type_manifest(self):
+        pass
+
+    def type_gencode(self):
+        pass
+
+    def manifest(self, stdout=None, stderr=None):
+        try:
+            if self.message_prefix:
+                self.message = message.Message(self.message_prefix,
+                                               self.local.messages_path)
+                self.env.update(self.message.env)
+            if stdout is not None:
+                stdout_save = sys.stdout
+                sys.stdout = stdout
+            if stderr is not None:
+                stderr_save = sys.stderr
+                sys.stderr = stderr
+            yield from self.type_manifest()
+        finally:
+            if self.message:
+                self.message.merge_messages()
+            if stdout is not None:
+                sys.stdout = stdout_save
+            if stderr is not None:
+                sys.stderr = stderr_save
+
+    def run(self, stdout=None, stderr=None):
+        try:
+            if self.message_prefix:
+                self.message = message.Message(self.message_prefix,
+                                               self.local.messages_path)
+            if stdout is not None:
+                stdout_save = sys.stdout
+                sys.stdout = stdout
+            if stderr is not None:
+                stderr_save = sys.stderr
+                sys.stderr = stderr
+            return self.type_gencode()
+        finally:
+            if self.message:
+                self.message.merge_messages()
+            if stdout is not None:
+                sys.stdout = stdout_save
+            if stderr is not None:
+                sys.stderr = stderr_save
+
+    def send_message(self, msg):
+        if self.message:
+            with open(self.message.messages_out, 'a') as f:
+                print(msg, file=f)
+
+    def receive_message(self, pattern):
+        if self.message:
+            with open(self.message.messages_in, 'r') as f:
+                for line in f:
+                    match = re.search(pattern, line)
+                    if match:
+                        return match
+        return None
+
+
+class ManifestEntry:
+    def __init__(self, name, stdin=None, parameters=None):
+        self.name = name
+        if parameters is None:
+            self.parameters = {}
+        else:
+            self.parameters = parameters
+        self.set_stdin(stdin)
+
+    def set_stdin(self, value):
+        # If file-like object then read its value.
+        if value is not None and isinstance(value, io.IOBase):
+            value = value.read()
+
+        # Convert to bytes file-like object.
+        if value is None:
+            self.stdin = None
+        elif isinstance(value, str):
+            self.stdin = io.BytesIO(value.encode('utf-8'))
+        elif isinstance(value, bytes) or isinstance(value, bytearray):
+            self.stdin = io.BytesIO(value)
+        else:
+            raise TypeError("value must be str, bytes, bytearray, file-like "
+                            "object or None")
+
+    def cmd_line(self):
+        argv = [self.name, ]
+        for param in self.parameters:
+            argv.append(param)
+            val = self.parameters[param]
+            if val:
+                argv.append(val)
+        return argv
+
+    def __repr__(self):
+        return '<ManifestEntry name={}, parameters={}, stdin={}>'.format(
+            self.name, self.parameters, self.stdin)

docs/dev/python-types/benchmark

@@ -0,0 +1,376 @@
+# sh type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125154.045799] 185.203.112.26: Starting configuration run
+INFO: [20181121125237.029892] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121125239.881529] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121125243.265672] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121125246.929903] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121125251.811258] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121125257.784416] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121125302.686275] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121125306.394030] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121125308.610023] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121125310.868538] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121125313.017972] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121125315.201342] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121125317.333055] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121125319.463929] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121125321.595410] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121125323.689697] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121125325.768283] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121125327.814793] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121125329.873073] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121125331.953886] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121125334.118290] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121125336.390849] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121125338.576698] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121125340.819044] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121125343.680419] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121125346.044907] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121125348.179574] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121125350.314970] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121125352.447394] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121125354.586637] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121125356.722699] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121125358.883538] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121125401.020967] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121125403.160146] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121125405.289048] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121125407.423994] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121125409.530135] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121125411.659683] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121125413.786177] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121125415.919152] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121125418.051496] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121125420.204577] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121125422.339697] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121125424.450966] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121125426.487831] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121125428.585516] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121125430.749002] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121125432.865290] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121125435.004009] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121125437.228566] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds
+
+# sh type, files exist at remote but content changes
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125529.952800] 185.203.112.26: Starting configuration run
+INFO: [20181121125541.175180] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121125543.219561] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121125545.116073] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121125547.011359] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121125548.916288] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121125550.821351] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121125552.723887] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121125554.635662] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121125556.568639] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121125558.508852] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121125600.464475] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121125602.429261] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121125604.428942] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121125606.442193] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121125608.474473] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121125610.535252] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121125612.609560] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121125614.708507] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121125616.824721] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121125618.924521] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121125621.007543] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121125623.133204] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121125625.333471] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121125627.396334] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121125629.526492] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121125631.628454] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121125633.743142] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121125635.952547] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121125637.986746] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121125640.020415] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121125642.081373] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121125644.174744] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121125646.286532] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121125648.396447] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121125650.460107] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121125652.557125] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121125654.667456] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121125656.746960] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121125658.854229] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121125700.968145] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121125703.109376] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121125705.318163] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121125707.440575] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121125709.551261] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121125711.657753] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121125713.774819] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121125715.887428] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121125717.995104] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121125720.110196] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121125722.232932] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds
+
+# py type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121125812.034197] 185.203.112.26: Starting configuration run
+INFO: [20181121125823.927353] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121125825.715361] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121125827.511296] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121125829.293455] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121125831.086696] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121125832.867582] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121125834.652511] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121125836.450393] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121125838.255152] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121125840.065808] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121125841.889049] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121125843.719280] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121125845.560165] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121125847.416138] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121125849.289851] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121125851.180203] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121125853.074978] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121125855.086107] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121125857.041100] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121125859.025581] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121125901.072067] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121125903.026711] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121125904.994824] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121125906.956296] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121125908.929231] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121125910.882672] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121125912.839834] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121125914.789904] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121125916.743930] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121125918.698258] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121125920.657118] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121125922.618898] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121125924.567847] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121125926.524617] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121125928.396400] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121125930.209237] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121125931.998377] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121125933.786883] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121125935.579348] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121125937.366197] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121125939.155643] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121125941.052837] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121125942.953670] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121125944.781567] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121125946.622485] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121125948.470701] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121125950.356949] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121125952.232014] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121125954.128887] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121125956.037541] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds
+
+# py type, files exist at remote but content changes
+echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130056.484643] 185.203.112.26: Starting configuration run
+INFO: [20181121130108.545059] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121130110.339217] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121130112.136448] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121130113.923820] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121130115.715667] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121130117.508696] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121130119.300839] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121130124.296312] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121130131.109195] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121130133.303817] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121130136.396440] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121130138.443128] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121130140.462868] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121130142.476196] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121130145.937900] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121130148.013672] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121130150.042588] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121130152.050793] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121130154.083089] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121130156.100091] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121130158.103005] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121130200.188390] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121130202.197574] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121130205.269102] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121130208.457011] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121130211.574321] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121130213.719894] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121130215.762977] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121130217.778624] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121130219.840477] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121130221.852389] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121130223.850898] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121130225.858812] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121130227.855295] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121130229.952673] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121130231.956904] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121130233.961954] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121130236.012158] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121130238.024422] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121130241.238800] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121130243.463237] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121130245.610314] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121130247.661385] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121130250.399845] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121130252.832133] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121130254.955658] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121130257.039587] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121130259.178847] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121130301.357922] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121130303.356299] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds
+
+
+
+# init test file content
+head -c 102400 /dev/random > /tmp/test.file
+
+# sh type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130612.519698] 185.203.112.26: Starting configuration run
+INFO: [20181121130624.219344] 185.203.112.26: Processing __file/root/foo0.bin
+INFO: [20181121130626.980052] 185.203.112.26: Processing __file/root/foo1.bin
+INFO: [20181121130631.200669] 185.203.112.26: Processing __file/root/foo2.bin
+INFO: [20181121130642.790229] 185.203.112.26: Processing __file/root/foo3.bin
+INFO: [20181121130646.565599] 185.203.112.26: Processing __file/root/foo4.bin
+INFO: [20181121130648.724875] 185.203.112.26: Processing __file/root/foo5.bin
+INFO: [20181121130651.464686] 185.203.112.26: Processing __file/root/foo6.bin
+INFO: [20181121130653.639581] 185.203.112.26: Processing __file/root/foo7.bin
+INFO: [20181121130655.773987] 185.203.112.26: Processing __file/root/foo8.bin
+INFO: [20181121130657.933136] 185.203.112.26: Processing __file/root/foo9.bin
+INFO: [20181121130700.065158] 185.203.112.26: Processing __file/root/foo10.bin
+INFO: [20181121130702.216456] 185.203.112.26: Processing __file/root/foo11.bin
+INFO: [20181121130704.429030] 185.203.112.26: Processing __file/root/foo12.bin
+INFO: [20181121130706.562114] 185.203.112.26: Processing __file/root/foo13.bin
+INFO: [20181121130708.696584] 185.203.112.26: Processing __file/root/foo14.bin
+INFO: [20181121130710.830002] 185.203.112.26: Processing __file/root/foo15.bin
+INFO: [20181121130712.966631] 185.203.112.26: Processing __file/root/foo16.bin
+INFO: [20181121130715.151833] 185.203.112.26: Processing __file/root/foo17.bin
+INFO: [20181121130717.355196] 185.203.112.26: Processing __file/root/foo18.bin
+INFO: [20181121130719.486316] 185.203.112.26: Processing __file/root/foo19.bin
+INFO: [20181121130721.619933] 185.203.112.26: Processing __file/root/foo20.bin
+INFO: [20181121130723.786670] 185.203.112.26: Processing __file/root/foo21.bin
+INFO: [20181121130725.924736] 185.203.112.26: Processing __file/root/foo22.bin
+INFO: [20181121130728.060224] 185.203.112.26: Processing __file/root/foo23.bin
+INFO: [20181121130730.178729] 185.203.112.26: Processing __file/root/foo24.bin
+INFO: [20181121130732.309264] 185.203.112.26: Processing __file/root/foo25.bin
+INFO: [20181121130734.479895] 185.203.112.26: Processing __file/root/foo26.bin
+INFO: [20181121130736.653085] 185.203.112.26: Processing __file/root/foo27.bin
+INFO: [20181121130738.814291] 185.203.112.26: Processing __file/root/foo28.bin
+INFO: [20181121130741.029646] 185.203.112.26: Processing __file/root/foo29.bin
+INFO: [20181121130743.128717] 185.203.112.26: Processing __file/root/foo30.bin
+INFO: [20181121130745.233272] 185.203.112.26: Processing __file/root/foo31.bin
+INFO: [20181121130747.364681] 185.203.112.26: Processing __file/root/foo32.bin
+INFO: [20181121130749.491793] 185.203.112.26: Processing __file/root/foo33.bin
+INFO: [20181121130751.620492] 185.203.112.26: Processing __file/root/foo34.bin
+INFO: [20181121130753.743519] 185.203.112.26: Processing __file/root/foo35.bin
+INFO: [20181121130755.862169] 185.203.112.26: Processing __file/root/foo36.bin
+INFO: [20181121130758.000172] 185.203.112.26: Processing __file/root/foo37.bin
+INFO: [20181121130800.090405] 185.203.112.26: Processing __file/root/foo38.bin
+INFO: [20181121130802.211849] 185.203.112.26: Processing __file/root/foo39.bin
+INFO: [20181121130804.356363] 185.203.112.26: Processing __file/root/foo40.bin
+INFO: [20181121130806.548412] 185.203.112.26: Processing __file/root/foo41.bin
+INFO: [20181121130808.671279] 185.203.112.26: Processing __file/root/foo42.bin
+INFO: [20181121130810.752813] 185.203.112.26: Processing __file/root/foo43.bin
+INFO: [20181121130812.844502] 185.203.112.26: Processing __file/root/foo44.bin
+INFO: [20181121130814.950501] 185.203.112.26: Processing __file/root/foo45.bin
+INFO: [20181121130817.040587] 185.203.112.26: Processing __file/root/foo46.bin
+INFO: [20181121130819.175850] 185.203.112.26: Processing __file/root/foo47.bin
+INFO: [20181121130821.332900] 185.203.112.26: Processing __file/root/foo48.bin
+INFO: [20181121130823.543119] 185.203.112.26: Processing __file/root/foo49.bin
+INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds
+
+# sh type, files exist at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121130854.980007] 185.203.112.26: Starting configuration run
+INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds
+
+# py type, no file at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121131110.179480] 185.203.112.26: Starting configuration run
+INFO: [20181121131122.086849] 185.203.112.26: Processing __file_py/root/foo0.bin
+INFO: [20181121131123.876029] 185.203.112.26: Processing __file_py/root/foo1.bin
+INFO: [20181121131125.668664] 185.203.112.26: Processing __file_py/root/foo2.bin
+INFO: [20181121131127.460721] 185.203.112.26: Processing __file_py/root/foo3.bin
+INFO: [20181121131129.591229] 185.203.112.26: Processing __file_py/root/foo4.bin
+INFO: [20181121131131.390379] 185.203.112.26: Processing __file_py/root/foo5.bin
+INFO: [20181121131133.195275] 185.203.112.26: Processing __file_py/root/foo6.bin
+INFO: [20181121131135.006282] 185.203.112.26: Processing __file_py/root/foo7.bin
+INFO: [20181121131136.834448] 185.203.112.26: Processing __file_py/root/foo8.bin
+INFO: [20181121131138.659301] 185.203.112.26: Processing __file_py/root/foo9.bin
+INFO: [20181121131140.496856] 185.203.112.26: Processing __file_py/root/foo10.bin
+INFO: [20181121131142.367813] 185.203.112.26: Processing __file_py/root/foo11.bin
+INFO: [20181121131144.239817] 185.203.112.26: Processing __file_py/root/foo12.bin
+INFO: [20181121131146.133314] 185.203.112.26: Processing __file_py/root/foo13.bin
+INFO: [20181121131148.049380] 185.203.112.26: Processing __file_py/root/foo14.bin
+INFO: [20181121131149.974696] 185.203.112.26: Processing __file_py/root/foo15.bin
+INFO: [20181121131151.929083] 185.203.112.26: Processing __file_py/root/foo16.bin
+INFO: [20181121131153.923590] 185.203.112.26: Processing __file_py/root/foo17.bin
+INFO: [20181121131155.874910] 185.203.112.26: Processing __file_py/root/foo18.bin
+INFO: [20181121131157.857904] 185.203.112.26: Processing __file_py/root/foo19.bin
+INFO: [20181121131159.902006] 185.203.112.26: Processing __file_py/root/foo20.bin
+INFO: [20181121131201.859840] 185.203.112.26: Processing __file_py/root/foo21.bin
+INFO: [20181121131203.810875] 185.203.112.26: Processing __file_py/root/foo22.bin
+INFO: [20181121131205.763291] 185.203.112.26: Processing __file_py/root/foo23.bin
+INFO: [20181121131207.710932] 185.203.112.26: Processing __file_py/root/foo24.bin
+INFO: [20181121131209.658154] 185.203.112.26: Processing __file_py/root/foo25.bin
+INFO: [20181121131211.615374] 185.203.112.26: Processing __file_py/root/foo26.bin
+INFO: [20181121131213.569721] 185.203.112.26: Processing __file_py/root/foo27.bin
+INFO: [20181121131215.522624] 185.203.112.26: Processing __file_py/root/foo28.bin
+INFO: [20181121131217.471128] 185.203.112.26: Processing __file_py/root/foo29.bin
+INFO: [20181121131219.421712] 185.203.112.26: Processing __file_py/root/foo30.bin
+INFO: [20181121131221.375699] 185.203.112.26: Processing __file_py/root/foo31.bin
+INFO: [20181121131223.327672] 185.203.112.26: Processing __file_py/root/foo32.bin
+INFO: [20181121131225.281373] 185.203.112.26: Processing __file_py/root/foo33.bin
+INFO: [20181121131227.256711] 185.203.112.26: Processing __file_py/root/foo34.bin
+INFO: [20181121131229.209255] 185.203.112.26: Processing __file_py/root/foo35.bin
+INFO: [20181121131231.170170] 185.203.112.26: Processing __file_py/root/foo36.bin
+INFO: [20181121131233.123407] 185.203.112.26: Processing __file_py/root/foo37.bin
+INFO: [20181121131235.077713] 185.203.112.26: Processing __file_py/root/foo38.bin
+INFO: [20181121131237.017138] 185.203.112.26: Processing __file_py/root/foo39.bin
+INFO: [20181121131238.988189] 185.203.112.26: Processing __file_py/root/foo40.bin
+INFO: [20181121131241.026849] 185.203.112.26: Processing __file_py/root/foo41.bin
+INFO: [20181121131242.978335] 185.203.112.26: Processing __file_py/root/foo42.bin
+INFO: [20181121131244.934562] 185.203.112.26: Processing __file_py/root/foo43.bin
+INFO: [20181121131246.885320] 185.203.112.26: Processing __file_py/root/foo44.bin
+INFO: [20181121131248.835008] 185.203.112.26: Processing __file_py/root/foo45.bin
+INFO: [20181121131250.789727] 185.203.112.26: Processing __file_py/root/foo46.bin
+INFO: [20181121131252.738686] 185.203.112.26: Processing __file_py/root/foo47.bin
+INFO: [20181121131254.691465] 185.203.112.26: Processing __file_py/root/foo48.bin
+INFO: [20181121131256.640896] 185.203.112.26: Processing __file_py/root/foo49.bin
+INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds
+
+# py type, files exist at remote
+echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
+
+INFO: [20181121131327.054523] 185.203.112.26: Starting configuration run
+INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds
+
+
+# Summary
+
+# sh type, no file at remote
+INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds
+# py type, no file at remote
+INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds
+
+# sh type, files exist at remote but content changes
+INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds
+# py type, files exist at remote but content changes
+INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds
+
+
+# sh type, no file at remote
+INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds
+# py type, no file at remote
+INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds
+
+# sh type, files exist at remote
+INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds
+# py type, files exist at remote
+INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds

docs/dev/python-types/benchmark.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+
+if [ $# -eq 0 ]
+then
+    N=1
+else
+    N=$1
+fi
+
+manifest() {
+    bytes=$(echo "$1 * 1024" | bc)
+    echo "head -c ${bytes} /dev/random | __file$2 /root/foo$3.bin --source - --mode 0640 --owner root --group root"
+}
+
+verbosity="-vv" #"-vvv"
+i=0
+while [ "$i" -lt "$N" ]
+do
+    if [ "$N" -ne 1 ]
+    then
+        printf "iteration %d\\n" "$i"
+    fi
+    printf "shinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foo$i.bin;'
+    manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "pyinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foo$i.bin;'
+    manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "shinit present state...\\n"
+    manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    printf "pyinit present state...\\n"
+    manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
+
+    i=$((i + 1))
+done

docs/dev/python-types/conf/manifest/pyinit

@@ -0,0 +1,7 @@
+#for x in 1; do
+#    echo xxx${x} | __file_py /root/foobar${x} --source - --mode 0640 --owner root --group root;
+#done
+#__dummy_config_py test1
+
+echo xxx | __file_py /root/foobar --source - --mode 0640 --owner root --group root
+__dummy_config_py test1

docs/dev/python-types/conf/manifest/shinit

@@ -0,0 +1,7 @@
+#for x in 1; do
+#    echo xxx${x} | __file /root/foobar${x} --source - --mode 0640 --owner root --group root;
+#done
+#__dummy_config_sh test1
+
+echo xxx | __file /root/foobar --source - --mode 0640 --owner root --group root
+__dummy_config_sh test1

docs/dev/python-types/conf/type/__dummy_config/__init__.py

@@ -0,0 +1,52 @@
+import os
+import sys
+from cdist.core import PythonType, ManifestEntry
+
+
+class DummyConfig(PythonType):
+    def type_manifest(self):
+        print('dummy py manifest stdout')
+        print('dummy py manifest stderr', file=sys.stderr)
+        filepy = ManifestEntry(name='__file_py', stdin='dummy=py\n',
+                               parameters={
+                                    '/root/dummypy.conf': None,
+                                    '--mode': '0640',
+                                    '--owner': 'root',
+                                    '--group': 'root',
+                                    '--source': '-',
+                               })
+        self.log.info('Created manifest entry %s', filepy)
+        yield filepy
+
+        self_path = os.path.dirname(os.path.realpath(__file__))
+        conf_path = os.path.join(self_path, 'files', 'dummypy.conf')
+        filepy = ManifestEntry(name='__file_py',
+                               parameters={
+                                    '/root/dummypy2.conf': None,
+                                    '--mode': '0640',
+                                    '--owner': 'root',
+                                    '--group': 'root',
+                                    '--source': conf_path,
+                               })
+        yield filepy
+
+        self_path = os.path.dirname(os.path.realpath(__file__))
+        conf_path = os.path.join(self_path, 'files', 'dummysh.conf')
+        with open(conf_path, 'r') as f:
+            filepy = ManifestEntry(name='__file', stdin=f,
+                                   parameters={
+                                        '/root/dummysh.conf': None,
+                                        '--mode': '0600',
+                                        '--owner': 'root',
+                                        '--group': 'root',
+                                        '--source': '-',
+                                   })
+            yield filepy
+
+    def type_gencode(self):
+        print('__dummy_config test stdout')
+        print('__dummy_config test stderr', file=sys.stderr)
+        pattern = "__file_py/root/dummypy2.conf:chgrp 'root'"
+        match = self.receive_message(pattern)
+        print('Received message:', match.string if match else None)
+        return None

docs/dev/python-types/conf/type/__dummy_config/files/dummypy.conf

@@ -0,0 +1 @@
+dummy=py2

docs/dev/python-types/conf/type/__dummy_config/files/dummysh.conf

@@ -0,0 +1 @@
+dummy=sh

docs/dev/python-types/conf/type/__dummy_config_py/__init__.py

@@ -0,0 +1,30 @@
+import os
+import sys
+from cdist.core import PythonType, ManifestEntry
+
+
+class DummyConfig(PythonType):
+    def type_manifest(self):
+        print('dummy manifest stdout')
+        print('dummy manifest stderr\n', file=sys.stderr)
+        filepy = ManifestEntry(name='__file_py', stdin='dummy=1\n',
+                               parameters={
+                                    '/root/dummy1.conf': None,
+                                    '--mode': '0640',
+                                    '--owner': 'root',
+                                    '--group': 'root',
+                                    '--source': '-',
+                               })
+        yield filepy
+
+        self_path = os.path.dirname(os.path.realpath(__file__))
+        conf_path = os.path.join(self_path, 'files', 'dummy.conf')
+        filepy = ManifestEntry(name='__file_py',
+                               parameters={
+                                    '/root/dummy2.conf': None,
+                                    '--mode': '0600',
+                                    '--owner': 'root',
+                                    '--group': 'root',
+                                    '--source': conf_path,
+                               })
+        yield filepy

docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf

@@ -0,0 +1 @@
+dummy=2

docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf

@@ -0,0 +1 @@
+dummy=2

docs/dev/python-types/conf/type/__dummy_config_sh/manifest

@@ -0,0 +1,6 @@
+printf 'dummy manifest stdout\n'
+printf 'dummy manifest stderr\n' >&2
+
+printf "dummy=1\\n" | __file /root/dummy1.conf --mode 0640 --owner root --group root --source -
+
+__file /root/dummy2.conf --mode 0600 --owner root --group root --source  "$__type/files/dummy.conf"

docs/dev/python-types/test.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+env | grep CDIST_PATH
+
+for streams in ' ' '-S'
+do
+    for x in sh py
+    do
+        printf "[%s] Removing old foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*'
+        printf "[%s] Listing foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'ls foobar* dummy*'
+        printf "[%s] Running cdist config, streams: %s\\n" "$x" "$streams"
+        printf -- "----------------\\n"
+        ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init  -- ${TARGET_HOST}
+        printf "[%s] Listing foobar* files\\n" "$x"
+        printf -- "----------------\\n"
+        ssh root@${TARGET_HOST} 'ls foobar* dummy*'
+        ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init  -- ${TARGET_HOST}
+    done
+done

docs/dev/python-types/timeit.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# Addapt to your env.
+CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
+export CDIST_PATH
+TARGET_HOST=185.203.112.26
+
+if [ $# -eq 0 ]
+then
+    N=1
+else
+    N=$1
+fi
+
+i=0
+while [ "$i" -lt "$N" ]
+do
+    if [ "$N" -ne 1 ]
+    then
+        printf "iteration %d\\n" "$i"
+    fi
+    printf "shinit clean state...\\n"
+    ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*;'
+
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST}
+    printf "pyinit clean state...\\n"
+    ssh root@$${TARGET_HOST} 'rm foobar*; rm dummy*;'
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST}
+
+    printf "shinit present state...\\n"
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST}
+
+    printf "pyinit present state...\\n"
+    time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST}
+    i=$((i + 1))
+done

scripts/cdist

@@ -60,7 +60,7 @@ def commandline():
 
 
 if __name__ == "__main__":
-    cdistpythonversion = '3.2'
+    cdistpythonversion = '3.5'
     if sys.version < cdistpythonversion:
         print('Python >= {} is required on the source host.'.format(
                 cdistpythonversion), file=sys.stderr)