Adapt order dependency impl to python types impl

cdist/__init__.py

@@ -47,6 +47,9 @@ REMOTE_COPY = "scp -o User=root -q"
 REMOTE_EXEC = "ssh -o User=root"
 REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
 
+ORDER_DEP_STATE_NAME = 'order_dep_state'
+TYPEORDER_DEP_NAME = 'typeorder_dep'
+
 
 class Error(Exception):
     """Base exception class for this project"""

cdist/argparse.py

@@ -5,11 +5,12 @@ import logging
 import collections
 import functools
 import cdist.configuration
+import cdist.trigger
 import cdist.preos
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', 'inventory', ))
+BETA_COMMANDS = set(('install', 'inventory', 'trigger', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
     'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@@ -436,6 +437,28 @@ def get_parsers():
                   ' should be POSIX compatible shell.'))
     parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
 
+    # Trigger
+    parser['trigger'] = parser['sub'].add_parser(
+            'trigger', parents=[parser['loglevel'],
+                                parser['beta'],
+                                parser['common'],
+                                parser['config_main']])
+    parser['trigger'].add_argument(
+            '-D', '--directory', action='store', required=False,
+            help=('Where to create local files'))
+    parser['trigger'].add_argument(
+            '-H', '--http-port', action='store', default=3000, required=False,
+            help=('Create trigger listener via http on specified port'))
+    parser['trigger'].add_argument(
+            '--ipv6', default=False,
+            help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
+            action='store_true')
+    parser['trigger'].add_argument(
+            '-O', '--source', action='store', required=False,
+            help=('Which file to copy for creation'))
+
+    parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
+
     for p in parser:
         parser[p].epilog = EPILOG
 

cdist/conf/type/__cdist_preos_trigger/gencode-remote

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+os=$(cat "$__global/explorer/os")
+
+case "$os" in
+    devuan)
+        echo "update-rc.d cdist-preos-trigger defaults > /dev/null"
+    ;;
+    *)
+    ;;
+esac
+

cdist/conf/type/__cdist_preos_trigger/man.rst

@@ -0,0 +1,45 @@
+cdist-type__cdist_preos_trigger(7)
+==================================
+
+NAME
+----
+cdist-type__cdist_preos_trigger - configure cdist preos trigger
+
+
+DESCRIPTION
+-----------
+Create cdist PreOS trigger by creating systemd unit file that will be started
+at boot and will execute trigger command - connect to specified host and port.
+
+
+REQUIRED PARAMETERS
+-------------------
+trigger-command
+    Command that will be executed as a PreOS cdist trigger.
+
+
+OPTIONAL PARAMETERS
+-------------------
+None
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    # Configure default curl trigger for host cdist.ungleich.ch at port 80.
+    __cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80'
+
+
+AUTHORS
+-------
+Darko Poljak <darko.poljak--@--ungleich.ch>
+
+
+COPYING
+-------
+Copyright \(C) 2016 Darko Poljak. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__cdist_preos_trigger/manifest

@@ -0,0 +1,67 @@
+#!/bin/sh
+
+os="$(cat "$__global/explorer/os")"
+trigger_command=$(cat "$__object/parameter/trigger-command")
+
+case "$os" in
+    devuan)
+        __file /etc/init.d/cdist-preos-trigger --owner root \
+            --group root \
+            --mode 755 \
+            --source - << EOF
+#!/bin/sh
+# /etc/init.d/cdist-preos-trigger
+
+### BEGIN INIT INFO
+# Provides:          cdist-preos-trigger
+# Required-Start:    \$all
+# Required-Stop:
+# Default-Start:     2 3 4 5 S
+# Default-Stop:      0 1 6
+# Short-Description: Execute cdist preos trigger command
+# Description:       Execute cdist preos trigger commnad.
+### END INIT INFO
+
+case "\$1" in
+    start)
+        echo "Starting cdist-preos-trigger command"
+        ${trigger_command} &
+        ;;
+    stop)
+        # no-op
+        ;;
+    *)
+        echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}"
+        exit 1
+        ;;
+esac
+
+exit 0
+EOF
+    ;;
+    *)
+        __file /etc/systemd/system/cdist-preos-trigger.service --owner root \
+            --group root \
+            --mode 644 \
+            --source - << EOF
+[Unit]
+Description=preos trigger
+Wants=network-online.target
+After=network.target network-online.target
+
+[Service]
+Type=simple
+Restart=no
+# Broken systemd
+ExecStartPre=/bin/sleep 5
+ExecStart=${trigger_command}
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+        require="__file/etc/systemd/system/cdist-preos-trigger.service" \
+            __start_on_boot cdist-preos-trigger
+    ;;
+esac
+

cdist/conf/type/__cdist_preos_trigger/parameter/required

@@ -0,0 +1 @@
+trigger-command

cdist/conf/type/__file/__init__.py

@@ -0,0 +1,124 @@
+import os
+import re
+from cdist.core.pytypes import *
+import argparse
+
+
+class FileType(PythonType):
+    def get_attribute(self, stat_file, attribute, value_should):
+        if os.path.exists(stat_file):
+            if re.match('[0-9]', value_should):
+                index = 1
+            else:
+                index = 2
+            with open(stat_file, 'r') as f:
+                for line in f:
+                    if re.match(attribute + ":", line):
+                        fields = line.split()
+                        return fields[index]
+            return None
+
+    def set_attribute(self, attribute, value_should, destination):
+        cmd = {
+            'group': 'chgrp',
+            'owner': 'chown',
+            'mode': 'chmod',
+        }
+        self.send_message("{} '{}'".format(cmd[attribute], value_should))
+        return "{} '{}' '{}'".format(cmd[attribute], value_should, destination)
+
+    def type_manifest(self):
+        yield from ()
+
+    def type_gencode(self):
+        typeis = self.get_explorer('type')
+        state_should = self.get_parameter('state')
+
+        if state_should == 'exists' and typeis == 'file':
+            return
+
+        source = self.get_parameter('source')
+        if source == '-':
+            source = self.stdin_path
+        destination = '/' + self.object_id
+        if state_should == 'pre-exists':
+            if source is not None:
+                self.die('--source cannot be used with --state pre-exists')
+            if typeis == 'file':
+                return None
+            else:
+                self.die('File {} does not exist'.format(destination))
+
+        create_file = False
+        upload_file = False
+        set_attributes = False
+        fire_onchange = False
+        code = []
+        if state_should == 'present' or state_should == 'exists':
+            if source is None:
+                remote_stat = self.get_explorer('stat')
+                if not remote_stat:
+                    create_file = True
+            else:
+                if os.path.exists(source):
+                    if typeis == 'file':
+                        local_cksum = self.run_local(['cksum', source, ])
+                        local_cksum = local_cksum.split()[0]
+                        remote_cksum = self.get_explorer('cksum')
+                        remote_cksum = remote_cksum.split()[0]
+                        upload_file = local_cksum != remote_cksum
+                    else:
+                        upload_file = True
+                else:
+                    self.die('Source {} does not exist'.format(source))
+            if create_file or upload_file:
+                set_attributes = True
+                fire_onchange = True
+                tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination)
+                destination_upload = self.run_remote(
+                    ["mktemp", tempfile_template, ])
+                if upload_file:
+                    self.transfer(source, destination_upload)
+                code.append('rm -rf {}'.format(destination))
+                code.append('mv {} {}'.format(destination_upload, destination))
+
+        if state_should in ('present', 'exists', 'pre-exists', ):
+            for attribute in ('group', 'owner', 'mode', ):
+                if attribute in self.parameters:
+                    value_should = self.get_parameter(attribute)
+                    if attribute == 'mode':
+                        value_should = re.sub('^0', '', value_should)
+                    stat_file = self.get_explorer_file('stat')
+                    value_is = self.get_attribute(stat_file, attribute,
+                                                  value_should)
+                    if set_attributes or value_should != value_is:
+                        fire_onchange = True
+                        code.append(self.set_attribute(attribute,
+                                                       value_should,
+                                                       destination))
+        elif state_should == 'absent':
+            if typeis == 'file':
+                code.append('rm -f {}'.format(destination))
+                self.send_message('remove')
+                fire_onchange = True
+        else:
+            self.die('Unknown state {}'.format(state_should))
+
+        if fire_onchange:
+            onchange = self.get_parameter('onchange')
+            if onchange:
+                code.append(onchange)
+
+        return "\n".join(code)
+
+    def get_args_parser(self):
+        parser = argparse.ArgumentParser(add_help=False,
+                                         argument_default=argparse.SUPPRESS)
+        parser.add_argument('--state', dest='state', action='store',
+                            required=False, default='present')
+        for param in ('group', 'mode', 'owner', 'source', 'onchange'):
+            parser.add_argument('--' + param, dest=param, action='store',
+                                required=False, default=None)
+
+        parser.add_argument("object_id", nargs=1)
+        return parser

cdist/conf/type/__file_old/explorer/cksum

@@ -0,0 +1,34 @@
+#!/bin/sh
+#
+# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Retrieve the md5sum of a file to be created, if it is already existing.
+#
+
+destination="/$__object_id"
+
+if [ -e "$destination" ]; then
+    if [ -f  "$destination" ]; then
+        cksum < "$destination"
+    else
+        echo "NO REGULAR FILE"
+    fi
+else
+    echo "NO FILE FOUND, NO CHECKSUM CALCULATED."
+fi

cdist/conf/type/__file_old/explorer/stat

@@ -0,0 +1,88 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+# 2019 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+destination="/$__object_id"
+
+# nothing to work with, nothing we could do
+[ -e "$destination" ] || exit 0
+
+os=$("$__explorer/os")
+case "$os" in
+   "freebsd"|"netbsd"|"openbsd"|"macosx")
+      stat -f "type: %HT
+owner: %Du %Su
+group: %Dg %Sg
+mode: %Lp %Sp
+size: %Dz
+links: %Dl
+" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }'
+      ;;
+   alpine)
+       # busybox stat
+      stat -c "type: %F
+owner: %u %U
+group: %g %G
+mode: %a %A
+size: %s
+links: %h
+" "$destination"
+      ;;
+    solaris)
+        ls1="$( ls -ld "$destination" )"
+        ls2="$( ls -ldn "$destination" )"
+
+        if [ -f "$__object/parameter/mode" ]
+        then mode_should="$( cat "$__object/parameter/mode" )"
+        fi
+
+        # yes, it is ugly hack, but if you know better way...
+        if [ -z "$( find "$destination" -perm "$mode_should" )" ]
+        then octets=888
+        else octets="$( echo "$mode_should" | sed 's/^0//' )"
+        fi
+
+        case "$( echo "$ls1" | cut -c1-1 )" in
+            -) echo 'type: regular file' ;;
+            d) echo 'type: directory' ;;
+        esac
+
+        echo "owner: $( echo "$ls2" \
+            | awk '{print $3}' ) $( echo "$ls1" \
+                | awk '{print $3}' )"
+
+        echo "group: $( echo "$ls2" \
+            | awk '{print $4}' ) $( echo "$ls1" \
+                | awk '{print $4}' )"
+
+        echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
+        echo "size: $( echo "$ls1" | awk '{print $5}' )"
+        echo "links: $( echo "$ls1" | awk '{print $2}' )"
+    ;;
+   *)
+      stat --printf="type: %F
+owner: %u %U
+group: %g %G
+mode: %a %A
+size: %s
+links: %h
+" "$destination"
+      ;;
+esac

cdist/conf/type/__file_old/explorer/type

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# 2013 Steven Armstrong (steven-cdist armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+destination="/$__object_id"
+
+if [ ! -e "$destination" ]; then
+   echo none
+elif [ -h "$destination" ]; then
+   echo symlink
+elif [ -f "$destination" ]; then
+   echo file
+elif [ -d "$destination" ]; then
+   echo directory
+else
+   echo unknown
+fi

cdist/conf/type/__file_py

@@ -0,0 +1 @@
+__file

cdist/config.py

@@ -38,6 +38,7 @@ import cdist.hostsource
 import cdist.exec.local
 import cdist.exec.remote
 import cdist.util.ipaddr as ipaddr
+import cdist.util.python_type_util as pytype_util
 import cdist.configuration
 from cdist import core, inventory
 from cdist.util.remoteutil import inspect_ssh_mux_opts
@@ -90,13 +91,15 @@ class Config(object):
                 shutil.rmtree(path)
 
     def __init__(self, local, remote, dry_run=False, jobs=None,
-                 cleanup_cmds=None, remove_remote_files_dirs=False):
+                 cleanup_cmds=None, remove_remote_files_dirs=False,
+                 timestamp=False):
 
         self.local = local
         self.remote = remote
         self._open_logger()
         self.dry_run = dry_run
         self.jobs = jobs
+        self.timestamp = timestamp
         if cleanup_cmds:
             self.cleanup_cmds = cleanup_cmds
         else:
@@ -428,7 +431,8 @@ class Config(object):
                 cleanup_cmds.append(cleanup_cmd)
             c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs,
                     cleanup_cmds=cleanup_cmds,
-                    remove_remote_files_dirs=remove_remote_files_dirs)
+                    remove_remote_files_dirs=remove_remote_files_dirs,
+                    timestamp=args.timestamp)
             c.run()
             cls._remove_paths()
 
@@ -466,6 +470,7 @@ class Config(object):
             'dry' if self.dry_run else 'configuration'))
 
         self._init_files_dirs()
+        self.local.collect_python_types()
 
         self.explorer.run_global_explorers(self.local.global_explorer_out_path)
         try:
@@ -779,15 +784,41 @@ class Config(object):
                     args = [param, cdist_type.name]
                 self.log.warning(format, *args)
 
+    def _timeit(self, func, msg_prefix):
+        def wrapper_func(*args, **kwargs):
+            loglevel = self.log.getEffectiveLevel()
+            if loglevel >= logging.VERBOSE and self.timestamp:
+                start_time = time.time()
+                rv = func(*args, **kwargs)
+                end_time = time.time()
+                duration = end_time - start_time
+                self.log.verbose("%s duration: %.6f seconds",
+                                 msg_prefix, duration)
+            else:
+                rv = func(*args, **kwargs)
+            return rv
+        return wrapper_func
+
     def object_prepare(self, cdist_object, transfer_type_explorers=True):
         """Prepare object: Run type explorer + manifest"""
         self._handle_deprecation(cdist_object)
         self.log.verbose("Preparing object {}".format(cdist_object.name))
         self.log.verbose(
             "Running manifest and explorers for " + cdist_object.name)
-        self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
         try:
-            self.manifest.run_type_manifest(cdist_object)
+            self.log.verbose("Preparing object {}".format(cdist_object.name))
+            self.log.verbose(
+                    "Running manifest and explorers for " + cdist_object.name)
+            self.explorer.run_type_explorers(cdist_object,
+                                             transfer_type_explorers)
+            if pytype_util.is_python_type(cdist_object.cdist_type):
+                self._timeit(self.manifest.run_py_type_manifest,
+                             "Python type manifest for {}".format(
+                                 cdist_object.name))(cdist_object)
+            else:
+                self._timeit(self.manifest.run_type_manifest,
+                             "Type manifest for {}".format(
+                                 cdist_object.name))(cdist_object)
             self.log.trace("[ORDER_DEP] Removing order dep files for %s",
                            cdist_object)
             cdist_object.cleanup()
@@ -805,9 +836,21 @@ class Config(object):
 
             # Generate
             self.log.debug("Generating code for %s" % (cdist_object.name))
-            cdist_object.code_local = self.code.run_gencode_local(cdist_object)
-            cdist_object.code_remote = self.code.run_gencode_remote(
-                cdist_object)
+            if pytype_util.is_python_type(cdist_object.cdist_type):
+                cdist_object.code_local = ''
+                cdist_object.code_remote = self._timeit(
+                    self.code.run_py,
+                    "Python type generate code for {}".format(
+                        cdist_object.name))(cdist_object)
+            else:
+                cdist_object.code_local = self._timeit(
+                    self.code.run_gencode_local,
+                    "Type generate code local for {}".format(
+                        cdist_object.name))(cdist_object)
+                cdist_object.code_remote = self._timeit(
+                    self.code.run_gencode_remote,
+                    "Type generate code remote for {}".format(
+                        cdist_object.name))(cdist_object)
             if cdist_object.code_local or cdist_object.code_remote:
                 cdist_object.changed = True
 
@@ -818,12 +861,16 @@ class Config(object):
                 if cdist_object.code_local:
                     self.log.trace("Executing local code for %s"
                                    % (cdist_object.name))
-                    self.code.run_code_local(cdist_object)
+                    self._timeit(self.code.run_code_local,
+                                 "Type run code local for {}".format(
+                                     cdist_object.name))(cdist_object)
                 if cdist_object.code_remote:
                     self.log.trace("Executing remote code for %s"
                                    % (cdist_object.name))
                     self.code.transfer_code_remote(cdist_object)
-                    self.code.run_code_remote(cdist_object)
+                    self._timeit(self.code.run_code_remote,
+                                 "Type run code remote for {}".format(
+                                     cdist_object.name))(cdist_object)
 
             # Mark this object as done
             self.log.trace("Finishing run of " + cdist_object.name)

cdist/core/__init__.py

@@ -29,3 +29,4 @@ from cdist.core.manifest import Manifest
 from cdist.core.code import Code
 from cdist.core.util import listdir
 from cdist.core.util import log_level_env_var_val, log_level_name_env_var_val
+import cdist.core.pytypes

cdist/core/code.py

@@ -22,6 +22,8 @@
 #
 
 import os
+import inspect
+from cdist.core.pytypes import get_pytype_class
 from . import util
 
 
@@ -116,6 +118,27 @@ class Code(object):
         if dry_run:
             self.env['__cdist_dry_run'] = '1'
 
+    def run_py(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        type_class = get_pytype_class(cdist_type)
+        if type_class is not None:
+            env = os.environ.copy()
+            env.update(self.env)
+            message_prefix = cdist_object.name
+            type_obj = type_class(env=env, cdist_object=cdist_object,
+                                  local=self.local, remote=self.remote,
+                                  message_prefix=message_prefix)
+            if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run):
+                if self.local.save_output_streams:
+                    which = 'gencode-py'
+                    stderr_path = os.path.join(cdist_object.stderr_path, which)
+                    stdout_path = os.path.join(cdist_object.stdout_path, which)
+                    with open(stderr_path, 'a+') as stderr, \
+                            open(stdout_path, 'a+') as stdout:
+                        return type_obj.run(stdout=stdout, stderr=stderr)
+                else:
+                    return type_obj.run()
+
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.cdist_type
         script = os.path.join(self.local.type_path,

cdist/core/manifest.py

@@ -22,9 +22,12 @@
 
 import logging
 import os
-
+import inspect
 import cdist
+import cdist.emulator
 from . import util
+from cdist.core.pytypes import Command, get_pytype_class
+
 
 '''
 common:
@@ -97,9 +100,6 @@ class Manifest(object):
 
     """
 
-    ORDER_DEP_STATE_NAME = 'order_dep_state'
-    TYPEORDER_DEP_NAME = 'typeorder_dep'
-
     def __init__(self, target_host, local, dry_run=False):
         self.target_host = target_host
         self.local = local
@@ -224,5 +224,58 @@ class Manifest(object):
                 os.remove(os.path.join(self.local.base_path, fname))
             except FileNotFoundError:
                 pass
-        _rm_file(Manifest.ORDER_DEP_STATE_NAME)
-        _rm_file(Manifest.TYPEORDER_DEP_NAME)
+        _rm_file(cdist.ORDER_DEP_STATE_NAME)
+        _rm_file(cdist.TYPEORDER_DEP_NAME)
+
+    def env_py_type_manifest(self, cdist_object):
+        env = os.environ.copy()
+        env.update(self.env)
+        env.update({
+            '__cdist_object_marker': self.local.object_marker_name,
+            '__cdist_manifest': cdist_object.cdist_type,
+            '__manifest': self.local.manifest_path,
+            '__object': cdist_object.absolute_path,
+            '__object_id': cdist_object.object_id,
+            '__object_name': cdist_object.name,
+            '__type': cdist_object.cdist_type.absolute_path,
+        })
+
+        return env
+
+    def run_py_type_manifest(self, cdist_object):
+        cdist_type = cdist_object.cdist_type
+        type_class = get_pytype_class(cdist_type)
+        if type_class is not None:
+            self.log.verbose("Running python type manifest for object %s",
+                             cdist_object.name)
+            message_prefix = cdist_object.name
+            env = self.env_py_type_manifest(cdist_object)
+            type_obj = type_class(env=env, cdist_object=cdist_object,
+                                  local=self.local, remote=None,
+                                  message_prefix=message_prefix)
+            if self.local.save_output_streams:
+                which = 'manifest'
+                stderr_path = os.path.join(cdist_object.stderr_path, which)
+                stdout_path = os.path.join(cdist_object.stdout_path, which)
+                with open(stderr_path, 'a+') as stderr, \
+                        open(stdout_path, 'a+') as stdout:
+                    self._process_py_type_manifest_entries(
+                        type_obj, env, stdout=stdout, stderr=stderr)
+            else:
+                self._process_py_type_manifest_entries(type_obj, env)
+
+    def _process_py_type_manifest_entries(self, type_obj, env, stdout=None,
+                                          stderr=None):
+        if hasattr(type_obj, 'manifest') and \
+                inspect.ismethod(type_obj.manifest):
+            for cmd in type_obj.manifest(stdout=stdout, stderr=stderr):
+                if not isinstance(cmd, Command):
+                    raise TypeError("Manifest command must be of type Command")
+                kwargs = {
+                    'argv': cmd.cmd_line(),
+                    'env': env,
+                }
+                if cmd.stdin:
+                    kwargs['stdin'] = cmd.stdin
+                emulator = cdist.emulator.Emulator(**kwargs)
+                emulator.run()

cdist/core/pytypes.py

@@ -0,0 +1,190 @@
+import logging
+import os
+import io
+import sys
+import re
+from cdist import message, Error
+import importlib.util
+import inspect
+import cdist
+
+
+__all__ = ["PythonType", "Command", "command"]
+
+
+class PythonType:
+    def __init__(self, env, cdist_object, local, remote, message_prefix=None):
+        self.env = env
+        self.cdist_object = cdist_object
+        self.local = local
+        self.remote = remote
+        if self.cdist_object:
+            self.object_id = cdist_object.object_id
+            self.object_name = cdist_object.name
+            self.cdist_type = cdist_object.cdist_type
+            self.object_path = cdist_object.absolute_path
+            self.explorer_path = os.path.join(self.object_path, 'explorer')
+            self.type_path = cdist_object.cdist_type.absolute_path
+            self.parameters = cdist_object.parameters
+            self.stdin_path = os.path.join(self.object_path, 'stdin')
+        if self.local:
+            self.log = logging.getLogger(
+                self.local.target_host[0] + ':' + self.object_name)
+
+        self.message_prefix = message_prefix
+        self.message = None
+
+    def get_parameter(self, name):
+        return self.parameters.get(name)
+
+    def get_explorer_file(self, name):
+        path = os.path.join(self.explorer_path, name)
+        return path
+
+    def get_explorer(self, name):
+        path = self.get_explorer_file(name)
+        with open(path, 'r') as f:
+            value = f.read()
+            if value:
+                value = value.strip()
+            return value
+
+    def run_local(self, command, env=None):
+        rv = self.local.run(command, env=env, return_output=True)
+        if rv:
+            rv = rv.rstrip('\n')
+        return rv
+
+    def run_remote(self, command, env=None):
+        rv = self.remote.run(command, env=env, return_output=True)
+        if rv:
+            rv = rv.rstrip('\n')
+        return rv
+
+    def transfer(self, source, destination):
+        self.remote.transfer(source, destination)
+
+    def die(self, msg):
+        raise Error("{}: {}".format(self.cdist_object, msg))
+
+    def manifest(self, stdout=None, stderr=None):
+        try:
+            if self.message_prefix:
+                self.message = message.Message(self.message_prefix,
+                                               self.local.messages_path)
+                self.env.update(self.message.env)
+            if stdout is not None:
+                stdout_save = sys.stdout
+                sys.stdout = stdout
+            if stderr is not None:
+                stderr_save = sys.stderr
+                sys.stderr = stderr
+            yield from self.type_manifest()
+        finally:
+            if self.message:
+                self.message.merge_messages()
+            if stdout is not None:
+                sys.stdout = stdout_save
+            if stderr is not None:
+                sys.stderr = stderr_save
+
+    def run(self, stdout=None, stderr=None):
+        try:
+            if self.message_prefix:
+                self.message = message.Message(self.message_prefix,
+                                               self.local.messages_path)
+            if stdout is not None:
+                stdout_save = sys.stdout
+                sys.stdout = stdout
+            if stderr is not None:
+                stderr_save = sys.stderr
+                sys.stderr = stderr
+            return self.type_gencode()
+        finally:
+            if self.message:
+                self.message.merge_messages()
+            if stdout is not None:
+                sys.stdout = stdout_save
+            if stderr is not None:
+                sys.stderr = stderr_save
+
+    def send_message(self, msg):
+        if self.message:
+            with open(self.message.messages_out, 'a') as f:
+                print(msg, file=f)
+
+    def receive_message(self, pattern):
+        if self.message:
+            with open(self.message.messages_in, 'r') as f:
+                for line in f:
+                    match = re.search(pattern, line)
+                    if match:
+                        return match
+        return None
+
+    def get_args_parser(self):
+        pass
+
+    def type_manifest(self):
+        pass
+
+    def type_gencode(self):
+        pass
+
+
+class Command:
+    def __init__(self, name, *args, **kwargs):
+        self.name = name
+        self.args = args
+        self.kwargs = kwargs
+        self.stdin = None
+
+    def feed_stdin(self, value):
+        # If file-like object then read its value.
+        if value is not None and isinstance(value, io.IOBase):
+            value = value.read()
+
+        # Convert to bytes file-like object.
+        if value is None:
+            self.stdin = None
+        elif isinstance(value, str):
+            self.stdin = io.BytesIO(value.encode('utf-8'))
+        elif isinstance(value, bytes) or isinstance(value, bytearray):
+            self.stdin = io.BytesIO(value)
+        else:
+            raise TypeError("value must be str, bytes, bytearray, file-like "
+                            "object or None")
+        return self
+
+    def cmd_line(self):
+        argv = [self.name, ]
+        for param in self.args:
+            argv.append(param)
+        for key, value in self.kwargs.items():
+            argv.append("--{}".format(key))
+            argv.append(value)
+        return argv
+
+
+def command(name, *args, **kwargs):
+    return Command(name, *args, **kwargs)
+
+
+def get_pytype_class(cdist_type):
+    module_name = cdist_type.name
+    file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
+    type_class = None
+    if os.path.isfile(file_path):
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        m = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(m)
+        classes = inspect.getmembers(m, inspect.isclass)
+        for _, cl in classes:
+            if cl != PythonType and issubclass(cl, PythonType):
+                if type_class:
+                    raise cdist.Error(
+                        "Only one python type class is supported, but at least"
+                        " two found: {}".format((type_class, cl, )))
+                else:
+                    type_class = cl
+    return type_class

cdist/emulator.py

@@ -29,7 +29,9 @@ import sys
 import cdist
 from cdist import core
 from cdist import flock
-from cdist.core.manifest import Manifest
+import cdist.util.python_type_util as pytype_util
+from cdist.core.pytypes import get_pytype_class
+import inspect
 
 
 class MissingRequiredEnvironmentVariableError(cdist.Error):
@@ -84,9 +86,9 @@ class Emulator(object):
         self.typeorder_path = os.path.join(self.global_path, "typeorder")
 
         self.typeorder_dep_path = os.path.join(self.global_path,
-                                               Manifest.TYPEORDER_DEP_NAME)
+                                               cdist.TYPEORDER_DEP_NAME)
         self.order_dep_state_path = os.path.join(self.global_path,
-                                                 Manifest.ORDER_DEP_STATE_NAME)
+                                                 cdist.ORDER_DEP_STATE_NAME)
 
         self.type_name = os.path.basename(argv[0])
         self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
@@ -100,7 +102,28 @@ class Emulator(object):
     def run(self):
         """Emulate type commands (i.e. __file and co)"""
 
-        self.commandline()
+        args_parser = None
+        if pytype_util.is_python_type(self.cdist_type):
+            type_class = get_pytype_class(self.cdist_type)
+            if type_class is not None:
+                # We only need to call parse_args so we need plain instance.
+                type_obj = type_class(env=None, cdist_object=None, local=None,
+                                      remote=None)
+                if (hasattr(type_obj, 'get_args_parser') and
+                        inspect.ismethod(type_obj.get_args_parser)):
+                    args_parser = type_obj.get_args_parser()
+                    self.log.trace("Using python type argument parser")
+                    print("Using python type argument parser")
+            if args_parser is None:
+                # Fallback to classic way.
+                args_parser = self.get_args_parser()
+                self.log.trace("Fallback to classic argument parser")
+                print("Fallback to classic argument parser")
+        else:
+            args_parser = self.get_args_parser()
+            self.log.trace("Using emulator classic argument parser")
+            print("Using emulator classic argument parser")
+        self.commandline(args_parser)
         self.init_object()
 
         # locking for parallel execution
@@ -131,9 +154,7 @@ class Emulator(object):
 
         self.log = logging.getLogger(self.target_host[0])
 
-    def commandline(self):
-        """Parse command line"""
-
+    def get_args_parser(self):
         parser = argparse.ArgumentParser(add_help=False,
                                          argument_default=argparse.SUPPRESS)
 
@@ -165,8 +186,10 @@ class Emulator(object):
         # If not singleton support one positional parameter
         if not self.cdist_type.is_singleton:
             parser.add_argument("object_id", nargs=1)
+        return parser
 
-        # And finally parse/verify parameter
+    def commandline(self, parser):
+        """Parse command line"""
         self.args = parser.parse_args(self.argv[1:])
         self.log.trace('Args: %s' % self.args)
 

cdist/exec/local.py

@@ -35,6 +35,9 @@ import cdist
 import cdist.message
 from cdist import core
 import cdist.exec.util as util
+import cdist.util.python_type_util as pytype_util
+from cdist.core import pytypes
+import functools
 
 CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
 
@@ -398,3 +401,16 @@ class Local(object):
                 raise cdist.Error(
                         "Linking emulator from %s to %s failed: %s" % (
                             src, dst, e.__str__()))
+
+    def collect_python_types(self):
+        for cdist_type in core.CdistType.list_types(self.type_path):
+            if pytype_util.is_python_type(cdist_type):
+                self.log.trace("Detected python type %s, collecting it".format(
+                    cdist_type.name))
+                f = functools.partial(pytypes.command, cdist_type.name)
+                if cdist_type.name.startswith('__'):
+                    attr_name = cdist_type.name.replace('__', '', 1)
+                else:
+                    attr_name = cdist_type.name
+                setattr(pytypes, attr_name, f)
+                pytypes.__all__.append(attr_name)

cdist/preos/debootstrap/debootstrap.py

@@ -127,6 +127,12 @@ class Debian(object):
                             help="suite used for debootstrap, "
                                  "by default '{}'".format(defargs.suite),
                             dest='suite', default=defargs.suite)
+        parser.add_argument(
+            '-t', '--trigger-command',
+            help=("trigger command that will be added to cdist config; "
+                  "'__cdist_preos_trigger http ...' type is appended to "
+                  "initial manifest"),
+            dest='trigger_command')
         parser.add_argument(
             '-y', '--remote-copy',
             help=("remote copy that cdist config will use, by default "

cdist/preos/debootstrap/files/code

@@ -127,6 +127,13 @@ then
         exit 1
     fi
 
+    if [ "${trigger_command}" ]
+    then
+        trigger_line="__cdist_preos_trigger http --trigger-command '${trigger_command}'\n"
+    else
+        trigger_line=""
+    fi
+
     if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ]
     then
         i="$((keyfile_cnt - 1))"
@@ -174,7 +181,7 @@ then
     fi
     grub_lines="${grub_manifest_line}${grub_kern_params_line}"
 
-    printf "${ssh_auth_keys_line}${grub_lines}" \
+    printf "${trigger_line}${ssh_auth_keys_line}${grub_lines}" \
         | cat "${manifest}" - |\
         cdist config \
             ${cdist_params} -i - \

cdist/trigger.py

@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2016 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import ipaddress
+import logging
+import re
+import socket
+import http.server
+import os
+import socketserver
+import shutil
+
+import cdist.config
+import cdist.log
+import cdist.util.ipaddr as ipaddr
+
+
+class Trigger():
+    """cdist trigger handling"""
+
+    # Arguments that are only trigger specific
+    triggers_args = ["http_port", "ipv6", "directory", "source", ]
+
+    def __init__(self, http_port=None, dry_run=False, ipv6=False,
+                 directory=None, source=None, cdistargs=None):
+        self.dry_run = dry_run
+        self.http_port = int(http_port)
+        self.ipv6 = ipv6
+        self.args = cdistargs
+
+        self.directory = directory
+        self.source = source
+
+        log.debug("IPv6: %s", self.ipv6)
+
+    def run_httpd(self):
+        server_address = ('', self.http_port)
+
+        if self.ipv6:
+            httpdcls = HTTPServerV6
+        else:
+            httpdcls = HTTPServerV4
+        httpd = httpdcls(self.args, self.directory, self.source,
+                         server_address, TriggerHttp)
+
+        log.debug("Starting server at port %d", self.http_port)
+        if self.dry_run:
+            log.debug("Running in dry run mode")
+        httpd.serve_forever()
+
+    def run(self):
+        if self.http_port:
+            self.run_httpd()
+
+    @classmethod
+    def commandline(cls, args):
+        global log
+
+        # remove root logger default cdist handler and configure trigger's own
+        logging.getLogger().handlers = []
+        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s')
+
+        log = logging.getLogger("trigger")
+        ownargs = {}
+        for targ in cls.triggers_args:
+            arg = getattr(args, targ)
+            ownargs[targ] = arg
+
+            del arg
+
+        t = cls(dry_run=args.dry_run, cdistargs=args, **ownargs)
+        t.run()
+
+
+class TriggerHttp(http.server.BaseHTTPRequestHandler):
+    actions = {
+        "cdist": ["config", "install", ],
+        "file":  ["present", "absent", ],
+    }
+
+    def do_HEAD(self):
+        self.dispatch_request()
+
+    def do_POST(self):
+        self.dispatch_request()
+
+    def do_GET(self):
+        self.dispatch_request()
+
+    def _actions_regex(self):
+        regex = ["^/(?P<subsystem>", ]
+        regex.extend("|".join(self.actions.keys()))
+        regex.append(")/(?P<action>")
+        regex.extend("|".join("|".join(self.actions[x]) for x in self.actions))
+        regex.append(")/")
+
+        return "".join(regex)
+
+    def dispatch_request(self):
+        host = self.client_address[0]
+        code = 200
+        message = None
+
+        self.cdistargs = self.server.cdistargs
+
+        actions_regex = self._actions_regex()
+        m = re.match(actions_regex, self.path)
+
+        if m:
+            subsystem = m.group('subsystem')
+            action = m.group('action')
+            handler = getattr(self, "handler_" + subsystem)
+
+            if action not in self.actions[subsystem]:
+                code = 404
+        else:
+            code = 404
+
+        if code == 200:
+            log.debug("Calling {} -> {}".format(subsystem, action))
+            try:
+                handler(action, host)
+            except cdist.Error as e:
+                # cdist is not broken, cdist run is broken
+                code = 599  # use arbitrary unassigned error code
+                message = str(e)
+            except Exception as e: