add file support

Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>

cdist/config.py

@@ -130,13 +130,32 @@ class Config(object):
                 for host in source:
                     yield host
 
+    @staticmethod
+    def construct_remote_exec_copy_patterns(args):
+        # default remote cmd patterns
+        args.remote_exec_pattern = None
+        args.remote_copy_pattern = None
+
+        args_dict = vars(args)
+        # if remote-exec and/or remote-copy args are None then user
+        # didn't specify command line options nor env vars:
+        # inspect multiplexing options for default cdist.REMOTE_COPY/EXEC
+        if (args_dict['remote_copy'] is None or
+                args_dict['remote_exec'] is None):
+            mux_opts = inspect_ssh_mux_opts()
+            if args_dict['remote_exec'] is None:
+                args.remote_exec_pattern = cdist.REMOTE_EXEC + mux_opts
+            if args_dict['remote_copy'] is None:
+                args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
+
+
     @classmethod
     def commandline(cls, args):
         """Configure remote system"""
         import multiprocessing
 
         # FIXME: Refactor relict - remove later
-        log = logging.getLogger("cdist")
+        log = logging.getLogger(__name__)
 
         if args.manifest == '-' and args.hostfile == '-':
             raise cdist.Error(("Cannot read both, manifest and host file, "
@@ -166,32 +185,14 @@ class Config(object):
         failed_hosts = []
         time_start = time.time()
 
-        # default remote cmd patterns
+        cls.construct_remote_exec_copy_patterns(args)
-        args.remote_exec_pattern = None
+        base_root_path = cls.create_base_root_path(args.out_path)
-        args.remote_copy_pattern = None
-
-        args_dict = vars(args)
-        # if remote-exec and/or remote-copy args are None then user
-        # didn't specify command line options nor env vars:
-        # inspect multiplexing options for default cdist.REMOTE_COPY/EXEC
-        if (args_dict['remote_copy'] is None or
-                args_dict['remote_exec'] is None):
-            mux_opts = inspect_ssh_mux_opts()
-            if args_dict['remote_exec'] is None:
-                args.remote_exec_pattern = cdist.REMOTE_EXEC + mux_opts
-            if args_dict['remote_copy'] is None:
-                args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
-
-        if args.out_path:
-            base_root_path = args.out_path
-        else:
-            base_root_path = tempfile.mkdtemp()
 
         hostcnt = 0
         for host in itertools.chain(cls.hosts(args.host),
                                     cls.hosts(args.hostfile)):
-            hostdir = cdist.str_hash(host)
+            host_base_path, hostdir = cls.create_host_base_dirs(
-            host_base_path = os.path.join(base_root_path, hostdir)
+                host, base_root_path)
 
             log.debug("Base root path for target host \"{}\" is \"{}\"".format(
                 host, host_base_path))
@@ -227,6 +228,8 @@ class Config(object):
             raise cdist.Error("Failed to configure the following hosts: " +
                               " ".join(failed_hosts))
 
+
+
     @classmethod
     def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
         """Configure ONE system"""
@@ -317,6 +320,25 @@ class Config(object):
             else:
                 raise
 
+
+    @staticmethod
+    def create_base_root_path(out_path=None):
+        if out_path:
+            base_root_path = out_path
+        else:
+            base_root_path = tempfile.mkdtemp()
+
+        return base_root_path
+
+
+    @staticmethod
+    def create_host_base_dirs(host, base_root_path):
+        hostdir = cdist.str_hash(host)
+        host_base_path = os.path.join(base_root_path, hostdir)
+
+        return (host_base_path, hostdir)
+
+
     def run(self):
         """Do what is most often done: deploy & cleanup"""
         start_time = time.time()

cdist/trigger.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2016 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import ipaddress
+import logging
+import re
+import socket
+import http.server
+import os
+import socketserver
+import shutil
+
+import multiprocessing
+
+import cdist.config
+import cdist.install
+
+log = logging.getLogger(__name__)
+
+class Trigger():
+    """cdist trigger handling"""
+
+    # Arguments that are only trigger specific
+    triggers_args = [ "http_port", "ipv6", "directory", "source" ]
+
+
+    def __init__(self, http_port=None, dry_run=False, ipv6=False,
+                 directory=None, source=None, cdistargs=None):
+        self.log = logging.getLogger("trigger")
+        self.dry_run = dry_run
+        self.http_port = int(http_port)
+        self.ipv6 = ipv6
+        self.args = cdistargs
+
+        self.directory = directory
+        self.source = source
+
+        log.debug("IPv6: %s", self.ipv6)
+
+    def run_httpd(self):
+        server_address = ('', self.http_port)
+
+        if self.ipv6:
+            httpdcls = HTTPServerV6
+        else:
+            httpdcls = HTTPServerV4
+        httpd = httpdcls(self.args, self.directory, self.source, server_address, TriggerHttp)
+
+        log.debug("Starting server at port %d", self.http_port)
+        if self.dry_run:
+            log.debug("Running in dry run mode")
+        httpd.serve_forever()
+
+    def run(self):
+        if self.http_port:
+            self.run_httpd()
+
+    @classmethod
+    def commandline(cls, args):
+        http_port = args.http_port
+        ipv6 = args.ipv6
+
+        ownargs = {}
+        for targ in cls.triggers_args:
+            arg = getattr(args, targ)
+            ownargs[targ] = arg
+
+            del arg
+
+        t = cls(**ownargs, dry_run=args.dry_run, cdistargs=args)
+        t.run()
+
+class TriggerHttp(http.server.BaseHTTPRequestHandler):
+    actions = { "cdist": [ "config", "install" ],
+                "file":  [ "present", "absent" ]
+    }
+
+    def do_HEAD(self):
+        self.dispatch_request()
+
+    def do_POST(self):
+        self.dispatch_request()
+
+    def do_GET(self):
+        self.dispatch_request()
+
+    def dispatch_request(self):
+        host = self.client_address[0]
+        code = 200
+
+        self.cdistargs = self.server.cdistargs
+
+        # FIXME: generate regexp based on self.actions
+        m = re.match("^/(?P<subsystem>cdist|file)/(?P<action>present|absent|config|install)/", self.path)
+
+        if m:
+            subsystem = m.group('subsystem')
+            action = m.group('action')
+            handler = getattr(self, "handler_" + subsystem)
+
+            if not action in self.actions[subsystem]:
+                code = 404
+        else:
+            code = 404
+
+        if code == 200:
+            log.debug("Calling {} -> {}".format(subsystem, action))
+            handler(action, host)
+
+        self.send_response(code)
+        self.end_headers()
+
+    def handler_file(self, action, host):
+        if not self.server.directory or not self.server.source:
+            log.info("Cannot server file request: directory or source not setup")
+            return
+
+        try:
+            ipaddress.ip_address(host)
+        except ValueError:
+            log.error("Host is not a valid IP address - aborting")
+            return
+
+        dst = os.path.join(self.server.directory, host)
+
+        if action == "present":
+            shutil.copyfile(self.server.source, dst)
+        if action == "absent":
+            if os.path.exists(dst):
+                os.remove(dst)
+
+    def handler_cdist(self, action, host):
+        log.debug("Running cdist for %s in mode %s", host, mode)
+
+        if self.server.dry_run:
+            log.info("Dry run, skipping cdist execution")
+            return
+
+        cname = action.title()
+        module = getattr(cdist, action)
+        theclass = getattr(module, cname)
+
+        if hasattr(self.cdistargs, 'out_path'):
+            out_path = self.cdistargs.out_path
+        else:
+            out_path = None
+        host_base_path, hostdir = theclass.create_host_base_dirs(
+            host, theclass.create_base_root_path(out_path))
+        theclass.construct_remote_exec_copy_patterns(self.cdistargs)
+        log.debug("Executing cdist onehost with params: %s, %s, %s, %s, ",
+                  host, host_base_path, hostdir, self.cdistargs)
+        theclass.onehost(host, host_base_path, hostdir, self.cdistargs,
+                         parallel=False)
+
+
+class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer):
+    """
+    Server that listens to both IPv4 and IPv6 requests.
+    """
+    address_family = socket.AF_INET6
+
+    def __init__(self, cdistargs, directory, source, *args, **kwargs):
+        self.cdistargs = cdistargs
+        self.dry_run = cdistargs.dry_run
+        self.directory = directory
+        self.source = source
+
+        http.server.HTTPServer.__init__(self, *args, **kwargs)
+
+class HTTPServerV4(HTTPServerV6):
+    """
+    Server that listens to IPv4 requests.
+    """
+    address_family = socket.AF_INET

docs/src/man1/cdist.rst

@@ -27,6 +27,11 @@ SYNOPSIS
 
     cdist shell [-h] [-d] [-v] [-s SHELL]
 
+    cdist trigger [-h] [-d] [-v] [-b] [-c CONF_DIR] [-i MANIFEST]
+                  [-j [JOBS]] [-n] [-o OUT_PATH]
+                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
+                  [-6] [-H HTTP_PORT]
+
 
 DESCRIPTION
 -----------
@@ -148,6 +153,67 @@ usage. Its primary use is for debugging type parameters.
     Select shell to use, defaults to current shell. Used shell should
     be POSIX compatible shell.
 
+
+TRIGGER
+-------
+Start trigger (simple http server) that waits for connections. When host
+connects then it triggers config or install command, cdist config is then
+executed which configures/installs host.
+Request path recognies following formats:
+
+* :strong:`/config/.*` for config
+* :strong:`/install/.*` for install
+
+
+.. option:: -6, --ipv6
+
+    Listen to both IPv4 and IPv6 (instead of only IPv4)
+
+.. option:: -b, --enable-beta
+
+    Enable beta functionalities.
+
+.. option:: -c CONF_DIR, --conf-dir CONF_DIR
+
+    Add configuration directory (can be repeated, last one wins)
+
+.. option:: -d, --debug
+
+    Set log level to debug
+
+.. option:: -H HTTP_PORT, --http-port HTTP_PORT
+
+    Create trigger listener via http on specified port
+
+.. option:: -h, --help
+
+    show this help message and exit
+
+.. option:: -i MANIFEST, --initial-manifest MANIFEST
+
+    path to a cdist manifest or '-' to read from stdin.
+
+.. option:: -n, --dry-run
+
+    do not execute code
+
+.. option:: -o OUT_PATH, --out-dir OUT_PATH
+
+    directory to save cdist output in
+
+.. option:: --remote-copy REMOTE_COPY
+
+    Command to use for remote copy (should behave like scp)
+
+.. option:: --remote-exec REMOTE_EXEC
+
+    Command to use for remote execution (should behave like ssh)
+
+.. option:: -v, --verbose
+
+    Set log level to info, be more verbose
+
+
 FILES
 -----
 ~/.cdist
@@ -199,6 +265,11 @@ EXAMPLES
     # Install ikq05.ethz.ch with debug enabled
     % cdist install -d ikq05.ethz.ch
 
+    # Start trigger in verbose mode that will configure host using specified
+    # init manifest
+    % cdist trigger -b -v -i ~/.cdist/manifest/init-for-triggered
+
+
 ENVIRONMENT
 -----------
 TMPDIR, TEMP, TMP

scripts/cdist

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
-# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2010-2016 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
@@ -23,7 +23,7 @@
 
 
 # list of beta sub-commands
-BETA_COMMANDS = ['install', ]
+BETA_COMMANDS = ['install', 'trigger' ]
 # list of beta arguments for sub-commands
 BETA_ARGS = {
     'config': ['jobs', ],
@@ -69,6 +69,7 @@ def commandline():
     import cdist.config
     import cdist.install
     import cdist.shell
+    import cdist.trigger
     import shutil
     import os
     import multiprocessing
@@ -84,6 +85,12 @@ def commandline():
             '-v', '--verbose', help='Set log level to info, be more verbose',
             action='store_true', default=False)
 
+    parser['beta'] = argparse.ArgumentParser(add_help=False)
+    parser['beta'].add_argument(
+           '-b', '--enable-beta',
+           help=('Enable beta functionalities.'),
+           action='store_true', dest='beta', default=False)
+
     # Main subcommand parser
     parser['main'] = argparse.ArgumentParser(
             description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
@@ -98,20 +105,57 @@ def commandline():
             'banner', parents=[parser['loglevel']])
     parser['banner'].set_defaults(func=cdist.banner.banner)
 
-    # Config
+    parser['config_main'] = argparse.ArgumentParser(add_help=False)
-    parser['config'] = parser['sub'].add_parser(
+    parser['config_main'].add_argument(
-            'config', parents=[parser['loglevel']])
-    parser['config'].add_argument(
-            'host', nargs='*', help='host(s) to operate on')
-    parser['config'].add_argument(
-           '-b', '--enable-beta',
-           help=('Enable beta functionalities. Beta functionalities '
-                 'include the following options: -j/--jobs.'),
-           action='store_true', dest='beta', default=False)
-    parser['config'].add_argument(
             '-c', '--conf-dir',
             help=('Add configuration directory (can be repeated, '
                   'last one wins)'), action='append')
+    parser['config_main'].add_argument(
+           '-i', '--initial-manifest',
+           help='path to a cdist manifest or \'-\' to read from stdin.',
+           dest='manifest', required=False)
+    parser['config_main'].add_argument(
+           '-j', '--jobs', nargs='?', type=check_positive_int,
+           help=('specify the maximum number of parallel jobs, currently '
+                 'only global explorers are supported (currently in beta'),
+           action='store', dest='jobs',
+           const=multiprocessing.cpu_count())
+    parser['config_main'].add_argument(
+           '-n', '--dry-run',
+           help='do not execute code', action='store_true')
+    parser['config_main'].add_argument(
+           '-o', '--out-dir',
+           help='directory to save cdist output in', dest="out_path")
+
+    # remote-copy and remote-exec defaults are environment variables
+    # if set; if not then None - these will be futher handled after
+    # parsing to determine implementation default
+    parser['config_main'].add_argument(
+           '--remote-copy',
+           help='Command to use for remote copy (should behave like scp)',
+           action='store', dest='remote_copy',
+           default=os.environ.get('CDIST_REMOTE_COPY'))
+    parser['config_main'].add_argument(
+           '--remote-exec',
+           help=('Command to use for remote execution '
+                 '(should behave like ssh)'),
+           action='store', dest='remote_exec',
+           default=os.environ.get('CDIST_REMOTE_EXEC'))
+
+    # Config
+    parser['config'] = parser['sub'].add_parser(
+            'config', parents=[parser['loglevel'], parser['beta'],
+                               parser['config_main']])
+    parser['config'].add_argument(
+            'host', nargs='*', help='host(s) to operate on')
+    parser['config'].add_argument(
+           '-s', '--sequential',
+           help='operate on multiple hosts sequentially (default)',
+           action='store_false', dest='parallel')
+    parser['config'].add_argument(
+           '-p', '--parallel',
+           help='operate on multiple hosts in parallel',
+           action='store_true', dest='parallel')
     parser['config'].add_argument(
             '-f', '--file',
             help=('Read additional hosts to operate on from specified file '
@@ -119,44 +163,6 @@ def commandline():
                   'If no host or host file is specified then, by default, '
                   'read hosts from stdin.'),
             dest='hostfile', required=False)
-    parser['config'].add_argument(
-           '-i', '--initial-manifest',
-           help='Path to a cdist manifest or \'-\' to read from stdin.',
-           dest='manifest', required=False)
-    parser['config'].add_argument(
-           '-j', '--jobs', nargs='?', type=check_positive_int,
-           help=('Specify the maximum number of parallel jobs, currently '
-                 'only global explorers are supported (currently in beta'),
-           action='store', dest='jobs',
-           const=multiprocessing.cpu_count())
-    parser['config'].add_argument(
-           '-n', '--dry-run',
-           help='Do not execute code', action='store_true')
-    parser['config'].add_argument(
-           '-o', '--out-dir',
-           help='Directory to save cdist output in', dest="out_path")
-    parser['config'].add_argument(
-           '-p', '--parallel',
-           help='Operate on multiple hosts in parallel',
-           action='store_true', dest='parallel')
-    parser['config'].add_argument(
-           '-s', '--sequential',
-           help='Operate on multiple hosts sequentially (default)',
-           action='store_false', dest='parallel')
-    # remote-copy and remote-exec defaults are environment variables
-    # if set; if not then None - these will be futher handled after
-    # parsing to determine implementation default
-    parser['config'].add_argument(
-           '--remote-copy',
-           help='Command to use for remote copy (should behave like scp)',
-           action='store', dest='remote_copy',
-           default=os.environ.get('CDIST_REMOTE_COPY'))
-    parser['config'].add_argument(
-           '--remote-exec',
-           help=('Command to use for remote execution '
-                 '(should behave like ssh)'),
-           action='store', dest='remote_exec',
-           default=os.environ.get('CDIST_REMOTE_EXEC'))
     parser['config'].set_defaults(func=cdist.config.Config.commandline)
 
     # Install
@@ -173,6 +179,34 @@ def commandline():
                   ' should be POSIX compatible shell.'))
     parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
 
+    # Trigger
+    parser['trigger'] = parser['sub'].add_parser(
+            'trigger', parents=[parser['loglevel'],
+                                parser['beta'],
+                                parser['config_main']])
+    parser['trigger'].add_argument(
+            '-6', '--ipv6', default=False,
+            help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
+            action='store_true')
+    parser['trigger'].add_argument(
+            '-H', '--http-port', action='store', default=3000, required=False,
+            help=('Create trigger listener via http on specified port'))
+
+    parser['trigger'].add_argument(
+            '-D', '--directory', action='store', required=False,
+            help=('Where to create local files'))
+
+    parser['trigger'].add_argument(
+            '-S', '--source', action='store', required=False,
+            help=('Which file to copy for creation'))
+
+    parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
+
+    # Install
+    parser['install'] = parser['sub'].add_parser('install', add_help=False,
+                                                 parents=[parser['config']])
+    parser['install'].set_defaults(func=cdist.install.Install.commandline)
+
     for p in parser:
         parser[p].epilog = (
                 "Get cdist at http://www.nico.schottelius.org/software/cdist/")