Compare commits

...

20 commits

Author SHA1 Message Date
Nico Schottelius 9afb57412d add file support
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-11-27 17:11:34 +01:00
Nico Schottelius 4207124c52 add debug output when known url was triggered
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-11-27 16:16:34 +01:00
Nico Schottelius ff31dcada0 begin to add local file support
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-11-27 15:59:48 +01:00
Darko Poljak ab5d941802 Merge branch 'master' into feature/trigger 2016-11-01 08:17:50 +01:00
Darko Poljak 7f0ad6665b Add trigger to cdist man page. 2016-10-31 18:42:16 +01:00
Darko Poljak aac9906fff Remove redundant log.debug. 2016-10-31 08:09:38 +01:00
Darko Poljak 376a031a95 Add more log.debug. 2016-10-31 08:07:09 +01:00
Darko Poljak 94e8f0b2b2 Minor log.debug message fixes. 2016-10-31 08:05:57 +01:00
Darko Poljak 686a484b03 Fix dry_run and logging. 2016-10-31 07:59:03 +01:00
Darko Poljak 05cf492744 Implement dry_run option, add log.debug lines. 2016-10-31 07:51:27 +01:00
Darko Poljak 63dc9632d2 Make IPv4 default. 2016-10-30 20:37:24 +01:00
Darko Poljak 8c985fe2cb Add forking support. 2016-10-30 20:25:32 +01:00
Darko Poljak 297367390f Fix bug. 2016-10-30 17:21:08 +01:00
Darko Poljak d316089842 Continue with trigger. 2016-10-30 16:41:59 +01:00
Darko Poljak 7c169f2d0a Merge remote-tracking branch 'ungleich/feature/trigger' into feature/trigger 2016-10-30 15:09:26 +01:00
Nico Schottelius 535181435f update for darko
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-10-30 15:01:45 +01:00
Darko Poljak ca67533ce4 Fix comment typos. 2016-10-30 14:49:07 +01:00
Darko Poljak 579b8d5c72 Fix typo. 2016-10-30 14:48:10 +01:00
Nico Schottelius 92bb0803eb Finish base functionality for trigger execution 2016-10-25 12:38:13 +02:00
Nico Schottelius 7d027225bc begin integration of trigger handler
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-10-24 15:58:39 +02:00
4 changed files with 394 additions and 74 deletions

View file

@ -130,13 +130,32 @@ class Config(object):
for host in source:
yield host
@staticmethod
def construct_remote_exec_copy_patterns(args):
# default remote cmd patterns
args.remote_exec_pattern = None
args.remote_copy_pattern = None
args_dict = vars(args)
# if remote-exec and/or remote-copy args are None then user
# didn't specify command line options nor env vars:
# inspect multiplexing options for default cdist.REMOTE_COPY/EXEC
if (args_dict['remote_copy'] is None or
args_dict['remote_exec'] is None):
mux_opts = inspect_ssh_mux_opts()
if args_dict['remote_exec'] is None:
args.remote_exec_pattern = cdist.REMOTE_EXEC + mux_opts
if args_dict['remote_copy'] is None:
args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
@classmethod
def commandline(cls, args):
"""Configure remote system"""
import multiprocessing
# FIXME: Refactor relict - remove later
log = logging.getLogger("cdist")
log = logging.getLogger(__name__)
if args.manifest == '-' and args.hostfile == '-':
raise cdist.Error(("Cannot read both, manifest and host file, "
@ -166,32 +185,14 @@ class Config(object):
failed_hosts = []
time_start = time.time()
# default remote cmd patterns
args.remote_exec_pattern = None
args.remote_copy_pattern = None
args_dict = vars(args)
# if remote-exec and/or remote-copy args are None then user
# didn't specify command line options nor env vars:
# inspect multiplexing options for default cdist.REMOTE_COPY/EXEC
if (args_dict['remote_copy'] is None or
args_dict['remote_exec'] is None):
mux_opts = inspect_ssh_mux_opts()
if args_dict['remote_exec'] is None:
args.remote_exec_pattern = cdist.REMOTE_EXEC + mux_opts
if args_dict['remote_copy'] is None:
args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
if args.out_path:
base_root_path = args.out_path
else:
base_root_path = tempfile.mkdtemp()
cls.construct_remote_exec_copy_patterns(args)
base_root_path = cls.create_base_root_path(args.out_path)
hostcnt = 0
for host in itertools.chain(cls.hosts(args.host),
cls.hosts(args.hostfile)):
hostdir = cdist.str_hash(host)
host_base_path = os.path.join(base_root_path, hostdir)
host_base_path, hostdir = cls.create_host_base_dirs(
host, base_root_path)
log.debug("Base root path for target host \"{}\" is \"{}\"".format(
host, host_base_path))
@ -227,6 +228,8 @@ class Config(object):
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
@classmethod
def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
"""Configure ONE system"""
@ -317,6 +320,25 @@ class Config(object):
else:
raise
@staticmethod
def create_base_root_path(out_path=None):
if out_path:
base_root_path = out_path
else:
base_root_path = tempfile.mkdtemp()
return base_root_path
@staticmethod
def create_host_base_dirs(host, base_root_path):
hostdir = cdist.str_hash(host)
host_base_path = os.path.join(base_root_path, hostdir)
return (host_base_path, hostdir)
def run(self):
"""Do what is most often done: deploy & cleanup"""
start_time = time.time()

193
cdist/trigger.py Normal file
View file

@ -0,0 +1,193 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2016 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import ipaddress
import logging
import re
import socket
import http.server
import os
import socketserver
import shutil
import multiprocessing
import cdist.config
import cdist.install
log = logging.getLogger(__name__)
class Trigger():
"""cdist trigger handling"""
# Arguments that are only trigger specific
triggers_args = [ "http_port", "ipv6", "directory", "source" ]
def __init__(self, http_port=None, dry_run=False, ipv6=False,
directory=None, source=None, cdistargs=None):
self.log = logging.getLogger("trigger")
self.dry_run = dry_run
self.http_port = int(http_port)
self.ipv6 = ipv6
self.args = cdistargs
self.directory = directory
self.source = source
log.debug("IPv6: %s", self.ipv6)
def run_httpd(self):
server_address = ('', self.http_port)
if self.ipv6:
httpdcls = HTTPServerV6
else:
httpdcls = HTTPServerV4
httpd = httpdcls(self.args, self.directory, self.source, server_address, TriggerHttp)
log.debug("Starting server at port %d", self.http_port)
if self.dry_run:
log.debug("Running in dry run mode")
httpd.serve_forever()
def run(self):
if self.http_port:
self.run_httpd()
@classmethod
def commandline(cls, args):
http_port = args.http_port
ipv6 = args.ipv6
ownargs = {}
for targ in cls.triggers_args:
arg = getattr(args, targ)
ownargs[targ] = arg
del arg
t = cls(**ownargs, dry_run=args.dry_run, cdistargs=args)
t.run()
class TriggerHttp(http.server.BaseHTTPRequestHandler):
actions = { "cdist": [ "config", "install" ],
"file": [ "present", "absent" ]
}
def do_HEAD(self):
self.dispatch_request()
def do_POST(self):
self.dispatch_request()
def do_GET(self):
self.dispatch_request()
def dispatch_request(self):
host = self.client_address[0]
code = 200
self.cdistargs = self.server.cdistargs
# FIXME: generate regexp based on self.actions
m = re.match("^/(?P<subsystem>cdist|file)/(?P<action>present|absent|config|install)/", self.path)
if m:
subsystem = m.group('subsystem')
action = m.group('action')
handler = getattr(self, "handler_" + subsystem)
if not action in self.actions[subsystem]:
code = 404
else:
code = 404
if code == 200:
log.debug("Calling {} -> {}".format(subsystem, action))
handler(action, host)
self.send_response(code)
self.end_headers()
def handler_file(self, action, host):
if not self.server.directory or not self.server.source:
log.info("Cannot server file request: directory or source not setup")
return
try:
ipaddress.ip_address(host)
except ValueError:
log.error("Host is not a valid IP address - aborting")
return
dst = os.path.join(self.server.directory, host)
if action == "present":
shutil.copyfile(self.server.source, dst)
if action == "absent":
if os.path.exists(dst):
os.remove(dst)
def handler_cdist(self, action, host):
log.debug("Running cdist for %s in mode %s", host, mode)
if self.server.dry_run:
log.info("Dry run, skipping cdist execution")
return
cname = action.title()
module = getattr(cdist, action)
theclass = getattr(module, cname)
if hasattr(self.cdistargs, 'out_path'):
out_path = self.cdistargs.out_path
else:
out_path = None
host_base_path, hostdir = theclass.create_host_base_dirs(
host, theclass.create_base_root_path(out_path))
theclass.construct_remote_exec_copy_patterns(self.cdistargs)
log.debug("Executing cdist onehost with params: %s, %s, %s, %s, ",
host, host_base_path, hostdir, self.cdistargs)
theclass.onehost(host, host_base_path, hostdir, self.cdistargs,
parallel=False)
class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer):
"""
Server that listens to both IPv4 and IPv6 requests.
"""
address_family = socket.AF_INET6
def __init__(self, cdistargs, directory, source, *args, **kwargs):
self.cdistargs = cdistargs
self.dry_run = cdistargs.dry_run
self.directory = directory
self.source = source
http.server.HTTPServer.__init__(self, *args, **kwargs)
class HTTPServerV4(HTTPServerV6):
"""
Server that listens to IPv4 requests.
"""
address_family = socket.AF_INET

View file

@ -27,6 +27,11 @@ SYNOPSIS
cdist shell [-h] [-d] [-v] [-s SHELL]
cdist trigger [-h] [-d] [-v] [-b] [-c CONF_DIR] [-i MANIFEST]
[-j [JOBS]] [-n] [-o OUT_PATH]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
[-6] [-H HTTP_PORT]
DESCRIPTION
-----------
@ -148,6 +153,67 @@ usage. Its primary use is for debugging type parameters.
Select shell to use, defaults to current shell. Used shell should
be POSIX compatible shell.
TRIGGER
-------
Start trigger (simple http server) that waits for connections. When host
connects then it triggers config or install command, cdist config is then
executed which configures/installs host.
Request path recognies following formats:
* :strong:`/config/.*` for config
* :strong:`/install/.*` for install
.. option:: -6, --ipv6
Listen to both IPv4 and IPv6 (instead of only IPv4)
.. option:: -b, --enable-beta
Enable beta functionalities.
.. option:: -c CONF_DIR, --conf-dir CONF_DIR
Add configuration directory (can be repeated, last one wins)
.. option:: -d, --debug
Set log level to debug
.. option:: -H HTTP_PORT, --http-port HTTP_PORT
Create trigger listener via http on specified port
.. option:: -h, --help
show this help message and exit
.. option:: -i MANIFEST, --initial-manifest MANIFEST
path to a cdist manifest or '-' to read from stdin.
.. option:: -n, --dry-run
do not execute code
.. option:: -o OUT_PATH, --out-dir OUT_PATH
directory to save cdist output in
.. option:: --remote-copy REMOTE_COPY
Command to use for remote copy (should behave like scp)
.. option:: --remote-exec REMOTE_EXEC
Command to use for remote execution (should behave like ssh)
.. option:: -v, --verbose
Set log level to info, be more verbose
FILES
-----
~/.cdist
@ -199,6 +265,11 @@ EXAMPLES
# Install ikq05.ethz.ch with debug enabled
% cdist install -d ikq05.ethz.ch
# Start trigger in verbose mode that will configure host using specified
# init manifest
% cdist trigger -b -v -i ~/.cdist/manifest/init-for-triggered
ENVIRONMENT
-----------
TMPDIR, TEMP, TMP

View file

@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
# 2010-2016 Nico Schottelius (nico-cdist at schottelius.org)
# 2016 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
@ -23,7 +23,7 @@
# list of beta sub-commands
BETA_COMMANDS = ['install', ]
BETA_COMMANDS = ['install', 'trigger' ]
# list of beta arguments for sub-commands
BETA_ARGS = {
'config': ['jobs', ],
@ -69,6 +69,7 @@ def commandline():
import cdist.config
import cdist.install
import cdist.shell
import cdist.trigger
import shutil
import os
import multiprocessing
@ -84,6 +85,12 @@ def commandline():
'-v', '--verbose', help='Set log level to info, be more verbose',
action='store_true', default=False)
parser['beta'] = argparse.ArgumentParser(add_help=False)
parser['beta'].add_argument(
'-b', '--enable-beta',
help=('Enable beta functionalities.'),
action='store_true', dest='beta', default=False)
# Main subcommand parser
parser['main'] = argparse.ArgumentParser(
description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
@ -98,20 +105,57 @@ def commandline():
'banner', parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
# Config
parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel']])
parser['config'].add_argument(
'host', nargs='*', help='host(s) to operate on')
parser['config'].add_argument(
'-b', '--enable-beta',
help=('Enable beta functionalities. Beta functionalities '
'include the following options: -j/--jobs.'),
action='store_true', dest='beta', default=False)
parser['config'].add_argument(
parser['config_main'] = argparse.ArgumentParser(add_help=False)
parser['config_main'].add_argument(
'-c', '--conf-dir',
help=('Add configuration directory (can be repeated, '
'last one wins)'), action='append')
parser['config_main'].add_argument(
'-i', '--initial-manifest',
help='path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False)
parser['config_main'].add_argument(
'-j', '--jobs', nargs='?', type=check_positive_int,
help=('specify the maximum number of parallel jobs, currently '
'only global explorers are supported (currently in beta'),
action='store', dest='jobs',
const=multiprocessing.cpu_count())
parser['config_main'].add_argument(
'-n', '--dry-run',
help='do not execute code', action='store_true')
parser['config_main'].add_argument(
'-o', '--out-dir',
help='directory to save cdist output in', dest="out_path")
# remote-copy and remote-exec defaults are environment variables
# if set; if not then None - these will be futher handled after
# parsing to determine implementation default
parser['config_main'].add_argument(
'--remote-copy',
help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy',
default=os.environ.get('CDIST_REMOTE_COPY'))
parser['config_main'].add_argument(
'--remote-exec',
help=('Command to use for remote execution '
'(should behave like ssh)'),
action='store', dest='remote_exec',
default=os.environ.get('CDIST_REMOTE_EXEC'))
# Config
parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel'], parser['beta'],
parser['config_main']])
parser['config'].add_argument(
'host', nargs='*', help='host(s) to operate on')
parser['config'].add_argument(
'-s', '--sequential',
help='operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
parser['config'].add_argument(
'-p', '--parallel',
help='operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['config'].add_argument(
'-f', '--file',
help=('Read additional hosts to operate on from specified file '
@ -119,44 +163,6 @@ def commandline():
'If no host or host file is specified then, by default, '
'read hosts from stdin.'),
dest='hostfile', required=False)
parser['config'].add_argument(
'-i', '--initial-manifest',
help='Path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False)
parser['config'].add_argument(
'-j', '--jobs', nargs='?', type=check_positive_int,
help=('Specify the maximum number of parallel jobs, currently '
'only global explorers are supported (currently in beta'),
action='store', dest='jobs',
const=multiprocessing.cpu_count())
parser['config'].add_argument(
'-n', '--dry-run',
help='Do not execute code', action='store_true')
parser['config'].add_argument(
'-o', '--out-dir',
help='Directory to save cdist output in', dest="out_path")
parser['config'].add_argument(
'-p', '--parallel',
help='Operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['config'].add_argument(
'-s', '--sequential',
help='Operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
# remote-copy and remote-exec defaults are environment variables
# if set; if not then None - these will be futher handled after
# parsing to determine implementation default
parser['config'].add_argument(
'--remote-copy',
help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy',
default=os.environ.get('CDIST_REMOTE_COPY'))
parser['config'].add_argument(
'--remote-exec',
help=('Command to use for remote execution '
'(should behave like ssh)'),
action='store', dest='remote_exec',
default=os.environ.get('CDIST_REMOTE_EXEC'))
parser['config'].set_defaults(func=cdist.config.Config.commandline)
# Install
@ -173,6 +179,34 @@ def commandline():
' should be POSIX compatible shell.'))
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
# Trigger
parser['trigger'] = parser['sub'].add_parser(
'trigger', parents=[parser['loglevel'],
parser['beta'],
parser['config_main']])
parser['trigger'].add_argument(
'-6', '--ipv6', default=False,
help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
action='store_true')
parser['trigger'].add_argument(
'-H', '--http-port', action='store', default=3000, required=False,
help=('Create trigger listener via http on specified port'))
parser['trigger'].add_argument(
'-D', '--directory', action='store', required=False,
help=('Where to create local files'))
parser['trigger'].add_argument(
'-S', '--source', action='store', required=False,
help=('Which file to copy for creation'))
parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
# Install
parser['install'] = parser['sub'].add_parser('install', add_help=False,
parents=[parser['config']])
parser['install'].set_defaults(func=cdist.install.Install.commandline)
for p in parser:
parser[p].epilog = (
"Get cdist at http://www.nico.schottelius.org/software/cdist/")