Compare commits

...

20 commits

Author SHA1 Message Date
Nico Schottelius
9afb57412d add file support
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-11-27 17:11:34 +01:00
Nico Schottelius
4207124c52 add debug output when known url was triggered
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-11-27 16:16:34 +01:00
Nico Schottelius
ff31dcada0 begin to add local file support
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-11-27 15:59:48 +01:00
Darko Poljak
ab5d941802 Merge branch 'master' into feature/trigger 2016-11-01 08:17:50 +01:00
Darko Poljak
7f0ad6665b Add trigger to cdist man page. 2016-10-31 18:42:16 +01:00
Darko Poljak
aac9906fff Remove redundant log.debug. 2016-10-31 08:09:38 +01:00
Darko Poljak
376a031a95 Add more log.debug. 2016-10-31 08:07:09 +01:00
Darko Poljak
94e8f0b2b2 Minor log.debug message fixes. 2016-10-31 08:05:57 +01:00
Darko Poljak
686a484b03 Fix dry_run and logging. 2016-10-31 07:59:03 +01:00
Darko Poljak
05cf492744 Implement dry_run option, add log.debug lines. 2016-10-31 07:51:27 +01:00
Darko Poljak
63dc9632d2 Make IPv4 default. 2016-10-30 20:37:24 +01:00
Darko Poljak
8c985fe2cb Add forking support. 2016-10-30 20:25:32 +01:00
Darko Poljak
297367390f Fix bug. 2016-10-30 17:21:08 +01:00
Darko Poljak
d316089842 Continue with trigger. 2016-10-30 16:41:59 +01:00
Darko Poljak
7c169f2d0a Merge remote-tracking branch 'ungleich/feature/trigger' into feature/trigger 2016-10-30 15:09:26 +01:00
Nico Schottelius
535181435f update for darko
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-10-30 15:01:45 +01:00
Darko Poljak
ca67533ce4 Fix comment typos. 2016-10-30 14:49:07 +01:00
Darko Poljak
579b8d5c72 Fix typo. 2016-10-30 14:48:10 +01:00
Nico Schottelius
92bb0803eb Finish base functionality for trigger execution 2016-10-25 12:38:13 +02:00
Nico Schottelius
7d027225bc begin integration of trigger handler
Signed-off-by: Nico Schottelius <nico@wurzel.schottelius.org>
2016-10-24 15:58:39 +02:00
4 changed files with 394 additions and 74 deletions

View file

@ -130,13 +130,32 @@ class Config(object):
for host in source: for host in source:
yield host yield host
@staticmethod
def construct_remote_exec_copy_patterns(args):
# default remote cmd patterns
args.remote_exec_pattern = None
args.remote_copy_pattern = None
args_dict = vars(args)
# if remote-exec and/or remote-copy args are None then user
# didn't specify command line options nor env vars:
# inspect multiplexing options for default cdist.REMOTE_COPY/EXEC
if (args_dict['remote_copy'] is None or
args_dict['remote_exec'] is None):
mux_opts = inspect_ssh_mux_opts()
if args_dict['remote_exec'] is None:
args.remote_exec_pattern = cdist.REMOTE_EXEC + mux_opts
if args_dict['remote_copy'] is None:
args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
@classmethod @classmethod
def commandline(cls, args): def commandline(cls, args):
"""Configure remote system""" """Configure remote system"""
import multiprocessing import multiprocessing
# FIXME: Refactor relict - remove later # FIXME: Refactor relict - remove later
log = logging.getLogger("cdist") log = logging.getLogger(__name__)
if args.manifest == '-' and args.hostfile == '-': if args.manifest == '-' and args.hostfile == '-':
raise cdist.Error(("Cannot read both, manifest and host file, " raise cdist.Error(("Cannot read both, manifest and host file, "
@ -166,32 +185,14 @@ class Config(object):
failed_hosts = [] failed_hosts = []
time_start = time.time() time_start = time.time()
# default remote cmd patterns cls.construct_remote_exec_copy_patterns(args)
args.remote_exec_pattern = None base_root_path = cls.create_base_root_path(args.out_path)
args.remote_copy_pattern = None
args_dict = vars(args)
# if remote-exec and/or remote-copy args are None then user
# didn't specify command line options nor env vars:
# inspect multiplexing options for default cdist.REMOTE_COPY/EXEC
if (args_dict['remote_copy'] is None or
args_dict['remote_exec'] is None):
mux_opts = inspect_ssh_mux_opts()
if args_dict['remote_exec'] is None:
args.remote_exec_pattern = cdist.REMOTE_EXEC + mux_opts
if args_dict['remote_copy'] is None:
args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
if args.out_path:
base_root_path = args.out_path
else:
base_root_path = tempfile.mkdtemp()
hostcnt = 0 hostcnt = 0
for host in itertools.chain(cls.hosts(args.host), for host in itertools.chain(cls.hosts(args.host),
cls.hosts(args.hostfile)): cls.hosts(args.hostfile)):
hostdir = cdist.str_hash(host) host_base_path, hostdir = cls.create_host_base_dirs(
host_base_path = os.path.join(base_root_path, hostdir) host, base_root_path)
log.debug("Base root path for target host \"{}\" is \"{}\"".format( log.debug("Base root path for target host \"{}\" is \"{}\"".format(
host, host_base_path)) host, host_base_path))
@ -227,6 +228,8 @@ class Config(object):
raise cdist.Error("Failed to configure the following hosts: " + raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts)) " ".join(failed_hosts))
@classmethod @classmethod
def onehost(cls, host, host_base_path, host_dir_name, args, parallel): def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
"""Configure ONE system""" """Configure ONE system"""
@ -317,6 +320,25 @@ class Config(object):
else: else:
raise raise
@staticmethod
def create_base_root_path(out_path=None):
if out_path:
base_root_path = out_path
else:
base_root_path = tempfile.mkdtemp()
return base_root_path
@staticmethod
def create_host_base_dirs(host, base_root_path):
hostdir = cdist.str_hash(host)
host_base_path = os.path.join(base_root_path, hostdir)
return (host_base_path, hostdir)
def run(self): def run(self):
"""Do what is most often done: deploy & cleanup""" """Do what is most often done: deploy & cleanup"""
start_time = time.time() start_time = time.time()

193
cdist/trigger.py Normal file
View file

@ -0,0 +1,193 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2016 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import ipaddress
import logging
import re
import socket
import http.server
import os
import socketserver
import shutil
import multiprocessing
import cdist.config
import cdist.install
log = logging.getLogger(__name__)
class Trigger():
"""cdist trigger handling"""
# Arguments that are only trigger specific
triggers_args = [ "http_port", "ipv6", "directory", "source" ]
def __init__(self, http_port=None, dry_run=False, ipv6=False,
directory=None, source=None, cdistargs=None):
self.log = logging.getLogger("trigger")
self.dry_run = dry_run
self.http_port = int(http_port)
self.ipv6 = ipv6
self.args = cdistargs
self.directory = directory
self.source = source
log.debug("IPv6: %s", self.ipv6)
def run_httpd(self):
server_address = ('', self.http_port)
if self.ipv6:
httpdcls = HTTPServerV6
else:
httpdcls = HTTPServerV4
httpd = httpdcls(self.args, self.directory, self.source, server_address, TriggerHttp)
log.debug("Starting server at port %d", self.http_port)
if self.dry_run:
log.debug("Running in dry run mode")
httpd.serve_forever()
def run(self):
if self.http_port:
self.run_httpd()
@classmethod
def commandline(cls, args):
http_port = args.http_port
ipv6 = args.ipv6
ownargs = {}
for targ in cls.triggers_args:
arg = getattr(args, targ)
ownargs[targ] = arg
del arg
t = cls(**ownargs, dry_run=args.dry_run, cdistargs=args)
t.run()
class TriggerHttp(http.server.BaseHTTPRequestHandler):
actions = { "cdist": [ "config", "install" ],
"file": [ "present", "absent" ]
}
def do_HEAD(self):
self.dispatch_request()
def do_POST(self):
self.dispatch_request()
def do_GET(self):
self.dispatch_request()
def dispatch_request(self):
host = self.client_address[0]
code = 200
self.cdistargs = self.server.cdistargs
# FIXME: generate regexp based on self.actions
m = re.match("^/(?P<subsystem>cdist|file)/(?P<action>present|absent|config|install)/", self.path)
if m:
subsystem = m.group('subsystem')
action = m.group('action')
handler = getattr(self, "handler_" + subsystem)
if not action in self.actions[subsystem]:
code = 404
else:
code = 404
if code == 200:
log.debug("Calling {} -> {}".format(subsystem, action))
handler(action, host)
self.send_response(code)
self.end_headers()
def handler_file(self, action, host):
if not self.server.directory or not self.server.source:
log.info("Cannot server file request: directory or source not setup")
return
try:
ipaddress.ip_address(host)
except ValueError:
log.error("Host is not a valid IP address - aborting")
return
dst = os.path.join(self.server.directory, host)
if action == "present":
shutil.copyfile(self.server.source, dst)
if action == "absent":
if os.path.exists(dst):
os.remove(dst)
def handler_cdist(self, action, host):
log.debug("Running cdist for %s in mode %s", host, mode)
if self.server.dry_run:
log.info("Dry run, skipping cdist execution")
return
cname = action.title()
module = getattr(cdist, action)
theclass = getattr(module, cname)
if hasattr(self.cdistargs, 'out_path'):
out_path = self.cdistargs.out_path
else:
out_path = None
host_base_path, hostdir = theclass.create_host_base_dirs(
host, theclass.create_base_root_path(out_path))
theclass.construct_remote_exec_copy_patterns(self.cdistargs)
log.debug("Executing cdist onehost with params: %s, %s, %s, %s, ",
host, host_base_path, hostdir, self.cdistargs)
theclass.onehost(host, host_base_path, hostdir, self.cdistargs,
parallel=False)
class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer):
"""
Server that listens to both IPv4 and IPv6 requests.
"""
address_family = socket.AF_INET6
def __init__(self, cdistargs, directory, source, *args, **kwargs):
self.cdistargs = cdistargs
self.dry_run = cdistargs.dry_run
self.directory = directory
self.source = source
http.server.HTTPServer.__init__(self, *args, **kwargs)
class HTTPServerV4(HTTPServerV6):
"""
Server that listens to IPv4 requests.
"""
address_family = socket.AF_INET

View file

@ -27,6 +27,11 @@ SYNOPSIS
cdist shell [-h] [-d] [-v] [-s SHELL] cdist shell [-h] [-d] [-v] [-s SHELL]
cdist trigger [-h] [-d] [-v] [-b] [-c CONF_DIR] [-i MANIFEST]
[-j [JOBS]] [-n] [-o OUT_PATH]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
[-6] [-H HTTP_PORT]
DESCRIPTION DESCRIPTION
----------- -----------
@ -148,6 +153,67 @@ usage. Its primary use is for debugging type parameters.
Select shell to use, defaults to current shell. Used shell should Select shell to use, defaults to current shell. Used shell should
be POSIX compatible shell. be POSIX compatible shell.
TRIGGER
-------
Start trigger (simple http server) that waits for connections. When host
connects then it triggers config or install command, cdist config is then
executed which configures/installs host.
Request path recognies following formats:
* :strong:`/config/.*` for config
* :strong:`/install/.*` for install
.. option:: -6, --ipv6
Listen to both IPv4 and IPv6 (instead of only IPv4)
.. option:: -b, --enable-beta
Enable beta functionalities.
.. option:: -c CONF_DIR, --conf-dir CONF_DIR
Add configuration directory (can be repeated, last one wins)
.. option:: -d, --debug
Set log level to debug
.. option:: -H HTTP_PORT, --http-port HTTP_PORT
Create trigger listener via http on specified port
.. option:: -h, --help
show this help message and exit
.. option:: -i MANIFEST, --initial-manifest MANIFEST
path to a cdist manifest or '-' to read from stdin.
.. option:: -n, --dry-run
do not execute code
.. option:: -o OUT_PATH, --out-dir OUT_PATH
directory to save cdist output in
.. option:: --remote-copy REMOTE_COPY
Command to use for remote copy (should behave like scp)
.. option:: --remote-exec REMOTE_EXEC
Command to use for remote execution (should behave like ssh)
.. option:: -v, --verbose
Set log level to info, be more verbose
FILES FILES
----- -----
~/.cdist ~/.cdist
@ -199,6 +265,11 @@ EXAMPLES
# Install ikq05.ethz.ch with debug enabled # Install ikq05.ethz.ch with debug enabled
% cdist install -d ikq05.ethz.ch % cdist install -d ikq05.ethz.ch
# Start trigger in verbose mode that will configure host using specified
# init manifest
% cdist trigger -b -v -i ~/.cdist/manifest/init-for-triggered
ENVIRONMENT ENVIRONMENT
----------- -----------
TMPDIR, TEMP, TMP TMPDIR, TEMP, TMP

View file

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2010-2016 Nico Schottelius (nico-cdist at schottelius.org)
# 2016 Darko Poljak (darko.poljak at gmail.com) # 2016 Darko Poljak (darko.poljak at gmail.com)
# #
# This file is part of cdist. # This file is part of cdist.
@ -23,7 +23,7 @@
# list of beta sub-commands # list of beta sub-commands
BETA_COMMANDS = ['install', ] BETA_COMMANDS = ['install', 'trigger' ]
# list of beta arguments for sub-commands # list of beta arguments for sub-commands
BETA_ARGS = { BETA_ARGS = {
'config': ['jobs', ], 'config': ['jobs', ],
@ -69,6 +69,7 @@ def commandline():
import cdist.config import cdist.config
import cdist.install import cdist.install
import cdist.shell import cdist.shell
import cdist.trigger
import shutil import shutil
import os import os
import multiprocessing import multiprocessing
@ -84,6 +85,12 @@ def commandline():
'-v', '--verbose', help='Set log level to info, be more verbose', '-v', '--verbose', help='Set log level to info, be more verbose',
action='store_true', default=False) action='store_true', default=False)
parser['beta'] = argparse.ArgumentParser(add_help=False)
parser['beta'].add_argument(
'-b', '--enable-beta',
help=('Enable beta functionalities.'),
action='store_true', dest='beta', default=False)
# Main subcommand parser # Main subcommand parser
parser['main'] = argparse.ArgumentParser( parser['main'] = argparse.ArgumentParser(
description='cdist ' + cdist.VERSION, parents=[parser['loglevel']]) description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
@ -98,20 +105,57 @@ def commandline():
'banner', parents=[parser['loglevel']]) 'banner', parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner) parser['banner'].set_defaults(func=cdist.banner.banner)
# Config parser['config_main'] = argparse.ArgumentParser(add_help=False)
parser['config'] = parser['sub'].add_parser( parser['config_main'].add_argument(
'config', parents=[parser['loglevel']])
parser['config'].add_argument(
'host', nargs='*', help='host(s) to operate on')
parser['config'].add_argument(
'-b', '--enable-beta',
help=('Enable beta functionalities. Beta functionalities '
'include the following options: -j/--jobs.'),
action='store_true', dest='beta', default=False)
parser['config'].add_argument(
'-c', '--conf-dir', '-c', '--conf-dir',
help=('Add configuration directory (can be repeated, ' help=('Add configuration directory (can be repeated, '
'last one wins)'), action='append') 'last one wins)'), action='append')
parser['config_main'].add_argument(
'-i', '--initial-manifest',
help='path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False)
parser['config_main'].add_argument(
'-j', '--jobs', nargs='?', type=check_positive_int,
help=('specify the maximum number of parallel jobs, currently '
'only global explorers are supported (currently in beta'),
action='store', dest='jobs',
const=multiprocessing.cpu_count())
parser['config_main'].add_argument(
'-n', '--dry-run',
help='do not execute code', action='store_true')
parser['config_main'].add_argument(
'-o', '--out-dir',
help='directory to save cdist output in', dest="out_path")
# remote-copy and remote-exec defaults are environment variables
# if set; if not then None - these will be futher handled after
# parsing to determine implementation default
parser['config_main'].add_argument(
'--remote-copy',
help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy',
default=os.environ.get('CDIST_REMOTE_COPY'))
parser['config_main'].add_argument(
'--remote-exec',
help=('Command to use for remote execution '
'(should behave like ssh)'),
action='store', dest='remote_exec',
default=os.environ.get('CDIST_REMOTE_EXEC'))
# Config
parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel'], parser['beta'],
parser['config_main']])
parser['config'].add_argument(
'host', nargs='*', help='host(s) to operate on')
parser['config'].add_argument(
'-s', '--sequential',
help='operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
parser['config'].add_argument(
'-p', '--parallel',
help='operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['config'].add_argument( parser['config'].add_argument(
'-f', '--file', '-f', '--file',
help=('Read additional hosts to operate on from specified file ' help=('Read additional hosts to operate on from specified file '
@ -119,44 +163,6 @@ def commandline():
'If no host or host file is specified then, by default, ' 'If no host or host file is specified then, by default, '
'read hosts from stdin.'), 'read hosts from stdin.'),
dest='hostfile', required=False) dest='hostfile', required=False)
parser['config'].add_argument(
'-i', '--initial-manifest',
help='Path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False)
parser['config'].add_argument(
'-j', '--jobs', nargs='?', type=check_positive_int,
help=('Specify the maximum number of parallel jobs, currently '
'only global explorers are supported (currently in beta'),
action='store', dest='jobs',
const=multiprocessing.cpu_count())
parser['config'].add_argument(
'-n', '--dry-run',
help='Do not execute code', action='store_true')
parser['config'].add_argument(
'-o', '--out-dir',
help='Directory to save cdist output in', dest="out_path")
parser['config'].add_argument(
'-p', '--parallel',
help='Operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['config'].add_argument(
'-s', '--sequential',
help='Operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
# remote-copy and remote-exec defaults are environment variables
# if set; if not then None - these will be futher handled after
# parsing to determine implementation default
parser['config'].add_argument(
'--remote-copy',
help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy',
default=os.environ.get('CDIST_REMOTE_COPY'))
parser['config'].add_argument(
'--remote-exec',
help=('Command to use for remote execution '
'(should behave like ssh)'),
action='store', dest='remote_exec',
default=os.environ.get('CDIST_REMOTE_EXEC'))
parser['config'].set_defaults(func=cdist.config.Config.commandline) parser['config'].set_defaults(func=cdist.config.Config.commandline)
# Install # Install
@ -173,6 +179,34 @@ def commandline():
' should be POSIX compatible shell.')) ' should be POSIX compatible shell.'))
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
# Trigger
parser['trigger'] = parser['sub'].add_parser(
'trigger', parents=[parser['loglevel'],
parser['beta'],
parser['config_main']])
parser['trigger'].add_argument(
'-6', '--ipv6', default=False,
help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
action='store_true')
parser['trigger'].add_argument(
'-H', '--http-port', action='store', default=3000, required=False,
help=('Create trigger listener via http on specified port'))
parser['trigger'].add_argument(
'-D', '--directory', action='store', required=False,
help=('Where to create local files'))
parser['trigger'].add_argument(
'-S', '--source', action='store', required=False,
help=('Which file to copy for creation'))
parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
# Install
parser['install'] = parser['sub'].add_parser('install', add_help=False,
parents=[parser['config']])
parser['install'].set_defaults(func=cdist.install.Install.commandline)
for p in parser: for p in parser:
parser[p].epilog = ( parser[p].epilog = (
"Get cdist at http://www.nico.schottelius.org/software/cdist/") "Get cdist at http://www.nico.schottelius.org/software/cdist/")