Implement triggering functionality

This commit is contained in:
Darko Poljak 2019-01-26 17:00:03 +01:00
parent 9859080217
commit 44178b94e2
15 changed files with 594 additions and 21 deletions

View file

@ -5,11 +5,12 @@ import logging
import collections
import functools
import cdist.configuration
import cdist.trigger
import cdist.preos
# set of beta sub-commands
BETA_COMMANDS = set(('install', 'inventory', ))
BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', ))
# set of beta arguments for sub-commands
BETA_ARGS = {
'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@ -436,6 +437,27 @@ def get_parsers():
' should be POSIX compatible shell.'))
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
# Trigger
parser['trigger'] = parser['sub'].add_parser(
'trigger', parents=[parser['loglevel'],
parser['beta'],
parser['config_main']])
parser['trigger'].add_argument(
'-6', '--ipv6', default=False,
help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
action='store_true')
parser['trigger'].add_argument(
'-D', '--directory', action='store', required=False,
help=('Where to create local files'))
parser['trigger'].add_argument(
'-H', '--http-port', action='store', default=3000, required=False,
help=('Create trigger listener via http on specified port'))
parser['trigger'].add_argument(
'-S', '--source', action='store', required=False,
help=('Which file to copy for creation'))
parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
for p in parser:
parser[p].epilog = EPILOG

View file

@ -0,0 +1,12 @@
#!/bin/sh
os=$(cat "$__global/explorer/os")
case "$os" in
devuan)
echo "update-rc.d cdist-preos-trigger defaults > /dev/null"
;;
*)
;;
esac

View file

@ -0,0 +1,45 @@
cdist-type__cdist_preos_trigger(7)
==================================
NAME
----
cdist-type__cdist_preos_trigger - configure cdist preos trigger
DESCRIPTION
-----------
Create cdist PreOS trigger by creating systemd unit file that will be started
at boot and will execute trigger command - connect to specified host and port.
REQUIRED PARAMETERS
-------------------
trigger-command
Command that will be executed as a PreOS cdist trigger.
OPTIONAL PARAMETERS
-------------------
None
EXAMPLES
--------
.. code-block:: sh
# Configure default curl trigger for host cdist.ungleich.ch at port 80.
__cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80'
AUTHORS
-------
Darko Poljak <darko.poljak--@--ungleich.ch>
COPYING
-------
Copyright \(C) 2016 Darko Poljak. You can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

View file

@ -0,0 +1,67 @@
#!/bin/sh
os="$(cat "$__global/explorer/os")"
trigger_command=$(cat "$__object/parameter/trigger-command")
case "$os" in
devuan)
__file /etc/init.d/cdist-preos-trigger --owner root \
--group root \
--mode 755 \
--source - << EOF
#!/bin/sh
# /etc/init.d/cdist-preos-trigger
### BEGIN INIT INFO
# Provides: cdist-preos-trigger
# Required-Start: \$all
# Required-Stop:
# Default-Start: 2 3 4 5 S
# Default-Stop: 0 1 6
# Short-Description: Execute cdist preos trigger command
# Description: Execute cdist preos trigger commnad.
### END INIT INFO
case "\$1" in
start)
echo "Starting cdist-preos-trigger command"
${trigger_command} &
;;
stop)
# no-op
;;
*)
echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}"
exit 1
;;
esac
exit 0
EOF
;;
*)
__file /etc/systemd/system/cdist-preos-trigger.service --owner root \
--group root \
--mode 644 \
--source - << EOF
[Unit]
Description=preos trigger
Wants=network-online.target
After=network.target network-online.target
[Service]
Type=simple
Restart=no
# Broken systemd
ExecStartPre=/bin/sleep 5
ExecStart=${trigger_command}
[Install]
WantedBy=multi-user.target
EOF
require="__file/etc/systemd/system/cdist-preos-trigger.service" \
__start_on_boot cdist-preos-trigger
;;
esac

View file

@ -0,0 +1 @@
trigger-command

View file

@ -127,6 +127,12 @@ class Debian(object):
help="suite used for debootstrap, "
"by default '{}'".format(defargs.suite),
dest='suite', default=defargs.suite)
parser.add_argument(
'-t', '--trigger-command',
help=("trigger command that will be added to cdist config; "
"'__cdist_preos_trigger http ...' type is appended to "
"initial manifest"),
dest='trigger_command')
parser.add_argument(
'-y', '--remote-copy',
help=("remote copy that cdist config will use, by default "

View file

@ -127,6 +127,13 @@ then
exit 1
fi
if [ "${trigger_command}" ]
then
trigger_line="__cdist_preos_trigger http --trigger-command '${trigger_command}'\n"
else
trigger_line=""
fi
if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ]
then
i="$((keyfile_cnt - 1))"
@ -174,7 +181,7 @@ then
fi
grub_lines="${grub_manifest_line}${grub_kern_params_line}"
printf "${ssh_auth_keys_line}${grub_lines}" \
printf "${trigger_line}${ssh_auth_keys_line}${grub_lines}" \
| cat "${manifest}" - |\
cdist config \
${cdist_params} -i - \

225
cdist/trigger.py Normal file
View file

@ -0,0 +1,225 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2016 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import ipaddress
import logging
import re
import socket
import http.server
import os
import socketserver
import shutil
import cdist.config
import cdist.log
import cdist.util.ipaddr as ipaddr
class Trigger():
"""cdist trigger handling"""
# Arguments that are only trigger specific
triggers_args = ["http_port", "ipv6", "directory", "source", ]
def __init__(self, http_port=None, dry_run=False, ipv6=False,
directory=None, source=None, cdistargs=None):
self.dry_run = dry_run
self.http_port = int(http_port)
self.ipv6 = ipv6
self.args = cdistargs
self.directory = directory
self.source = source
log.debug("IPv6: %s", self.ipv6)
def run_httpd(self):
server_address = ('', self.http_port)
if self.ipv6:
httpdcls = HTTPServerV6
else:
httpdcls = HTTPServerV4
httpd = httpdcls(self.args, self.directory, self.source,
server_address, TriggerHttp)
log.debug("Starting server at port %d", self.http_port)
if self.dry_run:
log.debug("Running in dry run mode")
httpd.serve_forever()
def run(self):
if self.http_port:
self.run_httpd()
@classmethod
def commandline(cls, args):
global log
# remove root logger default cdist handler and configure trigger's own
logging.getLogger().handlers = []
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s')
log = logging.getLogger("trigger")
ownargs = {}
for targ in cls.triggers_args:
arg = getattr(args, targ)
ownargs[targ] = arg
del arg
t = cls(dry_run=args.dry_run, cdistargs=args, **ownargs)
t.run()
class TriggerHttp(http.server.BaseHTTPRequestHandler):
actions = {
"cdist": ["config", "install", ],
"file": ["present", "absent", ],
}
def do_HEAD(self):
self.dispatch_request()
def do_POST(self):
self.dispatch_request()
def do_GET(self):
self.dispatch_request()
def _actions_regex(self):
regex = ["^/(?P<subsystem>", ]
regex.extend("|".join(self.actions.keys()))
regex.append(")/(?P<action>")
regex.extend("|".join("|".join(self.actions[x]) for x in self.actions))
regex.append(")/")
return "".join(regex)
def dispatch_request(self):
host = self.client_address[0]
code = 200
message = None
self.cdistargs = self.server.cdistargs
actions_regex = self._actions_regex()
m = re.match(actions_regex, self.path)
if m:
subsystem = m.group('subsystem')
action = m.group('action')
handler = getattr(self, "handler_" + subsystem)
if action not in self.actions[subsystem]:
code = 404
else:
code = 404
if code == 200:
log.debug("Calling {} -> {}".format(subsystem, action))
try:
handler(action, host)
except cdist.Error as e:
# cdist is not broken, cdist run is broken
code = 599 # use arbitrary unassigned error code
message = str(e)
except Exception as e:
# cdist/trigger server is broken
code = 500
self.send_response(code=code, message=message)
self.end_headers()
def handler_file(self, action, host):
if not self.server.directory or not self.server.source:
log.info("Cannot serve file request: directory or source "
"not setup")
return
try:
ipaddress.ip_address(host)
except ValueError:
log.error("Host is not a valid IP address - aborting")
return
dst = os.path.join(self.server.directory, host)
if action == "present":
shutil.copyfile(self.server.source, dst)
if action == "absent":
if os.path.exists(dst):
os.remove(dst)
def handler_cdist(self, action, host):
log.debug("Running cdist action %s for %s", action, host)
if self.server.dry_run:
log.info("Dry run, skipping cdist execution")
return
cname = action.title()
module = getattr(cdist, action)
theclass = getattr(module, cname)
if hasattr(self.cdistargs, 'out_path'):
out_path = self.cdistargs.out_path
else:
out_path = None
host_base_path, hostdir = theclass.create_host_base_dirs(
host, theclass.create_base_root_path(out_path))
theclass.construct_remote_exec_copy_patterns(self.cdistargs)
host_tags = None
host_name = ipaddr.resolve_target_host_name(host)
log.debug('Resolved target host name: %s', host_name)
if host_name:
target_host = host_name
else:
target_host = host
log.debug('Using target_host: %s', target_host)
log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ",
target_host, host_tags, host_base_path, hostdir,
self.cdistargs)
theclass.onehost(target_host, host_tags, host_base_path, hostdir,
self.cdistargs, parallel=False)
class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer):
"""
Server that listens to both IPv4 and IPv6 requests.
"""
address_family = socket.AF_INET6
def __init__(self, cdistargs, directory, source, *args, **kwargs):
self.cdistargs = cdistargs
self.dry_run = cdistargs.dry_run
self.directory = directory
self.source = source
http.server.HTTPServer.__init__(self, *args, **kwargs)
class HTTPServerV4(HTTPServerV6):
"""
Server that listens to IPv4 requests.
"""
address_family = socket.AF_INET

View file

@ -6,7 +6,7 @@ _cdist()
prev="${COMP_WORDS[COMP_CWORD-1]}"
prevprev="${COMP_WORDS[COMP_CWORD-2]}"
opts="-h --help -q --quiet -v --verbose -V --version"
cmds="banner config install inventory preos shell"
cmds="banner config install inventory preos shell trigger"
case "${prevprev}" in
shell)
@ -80,6 +80,14 @@ _cdist()
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
return 0
;;
trigger)
opts="-h --help -d --debug -v --verbose -b --beta \
-C --cache-path-pattern -c --conf-dir -i --initial-manifest \
-j --jobs -n --dry-run -o --out-dir --remote-copy \
--remote-exec -6 --ipv6 -H --http-port -D --directory -S --source"
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
return 0
;;
inventory)
cmds="list add-host del-host add-tag del-tag"
opts="-h --help -q --quiet -v --verbose"

View file

@ -11,7 +11,7 @@ _cdist()
case $state in
opts_cmds)
_arguments '1:Options and commands:(banner config install inventory preos shell -h --help -q --quiet -v --verbose -V --version)'
_arguments '1:Options and commands:(banner config install inventory preos shell trigger -h --help -q --quiet -v --verbose -V --version)'
;;
*)
case $words[2] in

View file

@ -1,6 +1,9 @@
Changelog
---------
next:
* Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
6.3.0: 2019-12-12
* Type __package_update_index: Fix Alpine part (Dominique Roux)
* Documentation: Fix man pages for install types (Darko Poljak)

View file

@ -25,15 +25,16 @@ For example, to create an ubuntu PreOS:
.. code-block:: sh
$ cdist preos ubuntu /preos/preos-ubuntu -B -C \
-k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu
-k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \
-t "/usr/bin/curl 192.168.111.5:3000/install/"
For more info about the available options see the cdist manual page.
This will bootstrap (``-B``) ubuntu PreOS in the ``/preos/preos-ubuntu``
directory, it will be configured (``-C``) using default the built-in initial
manifest and with specified ssh authorized key (``-k``).
After bootstrapping and configuration, the PXE boot directory will be
created (``-p``) in ``/preos/pxe-ubuntu``.
This will bootstrap (``-B``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it
will be configured (``-C``) using default built-in initial manifest and with
specified ssh authorized key (``-k``) and with specified trigger command (``-t``).
After bootstrapping and configuration PXE
boot directory will be created (``-p``) in ``/preos/pxe-ubuntu``.
After PreOS is created, new machines can be booted using the created PXE
(after proper dhcp and tftp settings).
@ -41,8 +42,17 @@ After PreOS is created, new machines can be booted using the created PXE
Since PreOS is configured with ssh authorized key it can be accessed through
ssh, i.e. it can be further installed and configured with cdist.
Implementing a new PreOS sub-command
------------------------------------
When installing and configuring new machines using cdist's PreOS concept
cdist can use triggering for host installation/configuration, which is described
in the previous chapter.
When new machine is booted with PreOS then trigger command is executed.
Machine will connect to cdist trigger server. If the request is, for example,
for installation then cdist trigger server will start install command for the
client host using parameters specified at trigger server startup.
Implementing new PreOS sub-command
----------------------------------
preos command is implemented as a plugin system. This plugin system scans for
preos subcommands in the ``cdist/preos/`` distribution directory and also in
``~/.cdist/preos/`` directory if it exists.
@ -127,3 +137,32 @@ When you try to run this new preos you will get:
In the ``commandline`` function/method you have all the freedom to actually create
a PreOS.
Simple tipical use case for using PreOS and trigger
---------------------------------------------------
Tipical use case for using PreOS and trigger command include the following steps.
#. Create PreOS PXE with ssh key and trigger command for installation.
.. code-block:: sh
$ cdist preos ubuntu /preos/ubuntu -b -C \
-k ~/.ssh/id_rsa.pub -p /preos/pxe \
-t "/usr/bin/curl 192.168.111.5:3000/install/"
#. Configure dhcp server and tftp server.
#. On cdist host (192.168.111.5 from above) start trigger command (it will use
default init manifest for installation).
.. code-block:: sh
$ cdist trigger -b -v
#. After all is set up start new machines (PXE boot).
#. New machine boots and executes trigger command, i.e. triggers installation.
#. Cdist trigger server starts installing host that has triggered it.
#. After cdist install is finished new host is installed.

View file

@ -0,0 +1,33 @@
Trigger
=======
Description
-----------
cdist supports triggering for host installation/configuration using trigger command.
This command starts trigger server at management node, for example:
.. code-block:: sh
$ cdist trigger -b -v
This will start cdist trigger server in verbose mode. cdist trigger server accepts
simple requests for configuration and for installation:
* :strong:`/cdist/install/.*` for installation
* :strong:`/cdist/config/.*` for configuration.
Machines can then trigger cdist trigger server with appropriate requests.
If the request is, for example, for installation (:strong:`/cdist/install/`)
then cdist trigger server will start install command for the client host using
parameters specified at trigger server startup. For the above example that means
that client will be installed using default initial manifest.
When triggered cdist will try to reverse DNS lookup for host name and if
host name is dervied then it is used for running cdist config. If no
host name is resolved then IP address is used.
This command returns the following response codes to client requests:
* 200 for success
* 599 for cdist run errors
* 500 for cdist/server errors.

View file

@ -32,6 +32,7 @@ is being used in small up to enterprise grade environments.
cdist-messaging
cdist-parallelization
cdist-inventory
cdist-trigger
cdist-preos
cdist-integration
cdist-reference

View file

@ -11,7 +11,7 @@ SYNOPSIS
::
cdist [-h] [-V] {banner,config,install,inventory,preos,shell} ...
cdist [-h] [-V] {banner,config,install,inventory,preos,shell,trigger} ...
cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
@ -65,25 +65,35 @@ SYNOPSIS
[-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
[-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
[-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
[-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
[-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
[-y REMOTE_COPY]
target_dir
cdist preos [preos-options] devuan [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
[-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
[-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
[-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
[-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
[-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
[-y REMOTE_COPY]
target_dir
cdist preos [preos-options] ubuntu [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
[-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC]
[-i MANIFEST] [-k KEYFILE ] [-m MIRROR]
[-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r]
[-S SCRIPT] [-s SUITE] [-y REMOTE_COPY]
[-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND]
[-y REMOTE_COPY]
target_dir
cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN]
[-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n]
[-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
[-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
[--remote-exec REMOTE_EXEC] [-6] [-D DIRECTORY]
[-H HTTP_PORT] [-S SOURCE]
DESCRIPTION
-----------
@ -527,6 +537,10 @@ PREOS DEBIAN/DEVUAN
**-s SUITE, --suite SUITE**
suite used for debootstrap, by default 'stable'
**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND**
trigger command that will be added to cdist config;
'``__cdist_preos_trigger http ...``' type is appended to initial manifest
**-y REMOTE_COPY, --remote-copy REMOTE_COPY**
remote copy that cdist config will use, by default
internal script is used
@ -587,6 +601,10 @@ PREOS UBUNTU
**-s SUITE, --suite SUITE**
suite used for debootstrap, by default 'xenial'
**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND**
trigger command that will be added to cdist config;
'``__cdist_preos_trigger http ...``' type is appended to initial manifest
**-y REMOTE_COPY, --remote-copy REMOTE_COPY**
remote copy that cdist config will use, by default
internal script is used
@ -604,6 +622,84 @@ usage. Its primary use is for debugging type parameters.
be POSIX compatible shell.
TRIGGER
-------
Start trigger (simple http server) that waits for connections. When host
connects then it triggers config or install command and then cdist
config/install is executed which configures/installs host.
When triggered cdist will try to reverse DNS lookup for host name and if
host name is dervied then it is used for running cdist config. If no
host name is resolved then IP address is used.
Request path recognizes following requests:
* :strong:`/cdist/config/.*` for config
* :strong:`/cdist/install/.*` for install.
This command returns the following response codes to client requests:
* 200 for success
* 599 for cdist run errors
* 500 for cdist/server errors.
**-6, --ipv6**
Listen to both IPv4 and IPv6 (instead of only IPv4)
**-b, --beta**
Enable beta functionality.
**-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN**
Sepcify custom cache path pattern. It can also be set by
CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then
default hostdir is used. For more info on format see
:strong:`CACHE PATH PATTERN FORMAT` below.
**-c CONF_DIR, --conf-dir CONF_DIR**
Add configuration directory (can be repeated, last one wins)
**-D DIRECTORY, --directory DIRECTORY**
Where to create local files
**-H HTTP_PORT, --http-port HTTP_PORT**
Create trigger listener via http on specified port
**-i MANIFEST, --initial-manifest MANIFEST**
path to a cdist manifest or '-' to read from stdin.
**-j [JOBS], --jobs [JOBS]**
Specify the maximum number of parallel jobs, currently
only global explorers are supported
**-n, --dry-run**
do not execute code
**-o OUT_PATH, --out-dir OUT_PATH**
directory to save cdist output in
**-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH**
Directory to save cdist output in on the target host
**--remote-copy REMOTE_COPY**
Command to use for remote copy (should behave like scp)
**--remote-exec REMOTE_EXEC**
Command to use for remote execution (should behave like ssh)
**-S SOURCE, --source SOURCE**
Which file to copy for creation
CONFIGURATION
-------------
cdist obtains configuration data from the following sources in the following
@ -798,20 +894,28 @@ EXAMPLES
# Configure all hosts from inventory db
$ cdist config -b -A
# Create default debian PreOS in debug mode
# Create default debian PreOS in debug mode with config
# trigger command
$ cdist preos debian /preos/preos-debian -vvvv -C \
-k ~/.ssh/id_rsa.pub -p /preos/pxe-debian
-k ~/.ssh/id_rsa.pub -p /preos/pxe-debian \
-t "/usr/bin/curl 192.168.111.5:3000/config/"
# Create ubuntu PreOS
# Create ubuntu PreOS with install trigger command
$ cdist preos ubuntu /preos/preos-ubuntu -C \
-k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu
-k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \
-t "/usr/bin/curl 192.168.111.5:3000/install/"
# Create ubuntu PreOS on drive /dev/sdb
# Create ubuntu PreOS on drive /dev/sdb with install trigger command
# and set root password to 'password'.
$ cdist preos ubuntu /mnt -B -C \
-k ~/.ssh/id_rsa.pub -D /dev/sdb \
-t "/usr/bin/curl 192.168.111.5:3000/install/" \
-P password
# Start trigger in verbose mode that will configure host using specified
# init manifest
% cdist trigger -v -i ~/.cdist/manifest/init-for-triggered
ENVIRONMENT
-----------