diff --git a/cdist/argparse.py b/cdist/argparse.py index cadac39a..f17315e7 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,19 +485,31 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger']) + choices=['scan', 'trigger', 'config']) + parser['scan'].add_argument( + '--list', + action='store_true', + help='List the known hosts and exit') parser['scan'].add_argument( '--config', action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interfaces', - action='append', default=[], + '-I', '--interface', + action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( - '-d', '--delay', - action='store', default=3600, - help='How long to wait before reconfiguring after last try') + '--name-mapper', + action='store', default=None, + help='Map addresses to names, required for config mode') + parser['scan'].add_argument( + '-d', '--config-delay', + action='store', default=3600, type=int, + help='How long (seconds) to wait before reconfiguring after last try') + parser['scan'].add_argument( + '-t', '--trigger-delay', + action='store', default=5, type=int, + help='How long (seconds) to wait between ICMPv6 echo requests') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index eca4cf13..ddbe4933 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,36 +20,98 @@ # import logging +import sys +from datetime import datetime log = logging.getLogger("scan") -# define this outside of the class to not handle scapy import errors by default -def commandline(args): - log.debug(args) - - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - print('cdist scan requires scapy to be installed') - +def run(scan, args): + # We run each component in a separate process since they + # must not block on each other. processes = [] - if not args.mode: - # By default scan and trigger, but do not call any action - args.mode = ['scan', 'trigger', ] - if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces) + t = scan.Trigger(interfaces=args.interface, + sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner( + autoconfigure='config' in args.mode, + interfaces=args.interface, + name_mapper=args.name_mapper) s.start() processes.append(s) log.debug("Scanner started") for process in processes: process.join() + + +def list(scan, args): + s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) + hosts = s.list() + + # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + + # 6 colons. + ipv6_max_size = 8 * 4 + 10 + date_max_size = len(datetime.now().strftime(scan.datetime_format)) + name_max_size = 25 + + print("{} | {} | {} | {}".format( + 'name'.ljust(name_max_size), + 'address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size), + 'last configured'.ljust(date_max_size))) + print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) + for host in hosts: + last_seen = host.last_seen() + if last_seen: + last_seen = last_seen.strftime(scan.datetime_format) + else: + last_seen = '-' + + last_configured = host.last_configured() + if last_configured is not None: + last_configured = last_configured.strftime(scan.datetime_format) + else: + last_configured = '-' + + print("{} | {} | {} | {}".format( + host.name(default='-').ljust(name_max_size), + host.address().ljust(ipv6_max_size), + last_seen.ljust(date_max_size), + last_configured.ljust(date_max_size))) + + +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). +def commandline(args): + log.debug(args) + + # Check if we have the optional scapy dependency available. + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + log.error('cdist scan requires scapy to be installed. Exiting.') + sys.exit(1) + + # Set default operation mode. + if not args.mode: + # By default scan and trigger, but do not call any action. + args.mode = ['scan', 'trigger', ] + + if 'config' in args.mode and args.name_mapper is None: + print('--name-mapper must be specified for scanner config mode.', + file=sys.stderr) + sys.exit(1) + + # Print known hosts and exit is --list is specified - do not start + # the scanner. + if args.list: + list(scan, args) + else: + run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index faee8a56..4a20f511 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,38 +19,6 @@ # # -# -# Interface to be implemented: -# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated -# scan: scan / listen for icmp6 replies -# trigger: send trigger to multicast -# config: configure newly detected hosts -# install: install newly detected hosts -# -# Scanner logic -# - save results to configdir: -# basedir = ~/.cdist/scan/ -# last_seen = ~/.cdist/scan//last_seen -- record unix time -# or similar -# last_configured = ~/.cdist/scan//last_configured -- record -# unix time or similar -# last_installed = ~/.cdist/scan//last_configured -- record -# unix time or similar -# -# -# -# -# cdist scan --list -# Show all known hosts including last seen flag -# -# Logic for reconfiguration: -# -# - record when configured last time -# - introduce a parameter --reconfigure-after that takes time argument -# - reconfigure if a) host alive and b) reconfigure-after time passed -# - - from multiprocessing import Process import os import logging @@ -61,7 +29,84 @@ import datetime import cdist.config +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") +datetime_format = '%Y-%m-%d %H:%M:%S' + + +class Host(object): + def __init__(self, addr, outdir, name_mapper=None): + self.addr = addr + self.workdir = os.path.join(outdir, addr) + self.name_mapper = name_mapper + + os.makedirs(self.workdir, exist_ok=True) + + def __get(self, key, default=None): + fname = os.path.join(self.workdir, key) + value = default + if os.path.isfile(fname): + with open(fname, "r") as fd: + value = fd.readline() + return value + + def __set(self, key, value): + fname = os.path.join(self.workdir, key) + with open(fname, "w") as fd: + fd.write(f"{value}") + + def name(self, default=None): + if self.name_mapper is None: + return default + + fpath = os.path.join(os.getcwd(), self.name_mapper) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: + value = out.stdout.decode() + return (default if len(value) == 0 else value) + else: + return default + + def address(self): + return self.addr + + def last_seen(self, default=None): + raw = self.__get('last_seen') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def last_configured(self, default=None): + raw = self.__get('last_configured') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def seen(self): + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_seen', now) + + # XXX: There's no easy way to use the config module without feeding it with + # CLI args. Might as well call everything from scratch! + def configure(self): + target = self.name() or self.address() + cmd = ['cdist', 'config', '-v', target] + + fname = os.path.join(self.workdir, 'last_configuration_log') + with open(fname, "w") as fd: + log.debug("Executing: %s", cmd) + completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) + if completed_process.returncode != 0: + log.error("%s return with non-zero code %i - see %s for \ + details.", cmd, completed_process.returncode, fname) + + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_configured', now) class Trigger(object): @@ -69,12 +114,14 @@ class Trigger(object): Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces=None, verbose=False): + def __init__(self, interfaces, sleeptime, verbose=False): self.interfaces = interfaces + + # Used by scapy / send in trigger/2. self.verbose = verbose - # Wait 5 seconds before triggering again - FIXME: add parameter - self.sleeptime = 5 + # Delay in seconds between sent ICMPv6EchoRequests. + self.sleeptime = sleeptime def start(self): self.processes = [] @@ -93,9 +140,14 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() - log.debug("Sending request on %s", interface) - send(packet, verbose=self.verbose) + try: + log.debug("Sending ICMPv6EchoRequest on %s", interface) + packet = IPv6( + dst="ff02::1%{}".format(interface) + ) / ICMPv6EchoRequest() + send(packet, verbose=self.verbose) + except Exception as e: + log.error("Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -103,41 +155,62 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces=None, args=None, outdir=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, + name_mapper=None): self.interfaces = interfaces + self.autoconfigure = autoconfigure + self.name_mapper = name_mapper + self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') + os.makedirs(self.outdir, exist_ok=True) + + self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = pkg['IPv6'].src - log.verbose("Host %s is alive", host) + host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) + if host.name(): + log.verbose("Host %s (%s) is alive", host.name(), + host.address()) + else: + log.verbose("Host %s is alive", host.address()) - dir = os.path.join(self.outdir, host) - fname = os.path.join(dir, "last_seen") + host.seen() - now = datetime.datetime.now() + # Configure if needed. + if self.autoconfigure and \ + host.last_configured(default=datetime.datetime.min) + \ + self.config_delay < datetime.datetime.now(): + self.config(host) - os.makedirs(dir, exist_ok=True) + def list(self): + hosts = [] + for addr in os.listdir(self.outdir): + hosts.append(Host(addr, self.outdir, self.name_mapper)) - # FIXME: maybe adjust the format so we can easily parse again - with open(fname, "w") as fd: - fd.write(f"{now}\n") + return hosts - def config(self): - """ - Configure a host + def config(self, host): + if host.name() is None: + log.debug("config - could not resolve name for %s, aborting.", + host.address()) + return - - Assume we are only called if necessary - - However we need to ensure to not run in parallel - - Maybe keep dict storing per host processes - - Save the result - - Save the output -> probably aligned to config mode + previous_config_process = self.running_configs.get(host.name()) + if previous_config_process is not None and \ + previous_config_process.is_alive(): + log.debug("config - is already running for %s, aborting.", + host.name()) - """ + log.info("config - running against host %s (%s).", host.name(), + host.address()) + p = Process(target=host.configure()) + p.start() + self.running_configs[host.name()] = p def start(self): self.process = Process(target=self.scan) @@ -148,47 +221,9 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - - -if __name__ == '__main__': - t = Trigger(interfaces=["wlan0"]) - t.start() - - # Scanner can listen on many interfaces at the same time - s = Scanner(interfaces=["wlan0"]) - s.scan() - - # Join back the trigger processes - t.join() - - # Test in my lan shows: - # [18:48] bridge:cdist% ls -1d fe80::* - # fe80::142d:f0a5:725b:1103 - # fe80::20d:b9ff:fe49:ac11 - # fe80::20d:b9ff:fe4c:547d - # fe80::219:d2ff:feb2:2e12 - # fe80::21b:fcff:feee:f446 - # fe80::21b:fcff:feee:f45c - # fe80::21b:fcff:feee:f4b1 - # fe80::21b:fcff:feee:f4ba - # fe80::21b:fcff:feee:f4bc - # fe80::21b:fcff:feee:f4c1 - # fe80::21d:72ff:fe86:46b - # fe80::42b0:34ff:fe6f:f6f0 - # fe80::42b0:34ff:fe6f:f863 - # fe80::42b0:34ff:fe6f:f9b2 - # fe80::4a5d:60ff:fea1:e55f - # fe80::77a3:5e3f:82cc:f2e5 - # fe80::9e93:4eff:fe6c:c1f4 - # fe80::ba69:f4ff:fec5:6041 - # fe80::ba69:f4ff:fec5:8db7 - # fe80::bad8:12ff:fe65:313d - # fe80::bad8:12ff:fe65:d9b1 - # fe80::ce2d:e0ff:fed4:2611 - # fe80::ce32:e5ff:fe79:7ea7 - # fe80::d66d:6dff:fe33:e00 - # fe80::e2ff:f7ff:fe00:20e6 - # fe80::f29f:c2ff:fe7c:275e + try: + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + except Exception as e: + log.error("Could not start listener: %s", e) diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst new file mode 100644 index 00000000..064e65ff --- /dev/null +++ b/docs/src/cdist-scan.rst @@ -0,0 +1,82 @@ +Scan +===== + +Description +----------- +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. It is especially useful in netboot-based environment where hosts +boot unconfigured, and to ensure your infrastructure stays in sync with your +configuration. + +This feature is still consider to be in **beta** stage, and only operate on +IPv6 (including link-local). + +Usage (Examples) +---------------- + +Discover hosts on local network and configure those whose name is resolved by +the name mapper script. + +.. code-block:: sh + + $ cdist scan --beta --interface eth0 \ + --mode scan --name-mapper path/to/script \ + --mode trigger --mode config + +List known hosts and exit. + +.. code-block:: sh + + $ cdist scan --beta --list --name-mapper path/to/script + +Please refer to `cdist(1)` for a detailed list of parameters. + +Modes +----- + +The scanner has 3 modes that can be independently toggled. If the `--mode` +parameter is not specified, only `tigger` and `scan` are enabled (= hosts are +not configured). + +trigger + Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to + trigger detection by the `scan` module. + +scan + Watch for incoming ICMPv6 replies and optionally configure detected hosts. + +config + Enable configuration of hosts detected by `scan`. + +Name Mapper Script +------------------ + +The name mapper script takes an IPv6 address as first argument and writes the +resolved name to stdout - if any. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + case "$1" in + "fe80::20d:b9ff:fe57:3524") + printf "my-host-01" + ;; + "fe80::7603:bdff:fe05:89bb") + printf "my-host-02" + ;; + esac + +Resolving name from `PTR` DNS record: + +.. code-block:: sh + #!/bin/sh + + for cmd in dig sed; do + if ! command -v $cmd > /dev/null; then + exit 1 + fi + done + + dig +short -x "$1" | sed -e 's/.$//' diff --git a/docs/src/index.rst b/docs/src/index.rst index 31c044dc..369d5309 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,6 +34,7 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos + cdist-scan cdist-integration cdist-reference cdist-best-practice diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 0ecb4a61..599ec3b7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,6 +88,9 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] + [-d CONFIG_DELAY] [-t TRIGGER_DELAY] + DESCRIPTION ----------- @@ -641,6 +644,31 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. +SCAN +---- + +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. + +**-I INTERFACE, --interfaces INTERFACE** + Interface to listen on. Can be specified multiple times. + +**-m MODE, --mode MODE** + Scanner components to enable. Can be specified multiple time to enable more + than one component. Supported modes are: scan, trigger and config. Defaults + to tiggger and scan. + +**--name-mapper PATH_TO_SCRIPT** + Path to script used to resolve a remote host name from an IPv6 address. + +**--list** + List known hosts and exit. + +**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** + How long (seconds) to wait before reconfiguring after last try (config mode only). + +**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** + How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION -------------