Merge branch 'scanner' into 'master'

usable cdist scan

See merge request ungleich-public/cdist!993

cdist/argparse.py

@@ -485,19 +485,31 @@ def get_parsers():
     parser['scan'].add_argument(
         '-m', '--mode', help='Which modes should run',
         action='append', default=[],
-        choices=['scan', 'trigger'])
+        choices=['scan', 'trigger', 'config'])
+    parser['scan'].add_argument(
+        '--list',
+        action='store_true',
+        help='List the known hosts and exit')
     parser['scan'].add_argument(
         '--config',
         action='store_true',
         help='Try to configure detected hosts')
     parser['scan'].add_argument(
-        '-I', '--interfaces',
+        '-I', '--interface',
-        action='append',  default=[],
+        action='append',  default=[], required=True,
         help='On which interfaces to scan/trigger')
     parser['scan'].add_argument(
-        '-d', '--delay',
+        '--name-mapper',
-        action='store',  default=3600,
+        action='store',  default=None,
-        help='How long to wait before reconfiguring after last try')
+        help='Map addresses to names, required for config mode')
+    parser['scan'].add_argument(
+        '-d', '--config-delay',
+        action='store',  default=3600, type=int,
+        help='How long (seconds) to wait before reconfiguring after last try')
+    parser['scan'].add_argument(
+        '-t', '--trigger-delay',
+        action='store',  default=5, type=int,
+        help='How long (seconds) to wait between ICMPv6 echo requests')
     parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
 
     for p in parser:

cdist/scan/commandline.py

@@ -20,36 +20,98 @@
 #
 
 import logging
+import sys
+from datetime import datetime
 
 log = logging.getLogger("scan")
 
 
-# define this outside of the class to not handle scapy import errors by default
+def run(scan, args):
-def commandline(args):
+    # We run each component in a separate process since they
-    log.debug(args)
+    # must not block on each other.
-
-    try:
-        import cdist.scan.scan as scan
-    except ModuleNotFoundError:
-        print('cdist scan requires scapy to be installed')
-
     processes = []
 
-    if not args.mode:
-        # By default scan and trigger, but do not call any action
-        args.mode = ['scan', 'trigger', ]
-
     if 'trigger' in args.mode:
-        t = scan.Trigger(interfaces=args.interfaces)
+        t = scan.Trigger(interfaces=args.interface,
+                         sleeptime=args.trigger_delay)
         t.start()
         processes.append(t)
         log.debug("Trigger started")
 
     if 'scan' in args.mode:
-        s = scan.Scanner(interfaces=args.interfaces, args=args)
+        s = scan.Scanner(
+                autoconfigure='config' in args.mode,
+                interfaces=args.interface,
+                name_mapper=args.name_mapper)
         s.start()
         processes.append(s)
         log.debug("Scanner started")
 
     for process in processes:
         process.join()
+
+
+def list(scan, args):
+    s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper)
+    hosts = s.list()
+
+    # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
+    # 6 colons.
+    ipv6_max_size = 8 * 4 + 10
+    date_max_size = len(datetime.now().strftime(scan.datetime_format))
+    name_max_size = 25
+
+    print("{} | {} | {} | {}".format(
+        'name'.ljust(name_max_size),
+        'address'.ljust(ipv6_max_size),
+        'last seen'.ljust(date_max_size),
+        'last configured'.ljust(date_max_size)))
+    print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
+    for host in hosts:
+        last_seen = host.last_seen()
+        if last_seen:
+            last_seen = last_seen.strftime(scan.datetime_format)
+        else:
+            last_seen = '-'
+
+        last_configured = host.last_configured()
+        if last_configured is not None:
+            last_configured = last_configured.strftime(scan.datetime_format)
+        else:
+            last_configured = '-'
+
+        print("{} | {} | {} | {}".format(
+            host.name(default='-').ljust(name_max_size),
+            host.address().ljust(ipv6_max_size),
+            last_seen.ljust(date_max_size),
+            last_configured.ljust(date_max_size)))
+
+
+# CLI processing is defined outside of the main scan class to handle
+# non-available optional scapy dependency (instead of crashing mid-flight).
+def commandline(args):
+    log.debug(args)
+
+    # Check if we have the optional scapy dependency available.
+    try:
+        import cdist.scan.scan as scan
+    except ModuleNotFoundError:
+        log.error('cdist scan requires scapy to be installed. Exiting.')
+        sys.exit(1)
+
+    # Set default operation mode.
+    if not args.mode:
+        # By default scan and trigger, but do not call any action.
+        args.mode = ['scan', 'trigger', ]
+
+    if 'config' in args.mode and args.name_mapper is None:
+        print('--name-mapper must be specified for scanner config mode.',
+              file=sys.stderr)
+        sys.exit(1)
+
+    # Print known hosts and exit is --list is specified - do not start
+    # the scanner.
+    if args.list:
+        list(scan, args)
+    else:
+        run(scan, args)

cdist/scan/scan.py

@@ -19,38 +19,6 @@
 #
 #
 
-#
-# Interface to be implemented:
-# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
-#   scan: scan / listen for icmp6 replies
-#   trigger: send trigger to multicast
-#   config: configure newly detected hosts
-#   install: install newly detected hosts
-#
-# Scanner logic
-#  - save results to configdir:
-#     basedir = ~/.cdist/scan/<ipv6-address>
-#     last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
-#           or similar
-#     last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#     last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
-#           unix time or similar
-#
-#
-#
-#
-# cdist scan --list
-#       Show all known hosts including last seen flag
-#
-# Logic for reconfiguration:
-#
-#  - record when configured last time
-#  - introduce a parameter --reconfigure-after that takes time argument
-#  - reconfigure if a) host alive and b) reconfigure-after time passed
-#
-
-
 from multiprocessing import Process
 import os
 import logging
@@ -61,7 +29,84 @@ import datetime
 
 import cdist.config
 
+logging.basicConfig(level=logging.DEBUG)
 log = logging.getLogger("scan")
+datetime_format = '%Y-%m-%d %H:%M:%S'
+
+
+class Host(object):
+    def __init__(self, addr, outdir, name_mapper=None):
+        self.addr = addr
+        self.workdir = os.path.join(outdir, addr)
+        self.name_mapper = name_mapper
+
+        os.makedirs(self.workdir, exist_ok=True)
+
+    def __get(self, key, default=None):
+        fname = os.path.join(self.workdir, key)
+        value = default
+        if os.path.isfile(fname):
+            with open(fname, "r") as fd:
+                value = fd.readline()
+        return value
+
+    def __set(self, key, value):
+        fname = os.path.join(self.workdir, key)
+        with open(fname, "w") as fd:
+            fd.write(f"{value}")
+
+    def name(self, default=None):
+        if self.name_mapper is None:
+            return default
+
+        fpath = os.path.join(os.getcwd(), self.name_mapper)
+        if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
+            out = subprocess.run([fpath, self.addr], capture_output=True)
+            if out.returncode != 0:
+                return default
+            else:
+                value = out.stdout.decode()
+                return (default if len(value) == 0 else value)
+        else:
+            return default
+
+    def address(self):
+        return self.addr
+
+    def last_seen(self, default=None):
+        raw = self.__get('last_seen')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def last_configured(self, default=None):
+        raw = self.__get('last_configured')
+        if raw:
+            return datetime.datetime.strptime(raw, datetime_format)
+        else:
+            return default
+
+    def seen(self):
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_seen', now)
+
+    # XXX: There's no easy way to use the config module without feeding it with
+    # CLI args. Might as well call everything from scratch!
+    def configure(self):
+        target = self.name() or self.address()
+        cmd = ['cdist', 'config', '-v', target]
+
+        fname = os.path.join(self.workdir, 'last_configuration_log')
+        with open(fname, "w") as fd:
+            log.debug("Executing: %s", cmd)
+            completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
+            if completed_process.returncode != 0:
+                log.error("%s return with non-zero code %i - see %s for \
+                        details.", cmd, completed_process.returncode, fname)
+
+        now = datetime.datetime.now().strftime(datetime_format)
+        self.__set('last_configured', now)
 
 
 class Trigger(object):
@@ -69,12 +114,14 @@ class Trigger(object):
     Trigger an ICMPv6EchoReply from all hosts that are alive
     """
 
-    def __init__(self, interfaces=None, verbose=False):
+    def __init__(self, interfaces, sleeptime, verbose=False):
         self.interfaces = interfaces
+
+        # Used by scapy / send in trigger/2.
         self.verbose = verbose
 
-        # Wait 5 seconds before triggering again - FIXME: add parameter
+        # Delay in seconds between sent ICMPv6EchoRequests.
-        self.sleeptime = 5
+        self.sleeptime = sleeptime
 
     def start(self):
         self.processes = []
@@ -93,9 +140,14 @@ class Trigger(object):
             time.sleep(self.sleeptime)
 
     def trigger(self, interface):
-        packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
+        try:
-        log.debug("Sending request on %s", interface)
+            log.debug("Sending ICMPv6EchoRequest on %s", interface)
-        send(packet, verbose=self.verbose)
+            packet = IPv6(
+                    dst="ff02::1%{}".format(interface)
+                    ) / ICMPv6EchoRequest()
+            send(packet, verbose=self.verbose)
+        except Exception as e:
+            log.error("Could not send ICMPv6EchoRequest: %s", e)
 
 
 class Scanner(object):
@@ -103,41 +155,62 @@ class Scanner(object):
     Scan for replies of hosts, maintain the up-to-date database
     """
 
-    def __init__(self, interfaces=None, args=None, outdir=None):
+    def __init__(self, interfaces, autoconfigure=False, outdir=None,
+                 name_mapper=None):
         self.interfaces = interfaces
+        self.autoconfigure = autoconfigure
+        self.name_mapper = name_mapper
+        self.config_delay = datetime.timedelta(seconds=3600)
 
         if outdir:
             self.outdir = outdir
         else:
             self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
+        os.makedirs(self.outdir, exist_ok=True)
+
+        self.running_configs = {}
 
     def handle_pkg(self, pkg):
         if ICMPv6EchoReply in pkg:
-            host = pkg['IPv6'].src
+            host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
-            log.verbose("Host %s is alive", host)
+            if host.name():
+                log.verbose("Host %s (%s) is alive", host.name(),
+                            host.address())
+            else:
+                log.verbose("Host %s is alive", host.address())
 
-            dir = os.path.join(self.outdir, host)
+            host.seen()
-            fname = os.path.join(dir, "last_seen")
 
-            now = datetime.datetime.now()
+            # Configure if needed.
+            if self.autoconfigure and \
+                    host.last_configured(default=datetime.datetime.min) + \
+                    self.config_delay < datetime.datetime.now():
+                self.config(host)
 
-            os.makedirs(dir, exist_ok=True)
+    def list(self):
+        hosts = []
+        for addr in os.listdir(self.outdir):
+            hosts.append(Host(addr, self.outdir, self.name_mapper))
 
-            # FIXME: maybe adjust the format so we can easily parse again
+        return hosts
-            with open(fname, "w") as fd:
-                fd.write(f"{now}\n")
 
-    def config(self):
+    def config(self, host):
-        """
+        if host.name() is None:
-        Configure a host
+            log.debug("config - could not resolve name for %s, aborting.",
+                      host.address())
+            return
 
-        - Assume we are only called if necessary
+        previous_config_process = self.running_configs.get(host.name())
-        - However we need to ensure to not run in parallel
+        if previous_config_process is not None and \
-        - Maybe keep dict storing per host processes
+                previous_config_process.is_alive():
-        - Save the result
+            log.debug("config - is already running for %s, aborting.",
-        - Save the output -> probably aligned to config mode
+                      host.name())
 
-        """
+        log.info("config - running against host %s (%s).", host.name(),
+                 host.address())
+        p = Process(target=host.configure())
+        p.start()
+        self.running_configs[host.name()] = p
 
     def start(self):
         self.process = Process(target=self.scan)
@@ -148,47 +221,9 @@ class Scanner(object):
 
     def scan(self):
         log.debug("Scanning - zzzzz")
-        sniff(iface=self.interfaces,
+        try:
-              filter="icmp6",
+            sniff(iface=self.interfaces,
-              prn=self.handle_pkg)
+                  filter="icmp6",
-
+                  prn=self.handle_pkg)
-
+        except Exception as e:
-if __name__ == '__main__':
+            log.error("Could not start listener: %s", e)
-    t = Trigger(interfaces=["wlan0"])
-    t.start()
-
-    # Scanner can listen on many interfaces at the same time
-    s = Scanner(interfaces=["wlan0"])
-    s.scan()
-
-    # Join back the trigger processes
-    t.join()
-
-    # Test in my lan shows:
-    # [18:48] bridge:cdist% ls -1d fe80::*
-    # fe80::142d:f0a5:725b:1103
-    # fe80::20d:b9ff:fe49:ac11
-    # fe80::20d:b9ff:fe4c:547d
-    # fe80::219:d2ff:feb2:2e12
-    # fe80::21b:fcff:feee:f446
-    # fe80::21b:fcff:feee:f45c
-    # fe80::21b:fcff:feee:f4b1
-    # fe80::21b:fcff:feee:f4ba
-    # fe80::21b:fcff:feee:f4bc
-    # fe80::21b:fcff:feee:f4c1
-    # fe80::21d:72ff:fe86:46b
-    # fe80::42b0:34ff:fe6f:f6f0
-    # fe80::42b0:34ff:fe6f:f863
-    # fe80::42b0:34ff:fe6f:f9b2
-    # fe80::4a5d:60ff:fea1:e55f
-    # fe80::77a3:5e3f:82cc:f2e5
-    # fe80::9e93:4eff:fe6c:c1f4
-    # fe80::ba69:f4ff:fec5:6041
-    # fe80::ba69:f4ff:fec5:8db7
-    # fe80::bad8:12ff:fe65:313d
-    # fe80::bad8:12ff:fe65:d9b1
-    # fe80::ce2d:e0ff:fed4:2611
-    # fe80::ce32:e5ff:fe79:7ea7
-    # fe80::d66d:6dff:fe33:e00
-    # fe80::e2ff:f7ff:fe00:20e6
-    # fe80::f29f:c2ff:fe7c:275e

docs/src/cdist-scan.rst

@@ -0,0 +1,82 @@
+Scan
+=====
+
+Description
+-----------
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically. It is especially useful in netboot-based environment where hosts
+boot unconfigured, and to ensure your infrastructure stays in sync with your
+configuration.
+
+This feature is still consider to be in **beta** stage, and only operate on
+IPv6 (including link-local).
+
+Usage (Examples)
+----------------
+
+Discover hosts on local network and configure those whose name is resolved by
+the name mapper script.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --interface eth0 \
+      --mode scan --name-mapper path/to/script \
+      --mode trigger --mode config
+
+List known hosts and exit.
+
+.. code-block:: sh
+
+    $ cdist scan --beta --list --name-mapper path/to/script
+
+Please refer to `cdist(1)` for a detailed list of parameters.
+
+Modes
+-----
+
+The scanner has 3 modes that can be independently toggled. If the `--mode`
+parameter is not specified, only `tigger` and `scan` are enabled (= hosts are
+not configured).
+
+trigger
+  Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to
+  trigger detection by the `scan` module.
+
+scan
+  Watch for incoming ICMPv6 replies and optionally configure detected hosts.
+
+config
+  Enable configuration of hosts detected by `scan`.
+
+Name Mapper Script
+------------------
+
+The name mapper script takes an IPv6 address as first argument and writes the
+resolved name to stdout - if any. The script must be executable.
+
+Simplest script:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  case "$1" in
+  	"fe80::20d:b9ff:fe57:3524")
+  		printf "my-host-01"
+  		;;
+  	"fe80::7603:bdff:fe05:89bb")
+  		printf "my-host-02"
+  		;;
+  esac
+
+Resolving name from `PTR` DNS record:
+
+.. code-block:: sh
+  #!/bin/sh
+
+  for cmd in dig sed; do
+  	if ! command -v $cmd > /dev/null; then
+  		exit 1
+  	fi
+  done
+
+  dig +short -x "$1" | sed -e 's/.$//'

docs/src/index.rst

@@ -34,6 +34,7 @@ It natively supports IPv6 since the first release.
    cdist-parallelization
    cdist-inventory
    cdist-preos
+   cdist-scan
    cdist-integration
    cdist-reference
    cdist-best-practice

docs/src/man1/cdist.rst

@@ -88,6 +88,9 @@ SYNOPSIS
     cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
                [pattern]
 
+    cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list]
+               [-d CONFIG_DELAY] [-t TRIGGER_DELAY]
+
 
 DESCRIPTION
 -----------
@@ -641,6 +644,31 @@ Display information for cdist (global explorers, types).
 **-t, --types**
     Display info for types.
 
+SCAN
+----
+
+Runs cdist as a daemon that discover/watch on hosts and reconfigure them
+periodically.
+
+**-I INTERFACE, --interfaces INTERFACE**
+    Interface to listen on. Can be specified multiple times.
+
+**-m MODE, --mode MODE**
+    Scanner components to enable. Can be specified multiple time to enable more
+    than one component. Supported modes are: scan, trigger and config. Defaults
+    to tiggger and scan.
+
+**--name-mapper PATH_TO_SCRIPT**
+    Path to script used to resolve a remote host name from an IPv6 address.
+
+**--list**
+    List known hosts and exit.
+
+**-d CONFIG_DELAY, --config-delay CONFIG_DELAY**
+    How long (seconds) to wait before reconfiguring after last try (config mode only).
+
+**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY**
+    How long (seconds) to wait between ICMPv6 echo requests (trigger mode only).
 
 CONFIGURATION
 -------------