Merge branch 'scanner' into 'master'
usable cdist scan See merge request ungleich-public/cdist!993
This commit is contained in:
fnux
commit
44eeb4bbfc
6 changed files with 342 additions and 122 deletions
|
|
@ -485,19 +485,31 @@ def get_parsers():
|
|||
parser['scan'].add_argument(
|
||||
'-m', '--mode', help='Which modes should run',
|
||||
action='append', default=[],
|
||||
choices=['scan', 'trigger'])
|
||||
choices=['scan', 'trigger', 'config'])
|
||||
parser['scan'].add_argument(
|
||||
'--list',
|
||||
action='store_true',
|
||||
help='List the known hosts and exit')
|
||||
parser['scan'].add_argument(
|
||||
'--config',
|
||||
action='store_true',
|
||||
help='Try to configure detected hosts')
|
||||
parser['scan'].add_argument(
|
||||
'-I', '--interfaces',
|
||||
action='append', default=[],
|
||||
'-I', '--interface',
|
||||
action='append', default=[], required=True,
|
||||
help='On which interfaces to scan/trigger')
|
||||
parser['scan'].add_argument(
|
||||
'-d', '--delay',
|
||||
action='store', default=3600,
|
||||
help='How long to wait before reconfiguring after last try')
|
||||
'--name-mapper',
|
||||
action='store', default=None,
|
||||
help='Map addresses to names, required for config mode')
|
||||
parser['scan'].add_argument(
|
||||
'-d', '--config-delay',
|
||||
action='store', default=3600, type=int,
|
||||
help='How long (seconds) to wait before reconfiguring after last try')
|
||||
parser['scan'].add_argument(
|
||||
'-t', '--trigger-delay',
|
||||
action='store', default=5, type=int,
|
||||
help='How long (seconds) to wait between ICMPv6 echo requests')
|
||||
parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
|
||||
|
||||
for p in parser:
|
||||
|
|
|
|||
|
|
@ -20,36 +20,98 @@
|
|||
#
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
log = logging.getLogger("scan")
|
||||
|
||||
|
||||
# define this outside of the class to not handle scapy import errors by default
|
||||
def commandline(args):
|
||||
log.debug(args)
|
||||
|
||||
try:
|
||||
import cdist.scan.scan as scan
|
||||
except ModuleNotFoundError:
|
||||
print('cdist scan requires scapy to be installed')
|
||||
|
||||
def run(scan, args):
|
||||
# We run each component in a separate process since they
|
||||
# must not block on each other.
|
||||
processes = []
|
||||
|
||||
if not args.mode:
|
||||
# By default scan and trigger, but do not call any action
|
||||
args.mode = ['scan', 'trigger', ]
|
||||
|
||||
if 'trigger' in args.mode:
|
||||
t = scan.Trigger(interfaces=args.interfaces)
|
||||
t = scan.Trigger(interfaces=args.interface,
|
||||
sleeptime=args.trigger_delay)
|
||||
t.start()
|
||||
processes.append(t)
|
||||
log.debug("Trigger started")
|
||||
|
||||
if 'scan' in args.mode:
|
||||
s = scan.Scanner(interfaces=args.interfaces, args=args)
|
||||
s = scan.Scanner(
|
||||
autoconfigure='config' in args.mode,
|
||||
interfaces=args.interface,
|
||||
name_mapper=args.name_mapper)
|
||||
s.start()
|
||||
processes.append(s)
|
||||
log.debug("Scanner started")
|
||||
|
||||
for process in processes:
|
||||
process.join()
|
||||
|
||||
|
||||
def list(scan, args):
|
||||
s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper)
|
||||
hosts = s.list()
|
||||
|
||||
# A full IPv6 addresses id composed of 8 blocks of 4 hexa chars +
|
||||
# 6 colons.
|
||||
ipv6_max_size = 8 * 4 + 10
|
||||
date_max_size = len(datetime.now().strftime(scan.datetime_format))
|
||||
name_max_size = 25
|
||||
|
||||
print("{} | {} | {} | {}".format(
|
||||
'name'.ljust(name_max_size),
|
||||
'address'.ljust(ipv6_max_size),
|
||||
'last seen'.ljust(date_max_size),
|
||||
'last configured'.ljust(date_max_size)))
|
||||
print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size)))
|
||||
for host in hosts:
|
||||
last_seen = host.last_seen()
|
||||
if last_seen:
|
||||
last_seen = last_seen.strftime(scan.datetime_format)
|
||||
else:
|
||||
last_seen = '-'
|
||||
|
||||
last_configured = host.last_configured()
|
||||
if last_configured is not None:
|
||||
last_configured = last_configured.strftime(scan.datetime_format)
|
||||
else:
|
||||
last_configured = '-'
|
||||
|
||||
print("{} | {} | {} | {}".format(
|
||||
host.name(default='-').ljust(name_max_size),
|
||||
host.address().ljust(ipv6_max_size),
|
||||
last_seen.ljust(date_max_size),
|
||||
last_configured.ljust(date_max_size)))
|
||||
|
||||
|
||||
# CLI processing is defined outside of the main scan class to handle
|
||||
# non-available optional scapy dependency (instead of crashing mid-flight).
|
||||
def commandline(args):
|
||||
log.debug(args)
|
||||
|
||||
# Check if we have the optional scapy dependency available.
|
||||
try:
|
||||
import cdist.scan.scan as scan
|
||||
except ModuleNotFoundError:
|
||||
log.error('cdist scan requires scapy to be installed. Exiting.')
|
||||
sys.exit(1)
|
||||
|
||||
# Set default operation mode.
|
||||
if not args.mode:
|
||||
# By default scan and trigger, but do not call any action.
|
||||
args.mode = ['scan', 'trigger', ]
|
||||
|
||||
if 'config' in args.mode and args.name_mapper is None:
|
||||
print('--name-mapper must be specified for scanner config mode.',
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Print known hosts and exit is --list is specified - do not start
|
||||
# the scanner.
|
||||
if args.list:
|
||||
list(scan, args)
|
||||
else:
|
||||
run(scan, args)
|
||||
|
|
|
|||
|
|
@ -19,38 +19,6 @@
|
|||
#
|
||||
#
|
||||
|
||||
#
|
||||
# Interface to be implemented:
|
||||
# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated
|
||||
# scan: scan / listen for icmp6 replies
|
||||
# trigger: send trigger to multicast
|
||||
# config: configure newly detected hosts
|
||||
# install: install newly detected hosts
|
||||
#
|
||||
# Scanner logic
|
||||
# - save results to configdir:
|
||||
# basedir = ~/.cdist/scan/<ipv6-address>
|
||||
# last_seen = ~/.cdist/scan/<ipv6-address>/last_seen -- record unix time
|
||||
# or similar
|
||||
# last_configured = ~/.cdist/scan/<ipv6-address>/last_configured -- record
|
||||
# unix time or similar
|
||||
# last_installed = ~/.cdist/scan/<ipv6-address>/last_configured -- record
|
||||
# unix time or similar
|
||||
#
|
||||
#
|
||||
#
|
||||
#
|
||||
# cdist scan --list
|
||||
# Show all known hosts including last seen flag
|
||||
#
|
||||
# Logic for reconfiguration:
|
||||
#
|
||||
# - record when configured last time
|
||||
# - introduce a parameter --reconfigure-after that takes time argument
|
||||
# - reconfigure if a) host alive and b) reconfigure-after time passed
|
||||
#
|
||||
|
||||
|
||||
from multiprocessing import Process
|
||||
import os
|
||||
import logging
|
||||
|
|
@ -61,7 +29,84 @@ import datetime
|
|||
|
||||
import cdist.config
|
||||
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
log = logging.getLogger("scan")
|
||||
datetime_format = '%Y-%m-%d %H:%M:%S'
|
||||
|
||||
|
||||
class Host(object):
|
||||
def __init__(self, addr, outdir, name_mapper=None):
|
||||
self.addr = addr
|
||||
self.workdir = os.path.join(outdir, addr)
|
||||
self.name_mapper = name_mapper
|
||||
|
||||
os.makedirs(self.workdir, exist_ok=True)
|
||||
|
||||
def __get(self, key, default=None):
|
||||
fname = os.path.join(self.workdir, key)
|
||||
value = default
|
||||
if os.path.isfile(fname):
|
||||
with open(fname, "r") as fd:
|
||||
value = fd.readline()
|
||||
return value
|
||||
|
||||
def __set(self, key, value):
|
||||
fname = os.path.join(self.workdir, key)
|
||||
with open(fname, "w") as fd:
|
||||
fd.write(f"{value}")
|
||||
|
||||
def name(self, default=None):
|
||||
if self.name_mapper is None:
|
||||
return default
|
||||
|
||||
fpath = os.path.join(os.getcwd(), self.name_mapper)
|
||||
if os.path.isfile(fpath) and os.access(fpath, os.X_OK):
|
||||
out = subprocess.run([fpath, self.addr], capture_output=True)
|
||||
if out.returncode != 0:
|
||||
return default
|
||||
else:
|
||||
value = out.stdout.decode()
|
||||
return (default if len(value) == 0 else value)
|
||||
else:
|
||||
return default
|
||||
|
||||
def address(self):
|
||||
return self.addr
|
||||
|
||||
def last_seen(self, default=None):
|
||||
raw = self.__get('last_seen')
|
||||
if raw:
|
||||
return datetime.datetime.strptime(raw, datetime_format)
|
||||
else:
|
||||
return default
|
||||
|
||||
def last_configured(self, default=None):
|
||||
raw = self.__get('last_configured')
|
||||
if raw:
|
||||
return datetime.datetime.strptime(raw, datetime_format)
|
||||
else:
|
||||
return default
|
||||
|
||||
def seen(self):
|
||||
now = datetime.datetime.now().strftime(datetime_format)
|
||||
self.__set('last_seen', now)
|
||||
|
||||
# XXX: There's no easy way to use the config module without feeding it with
|
||||
# CLI args. Might as well call everything from scratch!
|
||||
def configure(self):
|
||||
target = self.name() or self.address()
|
||||
cmd = ['cdist', 'config', '-v', target]
|
||||
|
||||
fname = os.path.join(self.workdir, 'last_configuration_log')
|
||||
with open(fname, "w") as fd:
|
||||
log.debug("Executing: %s", cmd)
|
||||
completed_process = subprocess.run(cmd, stdout=fd, stderr=fd)
|
||||
if completed_process.returncode != 0:
|
||||
log.error("%s return with non-zero code %i - see %s for \
|
||||
details.", cmd, completed_process.returncode, fname)
|
||||
|
||||
now = datetime.datetime.now().strftime(datetime_format)
|
||||
self.__set('last_configured', now)
|
||||
|
||||
|
||||
class Trigger(object):
|
||||
|
|
@ -69,12 +114,14 @@ class Trigger(object):
|
|||
Trigger an ICMPv6EchoReply from all hosts that are alive
|
||||
"""
|
||||
|
||||
def __init__(self, interfaces=None, verbose=False):
|
||||
def __init__(self, interfaces, sleeptime, verbose=False):
|
||||
self.interfaces = interfaces
|
||||
|
||||
# Used by scapy / send in trigger/2.
|
||||
self.verbose = verbose
|
||||
|
||||
# Wait 5 seconds before triggering again - FIXME: add parameter
|
||||
self.sleeptime = 5
|
||||
# Delay in seconds between sent ICMPv6EchoRequests.
|
||||
self.sleeptime = sleeptime
|
||||
|
||||
def start(self):
|
||||
self.processes = []
|
||||
|
|
@ -93,9 +140,14 @@ class Trigger(object):
|
|||
time.sleep(self.sleeptime)
|
||||
|
||||
def trigger(self, interface):
|
||||
packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest()
|
||||
log.debug("Sending request on %s", interface)
|
||||
send(packet, verbose=self.verbose)
|
||||
try:
|
||||
log.debug("Sending ICMPv6EchoRequest on %s", interface)
|
||||
packet = IPv6(
|
||||
dst="ff02::1%{}".format(interface)
|
||||
) / ICMPv6EchoRequest()
|
||||
send(packet, verbose=self.verbose)
|
||||
except Exception as e:
|
||||
log.error("Could not send ICMPv6EchoRequest: %s", e)
|
||||
|
||||
|
||||
class Scanner(object):
|
||||
|
|
@ -103,41 +155,62 @@ class Scanner(object):
|
|||
Scan for replies of hosts, maintain the up-to-date database
|
||||
"""
|
||||
|
||||
def __init__(self, interfaces=None, args=None, outdir=None):
|
||||
def __init__(self, interfaces, autoconfigure=False, outdir=None,
|
||||
name_mapper=None):
|
||||
self.interfaces = interfaces
|
||||
self.autoconfigure = autoconfigure
|
||||
self.name_mapper = name_mapper
|
||||
self.config_delay = datetime.timedelta(seconds=3600)
|
||||
|
||||
if outdir:
|
||||
self.outdir = outdir
|
||||
else:
|
||||
self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan')
|
||||
os.makedirs(self.outdir, exist_ok=True)
|
||||
|
||||
self.running_configs = {}
|
||||
|
||||
def handle_pkg(self, pkg):
|
||||
if ICMPv6EchoReply in pkg:
|
||||
host = pkg['IPv6'].src
|
||||
log.verbose("Host %s is alive", host)
|
||||
host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper)
|
||||
if host.name():
|
||||
log.verbose("Host %s (%s) is alive", host.name(),
|
||||
host.address())
|
||||
else:
|
||||
log.verbose("Host %s is alive", host.address())
|
||||
|
||||
dir = os.path.join(self.outdir, host)
|
||||
fname = os.path.join(dir, "last_seen")
|
||||
host.seen()
|
||||
|
||||
now = datetime.datetime.now()
|
||||
# Configure if needed.
|
||||
if self.autoconfigure and \
|
||||
host.last_configured(default=datetime.datetime.min) + \
|
||||
self.config_delay < datetime.datetime.now():
|
||||
self.config(host)
|
||||
|
||||
os.makedirs(dir, exist_ok=True)
|
||||
def list(self):
|
||||
hosts = []
|
||||
for addr in os.listdir(self.outdir):
|
||||
hosts.append(Host(addr, self.outdir, self.name_mapper))
|
||||
|
||||
# FIXME: maybe adjust the format so we can easily parse again
|
||||
with open(fname, "w") as fd:
|
||||
fd.write(f"{now}\n")
|
||||
return hosts
|
||||
|
||||
def config(self):
|
||||
"""
|
||||
Configure a host
|
||||
def config(self, host):
|
||||
if host.name() is None:
|
||||
log.debug("config - could not resolve name for %s, aborting.",
|
||||
host.address())
|
||||
return
|
||||
|
||||
- Assume we are only called if necessary
|
||||
- However we need to ensure to not run in parallel
|
||||
- Maybe keep dict storing per host processes
|
||||
- Save the result
|
||||
- Save the output -> probably aligned to config mode
|
||||
previous_config_process = self.running_configs.get(host.name())
|
||||
if previous_config_process is not None and \
|
||||
previous_config_process.is_alive():
|
||||
log.debug("config - is already running for %s, aborting.",
|
||||
host.name())
|
||||
|
||||
"""
|
||||
log.info("config - running against host %s (%s).", host.name(),
|
||||
host.address())
|
||||
p = Process(target=host.configure())
|
||||
p.start()
|
||||
self.running_configs[host.name()] = p
|
||||
|
||||
def start(self):
|
||||
self.process = Process(target=self.scan)
|
||||
|
|
@ -148,47 +221,9 @@ class Scanner(object):
|
|||
|
||||
def scan(self):
|
||||
log.debug("Scanning - zzzzz")
|
||||
sniff(iface=self.interfaces,
|
||||
filter="icmp6",
|
||||
prn=self.handle_pkg)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
t = Trigger(interfaces=["wlan0"])
|
||||
t.start()
|
||||
|
||||
# Scanner can listen on many interfaces at the same time
|
||||
s = Scanner(interfaces=["wlan0"])
|
||||
s.scan()
|
||||
|
||||
# Join back the trigger processes
|
||||
t.join()
|
||||
|
||||
# Test in my lan shows:
|
||||
# [18:48] bridge:cdist% ls -1d fe80::*
|
||||
# fe80::142d:f0a5:725b:1103
|
||||
# fe80::20d:b9ff:fe49:ac11
|
||||
# fe80::20d:b9ff:fe4c:547d
|
||||
# fe80::219:d2ff:feb2:2e12
|
||||
# fe80::21b:fcff:feee:f446
|
||||
# fe80::21b:fcff:feee:f45c
|
||||
# fe80::21b:fcff:feee:f4b1
|
||||
# fe80::21b:fcff:feee:f4ba
|
||||
# fe80::21b:fcff:feee:f4bc
|
||||
# fe80::21b:fcff:feee:f4c1
|
||||
# fe80::21d:72ff:fe86:46b
|
||||
# fe80::42b0:34ff:fe6f:f6f0
|
||||
# fe80::42b0:34ff:fe6f:f863
|
||||
# fe80::42b0:34ff:fe6f:f9b2
|
||||
# fe80::4a5d:60ff:fea1:e55f
|
||||
# fe80::77a3:5e3f:82cc:f2e5
|
||||
# fe80::9e93:4eff:fe6c:c1f4
|
||||
# fe80::ba69:f4ff:fec5:6041
|
||||
# fe80::ba69:f4ff:fec5:8db7
|
||||
# fe80::bad8:12ff:fe65:313d
|
||||
# fe80::bad8:12ff:fe65:d9b1
|
||||
# fe80::ce2d:e0ff:fed4:2611
|
||||
# fe80::ce32:e5ff:fe79:7ea7
|
||||
# fe80::d66d:6dff:fe33:e00
|
||||
# fe80::e2ff:f7ff:fe00:20e6
|
||||
# fe80::f29f:c2ff:fe7c:275e
|
||||
try:
|
||||
sniff(iface=self.interfaces,
|
||||
filter="icmp6",
|
||||
prn=self.handle_pkg)
|
||||
except Exception as e:
|
||||
log.error("Could not start listener: %s", e)
|
||||
|
|
|
|||
82
docs/src/cdist-scan.rst
Normal file
82
docs/src/cdist-scan.rst
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
Scan
|
||||
=====
|
||||
|
||||
Description
|
||||
-----------
|
||||
Runs cdist as a daemon that discover/watch on hosts and reconfigure them
|
||||
periodically. It is especially useful in netboot-based environment where hosts
|
||||
boot unconfigured, and to ensure your infrastructure stays in sync with your
|
||||
configuration.
|
||||
|
||||
This feature is still consider to be in **beta** stage, and only operate on
|
||||
IPv6 (including link-local).
|
||||
|
||||
Usage (Examples)
|
||||
----------------
|
||||
|
||||
Discover hosts on local network and configure those whose name is resolved by
|
||||
the name mapper script.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
$ cdist scan --beta --interface eth0 \
|
||||
--mode scan --name-mapper path/to/script \
|
||||
--mode trigger --mode config
|
||||
|
||||
List known hosts and exit.
|
||||
|
||||
.. code-block:: sh
|
||||
|
||||
$ cdist scan --beta --list --name-mapper path/to/script
|
||||
|
||||
Please refer to `cdist(1)` for a detailed list of parameters.
|
||||
|
||||
Modes
|
||||
-----
|
||||
|
||||
The scanner has 3 modes that can be independently toggled. If the `--mode`
|
||||
parameter is not specified, only `tigger` and `scan` are enabled (= hosts are
|
||||
not configured).
|
||||
|
||||
trigger
|
||||
Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to
|
||||
trigger detection by the `scan` module.
|
||||
|
||||
scan
|
||||
Watch for incoming ICMPv6 replies and optionally configure detected hosts.
|
||||
|
||||
config
|
||||
Enable configuration of hosts detected by `scan`.
|
||||
|
||||
Name Mapper Script
|
||||
------------------
|
||||
|
||||
The name mapper script takes an IPv6 address as first argument and writes the
|
||||
resolved name to stdout - if any. The script must be executable.
|
||||
|
||||
Simplest script:
|
||||
|
||||
.. code-block:: sh
|
||||
#!/bin/sh
|
||||
|
||||
case "$1" in
|
||||
"fe80::20d:b9ff:fe57:3524")
|
||||
printf "my-host-01"
|
||||
;;
|
||||
"fe80::7603:bdff:fe05:89bb")
|
||||
printf "my-host-02"
|
||||
;;
|
||||
esac
|
||||
|
||||
Resolving name from `PTR` DNS record:
|
||||
|
||||
.. code-block:: sh
|
||||
#!/bin/sh
|
||||
|
||||
for cmd in dig sed; do
|
||||
if ! command -v $cmd > /dev/null; then
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
dig +short -x "$1" | sed -e 's/.$//'
|
||||
|
|
@ -34,6 +34,7 @@ It natively supports IPv6 since the first release.
|
|||
cdist-parallelization
|
||||
cdist-inventory
|
||||
cdist-preos
|
||||
cdist-scan
|
||||
cdist-integration
|
||||
cdist-reference
|
||||
cdist-best-practice
|
||||
|
|
|
|||
|
|
@ -88,6 +88,9 @@ SYNOPSIS
|
|||
cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
|
||||
[pattern]
|
||||
|
||||
cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list]
|
||||
[-d CONFIG_DELAY] [-t TRIGGER_DELAY]
|
||||
|
||||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
|
|
@ -641,6 +644,31 @@ Display information for cdist (global explorers, types).
|
|||
**-t, --types**
|
||||
Display info for types.
|
||||
|
||||
SCAN
|
||||
----
|
||||
|
||||
Runs cdist as a daemon that discover/watch on hosts and reconfigure them
|
||||
periodically.
|
||||
|
||||
**-I INTERFACE, --interfaces INTERFACE**
|
||||
Interface to listen on. Can be specified multiple times.
|
||||
|
||||
**-m MODE, --mode MODE**
|
||||
Scanner components to enable. Can be specified multiple time to enable more
|
||||
than one component. Supported modes are: scan, trigger and config. Defaults
|
||||
to tiggger and scan.
|
||||
|
||||
**--name-mapper PATH_TO_SCRIPT**
|
||||
Path to script used to resolve a remote host name from an IPv6 address.
|
||||
|
||||
**--list**
|
||||
List known hosts and exit.
|
||||
|
||||
**-d CONFIG_DELAY, --config-delay CONFIG_DELAY**
|
||||
How long (seconds) to wait before reconfiguring after last try (config mode only).
|
||||
|
||||
**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY**
|
||||
How long (seconds) to wait between ICMPv6 echo requests (trigger mode only).
|
||||
|
||||
CONFIGURATION
|
||||
-------------
|
||||
|
|
|
|||
Loading…
Reference in a new issue