From 05a6b8aa42e0c3d34dd9f4512a8e631c7c28f05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 15 May 2020 07:57:16 +0200 Subject: [PATCH 01/98] Import __matterbridge type from ungleich's dot-cdist --- .../files/matterbridge.service.sh | 18 ++++ type/__matterbridge/man.rst | 56 ++++++++++ type/__matterbridge/manifest | 101 ++++++++++++++++++ type/__matterbridge/parameter/required | 2 + type/__matterbridge/singleton | 0 5 files changed, 177 insertions(+) create mode 100755 type/__matterbridge/files/matterbridge.service.sh create mode 100644 type/__matterbridge/man.rst create mode 100755 type/__matterbridge/manifest create mode 100644 type/__matterbridge/parameter/required create mode 100644 type/__matterbridge/singleton diff --git a/type/__matterbridge/files/matterbridge.service.sh b/type/__matterbridge/files/matterbridge.service.sh new file mode 100755 index 0000000..9dbd1cb --- /dev/null +++ b/type/__matterbridge/files/matterbridge.service.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +cat <`_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__matterbridge/manifest b/type/__matterbridge/manifest new file mode 100755 index 0000000..56f470a --- /dev/null +++ b/type/__matterbridge/manifest @@ -0,0 +1,101 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "$__global/explorer/os") +case "$os" in + debian) + # This type assume systemd for service installation. + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +# Required parameters. +VERSION=$(cat "$__object/parameter/version") +if [ -f "$__object/parameter/config" ]; then + CONFIG="$(cat "$__object/parameter/config")" + if [ "$CONFIG" = "-" ]; then + CONFIG=$(cat "$__object/stdin") + fi +fi + +# Hardcoded values used in templates. +export BINARY_PATH=/usr/local/bin/matterbridge +export CONFIG_PATH=/etc/matterbridge/matterbridge.toml +export USER=matterbridge +export GROUP=$USER + +# Internal variables. +artefact="matterbridge-$VERSION-linux-64bit" +checksum_file="checksums.txt" +release_download_url=https://github.com/42wim/matterbridge/releases/download +binary_url="$release_download_url/v$VERSION/$artefact" +checksum_file_url="$release_download_url/v$VERSION/$checksum_file" +config_dir=$(dirname $CONFIG_PATH) +systemd_unit_path='/etc/systemd/system/matterbridge.service' + +# Check if curl is available. +if [ ! -x "$(which curl)" ]; then + echo "curl is required for this type, but could not be found. Exiting." &>2 + exit 1 +fi + +# Initialize working directory. +mkdir -p "$__object/files" + +# Download and check matterbridge binary. +curl -L $binary_url -o "$__object/files/$artefact" +curl -Ls $checksum_file_url | grep $artefact > "$__object/files/$checksum_file" +ls $__object/files/ >&2 +cat $__object/files/checksums.txt >&2 +(cd "$__object/files"; sha256sum --check $checksum_file) +if [ $? -ne 0 ]; then + echo "Matterbridge binary checksum failed." >&2 + exit 1 +fi + +# Create service user. +__user $USER --home "/var/lib/$USER" + +# Deploy matterbridge binary. +require="__user/$USER" __file "$BINARY_PATH" \ + --source "$__object/files/$artefact" \ + --owner "$USER" --mode 755 + +# Generate and deploy configuration file. +"$__type/files/matterbridge.service.sh" > "$__object/files/matterbridge.service" + +require="__user/$USER" __directory "$config_dir" \ + --owner "$USER" --mode 0755 --parents \ + +require="__directory/$config_dir" __file "$CONFIG_PATH" \ + --owner "$USER" \ + --mode 0640 \ + --source "$CONFIG" + +__file "$systemd_unit_path" \ + --source "$__object/files/matterbridge.service" + +# Deal with init system. +require="__file/$systemd_unit_path" __start_on_boot matterbridge +require="__file/$BINARY_PATH __file/$CONFIG_PATH __file/$systemd_unit_path" __service matterbridge --action restart diff --git a/type/__matterbridge/parameter/required b/type/__matterbridge/parameter/required new file mode 100644 index 0000000..ed5d8b3 --- /dev/null +++ b/type/__matterbridge/parameter/required @@ -0,0 +1,2 @@ +version +config diff --git a/type/__matterbridge/singleton b/type/__matterbridge/singleton new file mode 100644 index 0000000..e69de29 From 43562ff09c7ab44f42e37893df5dc48af1b1e15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 15 May 2020 08:08:45 +0200 Subject: [PATCH 02/98] Make __matterbridge shellcheck-friendly --- type/__matterbridge/manifest | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/type/__matterbridge/manifest b/type/__matterbridge/manifest index 56f470a..ef02112 100755 --- a/type/__matterbridge/manifest +++ b/type/__matterbridge/manifest @@ -55,8 +55,8 @@ config_dir=$(dirname $CONFIG_PATH) systemd_unit_path='/etc/systemd/system/matterbridge.service' # Check if curl is available. -if [ ! -x "$(which curl)" ]; then - echo "curl is required for this type, but could not be found. Exiting." &>2 +if ! command -v curl; then + echo "curl is required for this type, but could not be found. Exiting." >&2 exit 1 fi @@ -64,12 +64,9 @@ fi mkdir -p "$__object/files" # Download and check matterbridge binary. -curl -L $binary_url -o "$__object/files/$artefact" -curl -Ls $checksum_file_url | grep $artefact > "$__object/files/$checksum_file" -ls $__object/files/ >&2 -cat $__object/files/checksums.txt >&2 -(cd "$__object/files"; sha256sum --check $checksum_file) -if [ $? -ne 0 ]; then +curl -L "$binary_url" -o "$__object/files/$artefact" +curl -Ls "$checksum_file_url" | grep "$artefact" > "$__object/files/$checksum_file" +if ! (cd "$__object/files"; sha256sum --check $checksum_file); then echo "Matterbridge binary checksum failed." >&2 exit 1 fi From 3adb5ac4cabe65b24c0a5139a62b38b35b9191bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 14:58:43 +0200 Subject: [PATCH 03/98] Import __ungleich_unbound type for ungleich's dot-cdist --- type/__ungleich_unbound/files/unbound.conf.sh | 1018 +++++++++++++++++ type/__ungleich_unbound/man.rst | 74 ++ type/__ungleich_unbound/manifest | 79 ++ type/__ungleich_unbound/parameter/boolean | 2 + .../parameter/optional_multiple | 2 + type/__ungleich_unbound/parameter/required | 1 + .../parameter/required_multiple | 1 + type/__ungleich_unbound/singleton | 0 8 files changed, 1177 insertions(+) create mode 100755 type/__ungleich_unbound/files/unbound.conf.sh create mode 100644 type/__ungleich_unbound/man.rst create mode 100755 type/__ungleich_unbound/manifest create mode 100644 type/__ungleich_unbound/parameter/boolean create mode 100644 type/__ungleich_unbound/parameter/optional_multiple create mode 100644 type/__ungleich_unbound/parameter/required create mode 100644 type/__ungleich_unbound/parameter/required_multiple create mode 100644 type/__ungleich_unbound/singleton diff --git a/type/__ungleich_unbound/files/unbound.conf.sh b/type/__ungleich_unbound/files/unbound.conf.sh new file mode 100755 index 0000000..e713843 --- /dev/null +++ b/type/__ungleich_unbound/files/unbound.conf.sh @@ -0,0 +1,1018 @@ +#!/bin/sh + +generate_interface() { + for i in $INTERFACES; do + echo " interface: $i" + done +} + +generate_access_control() { + echo "$ACCESS_CONTROLS" | while read entry; do + echo " access-control: $entry" + done +} + +generate_forward_addr() { + for addr in $FORWARD_ADDRS; do + echo " forward-addr: $addr" + done +} + +cat << EOF +# +# THIS FILE HAS BEEN GENERATED BY CDIST, DO NOT EDIT BY HAND. +# +# See unbound.conf(5) man page, version 1.9.1. +# +# this is a comment. + +#Use this to include other text into the file. +#include: "otherfile.conf" + +# The server clause sets the main parameters. +server: + # whitespace is not necessary, but looks cleaner. + + # verbosity number, 0 is least verbose. 1 is default. + verbosity: 1 + + # print statistics to the log (for every thread) every N seconds. + # Set to "" or 0 to disable. Default is disabled. + # statistics-interval: 0 + + # enable shm for stats, default no. if you enable also enable + # statistics-interval, every time it also writes stats to the + # shared memory segment keyed with shm-key. + # shm-enable: no + + # shm for stats uses this key, and key+1 for the shared mem segment. + # shm-key: 11777 + + # enable cumulative statistics, without clearing them after printing. + # statistics-cumulative: no + + # enable extended statistics (query types, answer codes, status) + # printed from unbound-control. default off, because of speed. + # extended-statistics: no + + # number of threads to create. 1 disables threading. + # num-threads: 1 + + # specify the interfaces to answer queries from by ip-address. + # The default is to listen to localhost (127.0.0.1 and ::1). + # specify 0.0.0.0 and ::0 to bind to all available interfaces. + # specify every interface[@port] on a new 'interface:' labelled line. + # The listen interfaces are not changed on reload, only on restart. + # interface: 192.0.2.153 + # interface: 192.0.2.154 + # interface: 192.0.2.154@5003 + # interface: 2001:DB8::5 +$(generate_interface) + + # enable this feature to copy the source address of queries to reply. + # Socket options are not supported on all platforms. experimental. + # interface-automatic: no + + # port to answer queries from + # port: 53 + + # specify the interfaces to send outgoing queries to authoritative + # server from by ip-address. If none, the default (all) interface + # is used. Specify every interface on a 'outgoing-interface:' line. + # outgoing-interface: 192.0.2.153 + # outgoing-interface: 2001:DB8::5 + # outgoing-interface: 2001:DB8::6 + + # Specify a netblock to use remainder 64 bits as random bits for + # upstream queries. Uses freebind option (Linux). + # outgoing-interface: 2001:DB8::/64 + # Also (Linux:) ip -6 addr add 2001:db8::/64 dev lo + # And: ip -6 route add local 2001:db8::/64 dev lo + # And set prefer-ip6: yes to use the ip6 randomness from a netblock. + # Set this to yes to prefer ipv6 upstream servers over ipv4. + # prefer-ip6: no + + # number of ports to allocate per thread, determines the size of the + # port range that can be open simultaneously. About double the + # num-queries-per-thread, or, use as many as the OS will allow you. + # outgoing-range: 4096 + + # permit unbound to use this port number or port range for + # making outgoing queries, using an outgoing interface. + # outgoing-port-permit: 32768 + + # deny unbound the use this of port number or port range for + # making outgoing queries, using an outgoing interface. + # Use this to make sure unbound does not grab a UDP port that some + # other server on this computer needs. The default is to avoid + # IANA-assigned port numbers. + # If multiple outgoing-port-permit and outgoing-port-avoid options + # are present, they are processed in order. + # outgoing-port-avoid: "3200-3208" + + # number of outgoing simultaneous tcp buffers to hold per thread. + # outgoing-num-tcp: 10 + + # number of incoming simultaneous tcp buffers to hold per thread. + # incoming-num-tcp: 10 + + # buffer size for UDP port 53 incoming (SO_RCVBUF socket option). + # 0 is system default. Use 4m to catch query spikes for busy servers. + # so-rcvbuf: 0 + + # buffer size for UDP port 53 outgoing (SO_SNDBUF socket option). + # 0 is system default. Use 4m to handle spikes on very busy servers. + # so-sndbuf: 0 + + # use SO_REUSEPORT to distribute queries over threads. + # so-reuseport: yes + + # use IP_TRANSPARENT so the interface: addresses can be non-local + # and you can config non-existing IPs that are going to work later on + # (uses IP_BINDANY on FreeBSD). + # ip-transparent: no + + # use IP_FREEBIND so the interface: addresses can be non-local + # and you can bind to nonexisting IPs and interfaces that are down. + # Linux only. On Linux you also have ip-transparent that is similar. + # ip-freebind: no + + # EDNS reassembly buffer to advertise to UDP peers (the actual buffer + # is set with msg-buffer-size). 1472 can solve fragmentation (timeouts) + # edns-buffer-size: 4096 + + # Maximum UDP response size (not applied to TCP response). + # Suggested values are 512 to 4096. Default is 4096. 65536 disables it. + # max-udp-size: 4096 + + # max memory to use for stream(tcp and tls) waiting result buffers. + # stream-wait-size: 4m + + # buffer size for handling DNS data. No messages larger than this + # size can be sent or received, by UDP or TCP. In bytes. + # msg-buffer-size: 65552 + + # the amount of memory to use for the message cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # msg-cache-size: 4m + + # the number of slabs to use for the message cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # msg-cache-slabs: 4 + + # the number of queries that a thread gets to service. + # num-queries-per-thread: 1024 + + # if very busy, 50% queries run to completion, 50% get timeout in msec + # jostle-timeout: 200 + + # msec to wait before close of port on timeout UDP. 0 disables. + # delay-close: 0 + + # msec for waiting for an unknown server to reply. Increase if you + # are behind a slow satellite link, to eg. 1128. + # unknown-server-time-limit: 376 + + # the amount of memory to use for the RRset cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # rrset-cache-size: 4m + + # the number of slabs to use for the RRset cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # rrset-cache-slabs: 4 + + # the time to live (TTL) value lower bound, in seconds. Default 0. + # If more than an hour could easily give trouble due to stale data. + # cache-min-ttl: 0 + + # the time to live (TTL) value cap for RRsets and messages in the + # cache. Items are not cached for longer. In seconds. + # cache-max-ttl: 86400 + + # the time to live (TTL) value cap for negative responses in the cache + # cache-max-negative-ttl: 3600 + + # the time to live (TTL) value for cached roundtrip times, lameness and + # EDNS version information for hosts. In seconds. + # infra-host-ttl: 900 + + # minimum wait time for responses, increase if uplink is long. In msec. + # infra-cache-min-rtt: 50 + + # the number of slabs to use for the Infrastructure cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # infra-cache-slabs: 4 + + # the maximum number of hosts that are cached (roundtrip, EDNS, lame). + # infra-cache-numhosts: 10000 + + # define a number of tags here, use with local-zone, access-control. + # repeat the define-tag statement to add additional tags. + # define-tag: "tag1 tag2 tag3" + + # Enable IPv4, "yes" or "no". + do-ip4: $DO_IP4 + + # Enable IPv6, "yes" or "no". + do-ip6: $DO_IP6 + + # Enable UDP, "yes" or "no". + # do-udp: yes + + # Enable TCP, "yes" or "no". + # do-tcp: yes + + # upstream connections use TCP only (and no UDP), "yes" or "no" + # useful for tunneling scenarios, default no. + # tcp-upstream: no + + # upstream connections also use UDP (even if do-udp is no). + # useful if if you want UDP upstream, but don't provide UDP downstream. + # udp-upstream-without-downstream: no + + # Maximum segment size (MSS) of TCP socket on which the server + # responds to queries. Default is 0, system default MSS. + # tcp-mss: 0 + + # Maximum segment size (MSS) of TCP socket for outgoing queries. + # Default is 0, system default MSS. + # outgoing-tcp-mss: 0 + + # Idle TCP timeout, connection closed in milliseconds + # tcp-idle-timeout: 30000 + + # Enable EDNS TCP keepalive option. + # edns-tcp-keepalive: no + + # Timeout for EDNS TCP keepalive, in msec. + # edns-tcp-keepalive-timeout: 120000 + + # Use systemd socket activation for UDP, TCP, and control sockets. + # use-systemd: no + + # Detach from the terminal, run in background, "yes" or "no". + # Set the value to "no" when unbound runs as systemd service. + # do-daemonize: yes + + # control which clients are allowed to make (recursive) queries + # to this server. Specify classless netblocks with /size and action. + # By default everything is refused, except for localhost. + # Choose deny (drop message), refuse (polite error reply), + # allow (recursive ok), allow_setrd (recursive ok, rd bit is forced on), + # allow_snoop (recursive and nonrecursive ok) + # deny_non_local (drop queries unless can be answered from local-data) + # refuse_non_local (like deny_non_local but polite error reply). + # access-control: 0.0.0.0/0 refuse + # access-control: 127.0.0.0/8 allow + # access-control: ::0/0 refuse + # access-control: ::1 allow + # access-control: ::ffff:127.0.0.1 allow +$(generate_access_control) + + # tag access-control with list of tags (in "" with spaces between) + # Clients using this access control element use localzones that + # are tagged with one of these tags. + # access-control-tag: 192.0.2.0/24 "tag2 tag3" + + # set action for particular tag for given access control element + # if you have multiple tag values, the tag used to lookup the action + # is the first tag match between access-control-tag and local-zone-tag + # where "first" comes from the order of the define-tag values. + # access-control-tag-action: 192.0.2.0/24 tag3 refuse + + # set redirect data for particular tag for access control element + # access-control-tag-data: 192.0.2.0/24 tag2 "A 127.0.0.1" + + # Set view for access control element + # access-control-view: 192.0.2.0/24 viewname + + # if given, a chroot(2) is done to the given directory. + # i.e. you can chroot to the working directory, for example, + # for extra security, but make sure all files are in that directory. + # + # If chroot is enabled, you should pass the configfile (from the + # commandline) as a full path from the original root. After the + # chroot has been performed the now defunct portion of the config + # file path is removed to be able to reread the config after a reload. + # + # All other file paths (working dir, logfile, roothints, and + # key files) can be specified in several ways: + # o as an absolute path relative to the new root. + # o as a relative path to the working directory. + # o as an absolute path relative to the original root. + # In the last case the path is adjusted to remove the unused portion. + # + # The pid file can be absolute and outside of the chroot, it is + # written just prior to performing the chroot and dropping permissions. + # + # Additionally, unbound may need to access /dev/random (for entropy). + # How to do this is specific to your OS. + # + # If you give "" no chroot is performed. The path must not end in a /. + # chroot: "" + + # if given, user privileges are dropped (after binding port), + # and the given username is assumed. Default is user "unbound". + # If you give "" no privileges are dropped. + # username: "unbound" + + # the working directory. The relative files in this config are + # relative to this directory. If you give "" the working directory + # is not changed. + # If you give a server: directory: dir before include: file statements + # then those includes can be relative to the working directory. + # directory: "" + + # the log file, "" means log to stderr. + # Use of this option sets use-syslog to "no". + # logfile: "" + + # Log to syslog(3) if yes. The log facility LOG_DAEMON is used to + # log to. If yes, it overrides the logfile. + # use-syslog: yes + + # Log identity to report. if empty, defaults to the name of argv[0] + # (usually "unbound"). + # log-identity: "" + + # print UTC timestamp in ascii to logfile, default is epoch in seconds. + # log-time-ascii: no + + # print one line with time, IP, name, type, class for every query. + # log-queries: no + + # print one line per reply, with time, IP, name, type, class, rcode, + # timetoresolve, fromcache and responsesize. + # log-replies: no + + # log with tag 'query' and 'reply' instead of 'info' for + # filtering log-queries and log-replies from the log. + # log-tag-queryreply: no + + # log the local-zone actions, like local-zone type inform is enabled + # also for the other local zone types. + # log-local-actions: no + + # print log lines that say why queries return SERVFAIL to clients. + # log-servfail: no + + # file to read root hints from. + # get one from https://www.internic.net/domain/named.cache + root-hints: /usr/share/dns-root-hints/named.root + + # enable to not answer id.server and hostname.bind queries. + # hide-identity: no + + # enable to not answer version.server and version.bind queries. + # hide-version: no + + # enable to not answer trustanchor.unbound queries. + # hide-trustanchor: no + + # the identity to report. Leave "" or default to return hostname. + # identity: "" + + # the version to report. Leave "" or default to return package version. + # version: "" + + # the target fetch policy. + # series of integers describing the policy per dependency depth. + # The number of values in the list determines the maximum dependency + # depth the recursor will pursue before giving up. Each integer means: + # -1 : fetch all targets opportunistically, + # 0: fetch on demand, + # positive value: fetch that many targets opportunistically. + # Enclose the list of numbers between quotes (""). + # target-fetch-policy: "3 2 1 0 0" + + # Harden against very small EDNS buffer sizes. + # harden-short-bufsize: no + + # Harden against unseemly large queries. + # harden-large-queries: no + + # Harden against out of zone rrsets, to avoid spoofing attempts. + # harden-glue: yes + + # Harden against receiving dnssec-stripped data. If you turn it + # off, failing to validate dnskey data for a trustanchor will + # trigger insecure mode for that zone (like without a trustanchor). + # Default on, which insists on dnssec data for trust-anchored zones. + # harden-dnssec-stripped: yes + + # Harden against queries that fall under dnssec-signed nxdomain names. + # harden-below-nxdomain: yes + + # Harden the referral path by performing additional queries for + # infrastructure data. Validates the replies (if possible). + # Default off, because the lookups burden the server. Experimental + # implementation of draft-wijngaards-dnsext-resolver-side-mitigation. + # harden-referral-path: no + + # Harden against algorithm downgrade when multiple algorithms are + # advertised in the DS record. If no, allows the weakest algorithm + # to validate the zone. + # harden-algo-downgrade: no + + # Sent minimum amount of information to upstream servers to enhance + # privacy. Only sent minimum required labels of the QNAME and set QTYPE + # to A when possible. + # qname-minimisation: yes + + # QNAME minimisation in strict mode. Do not fall-back to sending full + # QNAME to potentially broken nameservers. A lot of domains will not be + # resolvable when this option in enabled. + # This option only has effect when qname-minimisation is enabled. + # qname-minimisation-strict: no + + # Aggressive NSEC uses the DNSSEC NSEC chain to synthesize NXDOMAIN + # and other denials, using information from previous NXDOMAINs answers. + # aggressive-nsec: no + + # Use 0x20-encoded random bits in the query to foil spoof attempts. + # This feature is an experimental implementation of draft dns-0x20. + # use-caps-for-id: no + + # Domains (and domains in them) without support for dns-0x20 and + # the fallback fails because they keep sending different answers. + # caps-whitelist: "licdn.com" + # caps-whitelist: "senderbase.org" + + # Enforce privacy of these addresses. Strips them away from answers. + # It may cause DNSSEC validation to additionally mark it as bogus. + # Protects against 'DNS Rebinding' (uses browser as network proxy). + # Only 'private-domain' and 'local-data' names are allowed to have + # these private addresses. No default. + # private-address: 10.0.0.0/8 + # private-address: 172.16.0.0/12 + # private-address: 192.168.0.0/16 + # private-address: 169.254.0.0/16 + # private-address: fd00::/8 + # private-address: fe80::/10 + # private-address: ::ffff:0:0/96 + + # Allow the domain (and its subdomains) to contain private addresses. + # local-data statements are allowed to contain private addresses too. + # private-domain: "example.com" + + # If nonzero, unwanted replies are not only reported in statistics, + # but also a running total is kept per thread. If it reaches the + # threshold, a warning is printed and a defensive action is taken, + # the cache is cleared to flush potential poison out of it. + # A suggested value is 10000000, the default is 0 (turned off). + # unwanted-reply-threshold: 0 + + # Do not query the following addresses. No DNS queries are sent there. + # List one address per entry. List classless netblocks with /size, + # do-not-query-address: 127.0.0.1/8 + # do-not-query-address: ::1 + + # if yes, the above default do-not-query-address entries are present. + # if no, localhost can be queried (for testing and debugging). + # do-not-query-localhost: yes + + # if yes, perform prefetching of almost expired message cache entries. + # prefetch: no + + # if yes, perform key lookups adjacent to normal lookups. + # prefetch-key: no + + # deny queries of type ANY with an empty response. + # deny-any: no + + # if yes, Unbound rotates RRSet order in response. + # rrset-roundrobin: no + + # if yes, Unbound doesn't insert authority/additional sections + # into response messages when those sections are not required. + # minimal-responses: yes + + # true to disable DNSSEC lameness check in iterator. + # disable-dnssec-lame-check: no + + # module configuration of the server. A string with identifiers + # separated by spaces. Syntax: "[dns64] [validator] iterator" + # most modules have to be listed at the beginning of the line, + # except cachedb(just before iterator), and python (at the beginning, + # or, just before the iterator). + module-config: "dns64 validator iterator" + + # File with trusted keys, kept uptodate using RFC5011 probes, + # initial file like trust-anchor-file, then it stores metadata. + # Use several entries, one per domain name, to track multiple zones. + # + # If you want to perform DNSSEC validation, run unbound-anchor before + # you start unbound (i.e. in the system boot scripts). And enable: + # Please note usage of unbound-anchor root anchor is at your own risk + # and under the terms of our LICENSE (see that file in the source). + # auto-trust-anchor-file: "" + + # trust anchor signaling sends a RFC8145 key tag query after priming. + # trust-anchor-signaling: yes + + # Root key trust anchor sentinel (draft-ietf-dnsop-kskroll-sentinel) + # root-key-sentinel: yes + + # File with DLV trusted keys. Same format as trust-anchor-file. + # There can be only one DLV configured, it is trusted from root down. + # DLV is going to be decommissioned. Please do not use it any more. + # dlv-anchor-file: "dlv.isc.org.key" + + # File with trusted keys for validation. Specify more than one file + # with several entries, one file per entry. + # Zone file format, with DS and DNSKEY entries. + # Note this gets out of date, use auto-trust-anchor-file please. + trust-anchor-file: "/usr/share/dnssec-root/trusted-key.key" + + # Trusted key for validation. DS or DNSKEY. specify the RR on a + # single line, surrounded by "". TTL is ignored. class is IN default. + # Note this gets out of date, use auto-trust-anchor-file please. + # (These examples are from August 2007 and may not be valid anymore). + # trust-anchor: "nlnetlabs.nl. DNSKEY 257 3 5 AQPzzTWMz8qSWIQlfRnPckx2BiVmkVN6LPupO3mbz7FhLSnm26n6iG9N Lby97Ji453aWZY3M5/xJBSOS2vWtco2t8C0+xeO1bc/d6ZTy32DHchpW 6rDH1vp86Ll+ha0tmwyy9QP7y2bVw5zSbFCrefk8qCUBgfHm9bHzMG1U BYtEIQ==" + # trust-anchor: "jelte.nlnetlabs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A" + + # File with trusted keys for validation. Specify more than one file + # with several entries, one file per entry. Like trust-anchor-file + # but has a different file format. Format is BIND-9 style format, + # the trusted-keys { name flag proto algo "key"; }; clauses are read. + # you need external update procedures to track changes in keys. + # trusted-keys-file: "" + + # Ignore chain of trust. Domain is treated as insecure. + # domain-insecure: "example.com" + + # Override the date for validation with a specific fixed date. + # Do not set this unless you are debugging signature inception + # and expiration. "" or "0" turns the feature off. -1 ignores date. + # val-override-date: "" + + # The time to live for bogus data, rrsets and messages. This avoids + # some of the revalidation, until the time interval expires. in secs. + # val-bogus-ttl: 60 + + # The signature inception and expiration dates are allowed to be off + # by 10% of the signature lifetime (expir-incep) from our local clock. + # This leeway is capped with a minimum and a maximum. In seconds. + # val-sig-skew-min: 3600 + # val-sig-skew-max: 86400 + + # Should additional section of secure message also be kept clean of + # unsecure data. Useful to shield the users of this validator from + # potential bogus data in the additional section. All unsigned data + # in the additional section is removed from secure messages. + # val-clean-additional: yes + + # Turn permissive mode on to permit bogus messages. Thus, messages + # for which security checks failed will be returned to clients, + # instead of SERVFAIL. It still performs the security checks, which + # result in interesting log files and possibly the AD bit in + # replies if the message is found secure. The default is off. + # val-permissive-mode: no + + # Ignore the CD flag in incoming queries and refuse them bogus data. + # Enable it if the only clients of unbound are legacy servers (w2008) + # that set CD but cannot validate themselves. + # ignore-cd-flag: no + + # Serve expired responses from cache, with TTL 0 in the response, + # and then attempt to fetch the data afresh. + # serve-expired: no + # + # Limit serving of expired responses to configured seconds after + # expiration. 0 disables the limit. + # serve-expired-ttl: 0 + # + # Set the TTL of expired records to the serve-expired-ttl value after a + # failed attempt to retrieve the record from upstream. This makes sure + # that the expired records will be served as long as there are queries + # for it. + # serve-expired-ttl-reset: no + + # Have the validator log failed validations for your diagnosis. + # 0: off. 1: A line per failed user query. 2: With reason and bad IP. + # val-log-level: 0 + + # It is possible to configure NSEC3 maximum iteration counts per + # keysize. Keep this table very short, as linear search is done. + # A message with an NSEC3 with larger count is marked insecure. + # List in ascending order the keysize and count values. + # val-nsec3-keysize-iterations: "1024 150 2048 500 4096 2500" + + # instruct the auto-trust-anchor-file probing to add anchors after ttl. + # add-holddown: 2592000 # 30 days + + # instruct the auto-trust-anchor-file probing to del anchors after ttl. + # del-holddown: 2592000 # 30 days + + # auto-trust-anchor-file probing removes missing anchors after ttl. + # If the value 0 is given, missing anchors are not removed. + # keep-missing: 31622400 # 366 days + + # debug option that allows very small holddown times for key rollover, + # otherwise the RFC mandates probe intervals must be at least 1 hour. + # permit-small-holddown: no + + # the amount of memory to use for the key cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # key-cache-size: 4m + + # the number of slabs to use for the key cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # key-cache-slabs: 4 + + # the amount of memory to use for the negative cache (used for DLV). + # plain value in bytes or you can append k, m or G. default is "1Mb". + # neg-cache-size: 1m + + # By default, for a number of zones a small default 'nothing here' + # reply is built-in. Query traffic is thus blocked. If you + # wish to serve such zone you can unblock them by uncommenting one + # of the nodefault statements below. + # You may also have to use domain-insecure: zone to make DNSSEC work, + # unless you have your own trust anchors for this zone. + # local-zone: "localhost." nodefault + # local-zone: "127.in-addr.arpa." nodefault + # local-zone: "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa." nodefault + # local-zone: "onion." nodefault + # local-zone: "test." nodefault + # local-zone: "invalid." nodefault + # local-zone: "10.in-addr.arpa." nodefault + # local-zone: "16.172.in-addr.arpa." nodefault + # local-zone: "17.172.in-addr.arpa." nodefault + # local-zone: "18.172.in-addr.arpa." nodefault + # local-zone: "19.172.in-addr.arpa." nodefault + # local-zone: "20.172.in-addr.arpa." nodefault + # local-zone: "21.172.in-addr.arpa." nodefault + # local-zone: "22.172.in-addr.arpa." nodefault + # local-zone: "23.172.in-addr.arpa." nodefault + # local-zone: "24.172.in-addr.arpa." nodefault + # local-zone: "25.172.in-addr.arpa." nodefault + # local-zone: "26.172.in-addr.arpa." nodefault + # local-zone: "27.172.in-addr.arpa." nodefault + # local-zone: "28.172.in-addr.arpa." nodefault + # local-zone: "29.172.in-addr.arpa." nodefault + # local-zone: "30.172.in-addr.arpa." nodefault + # local-zone: "31.172.in-addr.arpa." nodefault + # local-zone: "168.192.in-addr.arpa." nodefault + # local-zone: "0.in-addr.arpa." nodefault + # local-zone: "254.169.in-addr.arpa." nodefault + # local-zone: "2.0.192.in-addr.arpa." nodefault + # local-zone: "100.51.198.in-addr.arpa." nodefault + # local-zone: "113.0.203.in-addr.arpa." nodefault + # local-zone: "255.255.255.255.in-addr.arpa." nodefault + # local-zone: "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa." nodefault + # local-zone: "d.f.ip6.arpa." nodefault + # local-zone: "8.e.f.ip6.arpa." nodefault + # local-zone: "9.e.f.ip6.arpa." nodefault + # local-zone: "a.e.f.ip6.arpa." nodefault + # local-zone: "b.e.f.ip6.arpa." nodefault + # local-zone: "8.b.d.0.1.0.0.2.ip6.arpa." nodefault + # And for 64.100.in-addr.arpa. to 127.100.in-addr.arpa. + + # If unbound is running service for the local host then it is useful + # to perform lan-wide lookups to the upstream, and unblock the + # long list of local-zones above. If this unbound is a dns server + # for a network of computers, disabled is better and stops information + # leakage of local lan information. + # unblock-lan-zones: no + + # The insecure-lan-zones option disables validation for + # these zones, as if they were all listed as domain-insecure. + # insecure-lan-zones: no + + # a number of locally served zones can be configured. + # local-zone: + # local-data: "" + # o deny serves local data (if any), else, drops queries. + # o refuse serves local data (if any), else, replies with error. + # o static serves local data, else, nxdomain or nodata answer. + # o transparent gives local data, but resolves normally for other names + # o redirect serves the zone data for any subdomain in the zone. + # o nodefault can be used to normally resolve AS112 zones. + # o typetransparent resolves normally for other types and other names + # o inform acts like transparent, but logs client IP address + # o inform_deny drops queries and logs client IP address + # o inform_redirect redirects queries and logs client IP address + # o always_transparent, always_refuse, always_nxdomain, resolve in + # that way but ignore local data for that name + # o noview breaks out of that view towards global local-zones. + # + # defaults are localhost address, reverse for 127.0.0.1 and ::1 + # and nxdomain for AS112 zones. If you configure one of these zones + # the default content is omitted, or you can omit it with 'nodefault'. + # + # If you configure local-data without specifying local-zone, by + # default a transparent local-zone is created for the data. + # + # You can add locally served data with + # local-zone: "local." static + # local-data: "mycomputer.local. IN A 192.0.2.51" + # local-data: 'mytext.local TXT "content of text record"' + # + # You can override certain queries with + # local-data: "adserver.example.com A 127.0.0.1" + # + # You can redirect a domain to a fixed address with + # (this makes example.com, www.example.com, etc, all go to 192.0.2.3) + # local-zone: "example.com" redirect + # local-data: "example.com A 192.0.2.3" + # + # Shorthand to make PTR records, "IPv4 name" or "IPv6 name". + # You can also add PTR records using local-data directly, but then + # you need to do the reverse notation yourself. + # local-data-ptr: "192.0.2.3 www.example.com" + + # tag a localzone with a list of tag names (in "" with spaces between) + # local-zone-tag: "example.com" "tag2 tag3" + + # add a netblock specific override to a localzone, with zone type + # local-zone-override: "example.com" 192.0.2.0/24 refuse + + # service clients over TLS (on the TCP sockets), with plain DNS inside + # the TLS stream. Give the certificate to use and private key. + # default is "" (disabled). requires restart to take effect. + # tls-service-key: "path/to/privatekeyfile.key" + # tls-service-pem: "path/to/publiccertfile.pem" + # tls-port: 853 + + # cipher setting for TLSv1.2 + # tls-ciphers: "DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256" + # cipher setting for TLSv1.3 + # tls-ciphersuites: "TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_8_SHA256:TLS_AES_128_CCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256" + + # Add the secret file for TLS Session Ticket. + # Secret file must be 80 bytes of random data. + # First key use to encrypt and decrypt TLS session tickets. + # Other keys use to decrypt only. + # requires restart to take effect. + # tls-session-ticket-keys: "path/to/secret_file1" + # tls-session-ticket-keys: "path/to/secret_file2" + + # request upstream over TLS (with plain DNS inside the TLS stream). + # Default is no. Can be turned on and off with unbound-control. + # tls-upstream: no + + # Certificates used to authenticate connections made upstream. + # tls-cert-bundle: "" + + # Add system certs to the cert bundle, from the Windows Cert Store + # tls-win-cert: no + + # Also serve tls on these port numbers (eg. 443, ...), by listing + # tls-additional-port: portno for each of the port numbers. + + # DNS64 prefix. Must be specified when DNS64 is use. + # Enable dns64 in module-config. Used to synthesize IPv6 from IPv4. + dns64-prefix: $DNS64_PREFIX + + # DNS64 ignore AAAA records for these domains and use A instead. + # dns64-ignore-aaaa: "example.com" + + # ratelimit for uncached, new queries, this limits recursion effort. + # ratelimiting is experimental, and may help against randomqueryflood. + # if 0(default) it is disabled, otherwise state qps allowed per zone. + # ratelimit: 0 + + # ratelimits are tracked in a cache, size in bytes of cache (or k,m). + # ratelimit-size: 4m + # ratelimit cache slabs, reduces lock contention if equal to cpucount. + # ratelimit-slabs: 4 + + # 0 blocks when ratelimited, otherwise let 1/xth traffic through + # ratelimit-factor: 10 + + # override the ratelimit for a specific domain name. + # give this setting multiple times to have multiple overrides. + # ratelimit-for-domain: example.com 1000 + # override the ratelimits for all domains below a domain name + # can give this multiple times, the name closest to the zone is used. + # ratelimit-below-domain: com 1000 + + # global query ratelimit for all ip addresses. + # feature is experimental. + # if 0(default) it is disabled, otherwise states qps allowed per ip address + # ip-ratelimit: 0 + + # ip ratelimits are tracked in a cache, size in bytes of cache (or k,m). + # ip-ratelimit-size: 4m + # ip ratelimit cache slabs, reduces lock contention if equal to cpucount. + # ip-ratelimit-slabs: 4 + + # 0 blocks when ip is ratelimited, otherwise let 1/xth traffic through + # ip-ratelimit-factor: 10 + + # Limit the number of connections simultaneous from a netblock + # tcp-connection-limit: 192.0.2.0/24 12 + + # select from the fastest servers this many times out of 1000. 0 means + # the fast server select is disabled. prefetches are not sped up. + # fast-server-permil: 0 + # the number of servers that will be used in the fast server selection. + # fast-server-num: 3 + + # Specific options for ipsecmod. unbound needs to be configured with + # --enable-ipsecmod for these to take effect. + # + # Enable or disable ipsecmod (it still needs to be defined in + # module-config above). Can be used when ipsecmod needs to be + # enabled/disabled via remote-control(below). + # ipsecmod-enabled: yes + # + # Path to executable external hook. It must be defined when ipsecmod is + # listed in module-config (above). + # ipsecmod-hook: "./my_executable" + # + # When enabled unbound will reply with SERVFAIL if the return value of + # the ipsecmod-hook is not 0. + # ipsecmod-strict: no + # + # Maximum time to live (TTL) for cached A/AAAA records with IPSECKEY. + # ipsecmod-max-ttl: 3600 + # + # Reply with A/AAAA even if the relevant IPSECKEY is bogus. Mainly used for + # testing. + # ipsecmod-ignore-bogus: no + # + # Domains for which ipsecmod will be triggered. If not defined (default) + # all domains are treated as being whitelisted. + # ipsecmod-whitelist: "example.com" + # ipsecmod-whitelist: "nlnetlabs.nl" + + +# Python config section. To enable: +# o use --with-pythonmodule to configure before compiling. +# o list python in the module-config string (above) to enable. +# It can be at the start, it gets validated results, or just before +# the iterator and process before DNSSEC validation. +# o and give a python-script to run. +python: + # Script file to load + # python-script: "/ubmodule-tst.py" + +# Remote control config section. +remote-control: + # Enable remote control with unbound-control(8) here. + # set up the keys and certificates with unbound-control-setup. + # control-enable: no + + # what interfaces are listened to for remote control. + # give 0.0.0.0 and ::0 to listen to all interfaces. + # set to an absolute path to use a unix local name pipe, certificates + # are not used for that, so key and cert files need not be present. + # control-interface: 127.0.0.1 + # control-interface: ::1 + + # port number for remote control operations. + # control-port: 8953 + + # for localhost, you can disable use of TLS by setting this to "no" + # For local sockets this option is ignored, and TLS is not used. + # control-use-cert: "yes" + + # unbound server key file. + # server-key-file: "/unbound_server.key" + + # unbound server certificate file. + # server-cert-file: "/unbound_server.pem" + + # unbound-control key file. + # control-key-file: "/unbound_control.key" + + # unbound-control certificate file. + # control-cert-file: "/unbound_control.pem" + +# Stub zones. +# Create entries like below, to make all queries for 'example.com' and +# 'example.org' go to the given list of nameservers. list zero or more +# nameservers by hostname or by ipaddress. If you set stub-prime to yes, +# the list is treated as priming hints (default is no). +# With stub-first yes, it attempts without the stub if it fails. +# Consider adding domain-insecure: name and local-zone: name nodefault +# to the server: section if the stub is a locally served zone. +# stub-zone: +# name: "example.com" +# stub-addr: 192.0.2.68 +# stub-prime: no +# stub-first: no +# stub-tls-upstream: no +# stub-no-cache: no +# stub-zone: +# name: "example.org" +# stub-host: ns.example.com. + +# Forward zones +# Create entries like below, to make all queries for 'example.com' and +# 'example.org' go to the given list of servers. These servers have to handle +# recursion to other nameservers. List zero or more nameservers by hostname +# or by ipaddress. Use an entry with name "." to forward all queries. +# If you enable forward-first, it attempts without the forward if it fails. +# forward-zone: +# name: "example.com" +# forward-addr: 192.0.2.68 +# forward-addr: 192.0.2.73@5355 # forward to port 5355. +# forward-first: no +# forward-tls-upstream: no +# forward-no-cache: no +# forward-zone: +# name: "example.org" +# forward-host: fwd.example.com + +forward-zone: + name: "." +$(generate_forward_addr) + +# Authority zones +# The data for these zones is kept locally, from a file or downloaded. +# The data can be served to downstream clients, or used instead of the +# upstream (which saves a lookup to the upstream). The first example +# has a copy of the root for local usage. The second serves example.org +# authoritatively. zonefile: reads from file (and writes to it if you also +# download it), master: fetches with AXFR and IXFR, or url to zonefile. +# With allow-notify: you can give additional (apart from masters) sources of +# notifies. +# auth-zone: +# name: "." +# master: 199.9.14.201 # b.root-servers.net +# master: 192.33.4.12 # c.root-servers.net +# master: 199.7.91.13 # d.root-servers.net +# master: 192.5.5.241 # f.root-servers.net +# master: 192.112.36.4 # g.root-servers.net +# master: 193.0.14.129 # k.root-servers.net +# master: 192.0.47.132 # xfr.cjr.dns.icann.org +# master: 192.0.32.132 # xfr.lax.dns.icann.org +# master: 2001:500:200::b # b.root-servers.net +# master: 2001:500:2::c # c.root-servers.net +# master: 2001:500:2d::d # d.root-servers.net +# master: 2001:500:2f::f # f.root-servers.net +# master: 2001:500:12::d0d # g.root-servers.net +# master: 2001:7fd::1 # k.root-servers.net +# master: 2620:0:2830:202::132 # xfr.cjr.dns.icann.org +# master: 2620:0:2d0:202::132 # xfr.lax.dns.icann.org +# fallback-enabled: yes +# for-downstream: no +# for-upstream: yes +# auth-zone: +# name: "example.org" +# for-downstream: yes +# for-upstream: yes +# zonefile: "example.org.zone" + +# Views +# Create named views. Name must be unique. Map views to requests using +# the access-control-view option. Views can contain zero or more local-zone +# and local-data options. Options from matching views will override global +# options. Global options will be used if no matching view is found. +# With view-first yes, it will try to answer using the global local-zone and +# local-data elements if there is no view specific match. +# view: +# name: "viewname" +# local-zone: "example.com" redirect +# local-data: "example.com A 192.0.2.3" +# local-data-ptr: "192.0.2.3 www.example.com" +# view-first: no +# view: +# name: "anotherview" +# local-zone: "example.com" refuse + +# DNSCrypt +# Caveats: +# 1. the keys/certs cannot be produced by unbound. You can use dnscrypt-wrapper +# for this: https://github.com/cofyc/dnscrypt-wrapper/blob/master/README.md#usage +# 2. dnscrypt channel attaches to an interface. you MUST set interfaces to +# listen on `dnscrypt-port` with the follo0wing snippet: +# server: +# interface: 0.0.0.0@443 +# interface: ::0@443 +# +# Finally, `dnscrypt` config has its own section. +# dnscrypt: +# dnscrypt-enable: yes +# dnscrypt-port: 443 +# dnscrypt-provider: 2.dnscrypt-cert.example.com. +# dnscrypt-secret-key: /path/unbound-conf/keys1/1.key +# dnscrypt-secret-key: /path/unbound-conf/keys2/1.key +# dnscrypt-provider-cert: /path/unbound-conf/keys1/1.cert +# dnscrypt-provider-cert: /path/unbound-conf/keys2/1.cert + +# CacheDB +# Enable external backend DB as auxiliary cache. Specify the backend name +# (default is "testframe", which has no use other than for debugging and +# testing) and backend-specific options. The 'cachedb' module must be +# included in module-config, just before the iterator module. +# cachedb: +# backend: "testframe" +# # secret seed string to calculate hashed keys +# secret-seed: "default" +# +# # For "redis" backend: +# # redis server's IP address or host name +# redis-server-host: 127.0.0.1 +# # redis server's TCP port +# redis-server-port: 6379 +# # timeout (in ms) for communication with the redis server +# redis-timeout: 100 +EOF diff --git a/type/__ungleich_unbound/man.rst b/type/__ungleich_unbound/man.rst new file mode 100644 index 0000000..417b343 --- /dev/null +++ b/type/__ungleich_unbound/man.rst @@ -0,0 +1,74 @@ +cdist-type__ungleich_unbound(7) +=============================== + +NAME +---- +cdist-type__ungleich_unbound - unbound server deployment for ungleich + + +DESCRIPTION +----------- +This unbound (dns resolver and cache) deployment provides DNS64 and fetch +answers from specified upstrean DNS server. This is a singleton type. + +REQUIRED PARAMETERS +------------------- +dns64_prefix + IPv6 prefix used for DNS64. + +forward_addr + DNS servers used to lookup names, can be provided multiple times. It can be + either an IPv4 or IPv6 address but no domain name. + +OPTIONAL PARAMETERS +------------------- +interface + Interface to listen on, can be provided multiple times. Defaults to + '127.0.0.1' and '::1'. + +access_control + Controls which clients are allowed queries to the unbound service (everything + but localhost is refused by default), can be provided multiple times. The + format is described in unbound.conf(5). + +BOOLEAN PARAMETERS +------------------ +disable-ip4 + Do not answer or issue queries over IPv4. Cannot be used alongside the + `--disable-ip6` flag. + +disable-ip6 + Do not answer or issue queries over IPv6. Cannot be used alongside the + `--disable-ip4` flag. + +EXAMPLES +-------- + +.. code-block:: sh + + __ungleich_unbound \ + --interface '::0' \ + --dns64_prefix '2a0a:e5c0:2:10::/96' \ + --forward_addr '2a0a:e5c0:2:1::5' \ + --forward_addr '2a0a:e5c0:2:1::6' \ + --access_control '::0/0 deny' \ + --access_control '2a0a:e5c0::/29 allow' \ + --access_control '2a09:2940::/29 allow' \ + --ip6 + +SEE ALSO +-------- +- `unbound.conf(5) `_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__ungleich_unbound/manifest b/type/__ungleich_unbound/manifest new file mode 100755 index 0000000..525a981 --- /dev/null +++ b/type/__ungleich_unbound/manifest @@ -0,0 +1,79 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + alpine) + __package unbound --state present + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +# Required parameters: +export DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") +export FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") + +# Optional parameters: +if [ -f "$__object/parameter/interface" ]; then + export INTERFACES=$(cat "$__object/parameter/interface") +fi + +if [ -f "$__object/parameter/access_control" ]; then + export ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") +fi + +# Boolean parameters: +if [ -f "$__object/parameter/disable_ip4" ] && \ + [ -f "$__object/parameter/disable_ip6" ]; then + echo "--disable-ip4 and --disable-ip6 cannot be used at the same time." >&2 + exit 1 +fi + +if [ -f "$__object/parameter/disable_ip4" ]; then + export DO_IP4='no' +else + export DO_IP4='yes' +fi + +if [ -f "$__object/parameter/disable_ip6" ]; then + export DO_IP6='no' +else + export DO_IP6='yes' +fi + +# Generate and deploy configuration files. +source_file="$__object/files/unbound.conf" +target_file="/etc/unbound/unbound.conf" + +mkdir -p "$__object/files" +"$__type/files/unbound.conf.sh" > $source_file +require="__package/unbound" __file $target_file \ + --source $source_file \ + --owner root \ + --mode 644 + +# Restart unbound server after reconfiguration. +require="__file/$target_file" __service unbound --action restart diff --git a/type/__ungleich_unbound/parameter/boolean b/type/__ungleich_unbound/parameter/boolean new file mode 100644 index 0000000..a2d5910 --- /dev/null +++ b/type/__ungleich_unbound/parameter/boolean @@ -0,0 +1,2 @@ +disable_ip6 +disable_ip4 diff --git a/type/__ungleich_unbound/parameter/optional_multiple b/type/__ungleich_unbound/parameter/optional_multiple new file mode 100644 index 0000000..ecacd1d --- /dev/null +++ b/type/__ungleich_unbound/parameter/optional_multiple @@ -0,0 +1,2 @@ +access_control +interface diff --git a/type/__ungleich_unbound/parameter/required b/type/__ungleich_unbound/parameter/required new file mode 100644 index 0000000..4ad232f --- /dev/null +++ b/type/__ungleich_unbound/parameter/required @@ -0,0 +1 @@ +dns64_prefix diff --git a/type/__ungleich_unbound/parameter/required_multiple b/type/__ungleich_unbound/parameter/required_multiple new file mode 100644 index 0000000..776abf5 --- /dev/null +++ b/type/__ungleich_unbound/parameter/required_multiple @@ -0,0 +1 @@ +forward_addr diff --git a/type/__ungleich_unbound/singleton b/type/__ungleich_unbound/singleton new file mode 100644 index 0000000..e69de29 From 6b38b248e2227dc29fca104db29f2656c4310202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 14:59:42 +0200 Subject: [PATCH 04/98] Rename __ungleich_unbound into __unbound --- type/{__ungleich_unbound => __unbound}/files/unbound.conf.sh | 0 type/{__ungleich_unbound => __unbound}/man.rst | 2 +- type/{__ungleich_unbound => __unbound}/manifest | 0 type/{__ungleich_unbound => __unbound}/parameter/boolean | 0 .../parameter/optional_multiple | 0 type/{__ungleich_unbound => __unbound}/parameter/required | 0 .../parameter/required_multiple | 0 type/{__ungleich_unbound => __unbound}/singleton | 0 8 files changed, 1 insertion(+), 1 deletion(-) rename type/{__ungleich_unbound => __unbound}/files/unbound.conf.sh (100%) rename type/{__ungleich_unbound => __unbound}/man.rst (98%) rename type/{__ungleich_unbound => __unbound}/manifest (100%) rename type/{__ungleich_unbound => __unbound}/parameter/boolean (100%) rename type/{__ungleich_unbound => __unbound}/parameter/optional_multiple (100%) rename type/{__ungleich_unbound => __unbound}/parameter/required (100%) rename type/{__ungleich_unbound => __unbound}/parameter/required_multiple (100%) rename type/{__ungleich_unbound => __unbound}/singleton (100%) diff --git a/type/__ungleich_unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh similarity index 100% rename from type/__ungleich_unbound/files/unbound.conf.sh rename to type/__unbound/files/unbound.conf.sh diff --git a/type/__ungleich_unbound/man.rst b/type/__unbound/man.rst similarity index 98% rename from type/__ungleich_unbound/man.rst rename to type/__unbound/man.rst index 417b343..6d113b0 100644 --- a/type/__ungleich_unbound/man.rst +++ b/type/__unbound/man.rst @@ -1,4 +1,4 @@ -cdist-type__ungleich_unbound(7) +cdist-type__unbound(7) =============================== NAME diff --git a/type/__ungleich_unbound/manifest b/type/__unbound/manifest similarity index 100% rename from type/__ungleich_unbound/manifest rename to type/__unbound/manifest diff --git a/type/__ungleich_unbound/parameter/boolean b/type/__unbound/parameter/boolean similarity index 100% rename from type/__ungleich_unbound/parameter/boolean rename to type/__unbound/parameter/boolean diff --git a/type/__ungleich_unbound/parameter/optional_multiple b/type/__unbound/parameter/optional_multiple similarity index 100% rename from type/__ungleich_unbound/parameter/optional_multiple rename to type/__unbound/parameter/optional_multiple diff --git a/type/__ungleich_unbound/parameter/required b/type/__unbound/parameter/required similarity index 100% rename from type/__ungleich_unbound/parameter/required rename to type/__unbound/parameter/required diff --git a/type/__ungleich_unbound/parameter/required_multiple b/type/__unbound/parameter/required_multiple similarity index 100% rename from type/__ungleich_unbound/parameter/required_multiple rename to type/__unbound/parameter/required_multiple diff --git a/type/__ungleich_unbound/singleton b/type/__unbound/singleton similarity index 100% rename from type/__ungleich_unbound/singleton rename to type/__unbound/singleton From 3d3a3ef8024a5dc201253bfa5a67b1508510e0d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 15:08:04 +0200 Subject: [PATCH 05/98] Fix shellcheck warnings in newly imported __unbound type --- type/__unbound/files/unbound.conf.sh | 6 +++--- type/__unbound/manifest | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index e713843..effea65 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -7,7 +7,7 @@ generate_interface() { } generate_access_control() { - echo "$ACCESS_CONTROLS" | while read entry; do + echo "$ACCESS_CONTROLS" | while read -r entry; do echo " access-control: $entry" done } @@ -983,12 +983,12 @@ $(generate_forward_addr) # 1. the keys/certs cannot be produced by unbound. You can use dnscrypt-wrapper # for this: https://github.com/cofyc/dnscrypt-wrapper/blob/master/README.md#usage # 2. dnscrypt channel attaches to an interface. you MUST set interfaces to -# listen on `dnscrypt-port` with the follo0wing snippet: +# listen on 'dnscrypt-port' with the follo0wing snippet: # server: # interface: 0.0.0.0@443 # interface: ::0@443 # -# Finally, `dnscrypt` config has its own section. +# Finally, 'dnscrypt' config has its own section. # dnscrypt: # dnscrypt-enable: yes # dnscrypt-port: 443 diff --git a/type/__unbound/manifest b/type/__unbound/manifest index 525a981..a25cdec 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -33,16 +33,20 @@ case "$os" in esac # Required parameters: -export DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") -export FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") +DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") +export DNS64_PREFIX +FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") +export FORWARD_ADDRS # Optional parameters: if [ -f "$__object/parameter/interface" ]; then - export INTERFACES=$(cat "$__object/parameter/interface") + INTERFACES=$(cat "$__object/parameter/interface") + export INTERFACES fi if [ -f "$__object/parameter/access_control" ]; then - export ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") + ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") + export ACCESS_CONTROLS fi # Boolean parameters: @@ -69,9 +73,9 @@ source_file="$__object/files/unbound.conf" target_file="/etc/unbound/unbound.conf" mkdir -p "$__object/files" -"$__type/files/unbound.conf.sh" > $source_file -require="__package/unbound" __file $target_file \ - --source $source_file \ +"$__type/files/unbound.conf.sh" > "$source_file" +require="__package/unbound" __file "$target_file" \ + --source "$source_file" \ --owner root \ --mode 644 From 89f0dc90ead21d338c271cb283cd10d1ebe57ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 15:09:07 +0200 Subject: [PATCH 06/98] Add changelog entry for __unbound type --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2ebad0..a61c82d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ # cdist-contrib changes +* 2020-06-03: New type: __unbound (Timothée Floure) * 2020-04-28: New type: __find_exec (Ander Punnar) From 7b9ffb4a413693556a6c19258ba3a17d1614613f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:46:18 +0200 Subject: [PATCH 07/98] __unbound: wire remote control configuration --- type/__unbound/files/unbound.conf.sh | 12 ++++++------ type/__unbound/gencode-remote | 8 ++++++++ type/__unbound/man.rst | 6 ++++++ type/__unbound/manifest | 17 +++++++++++++++++ type/__unbound/parameter/boolean | 1 + type/__unbound/parameter/default/rc_interface | 1 + type/__unbound/parameter/optional | 1 + 7 files changed, 40 insertions(+), 6 deletions(-) create mode 100755 type/__unbound/gencode-remote create mode 100644 type/__unbound/parameter/default/rc_interface create mode 100644 type/__unbound/parameter/optional diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index effea65..46d2824 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -857,14 +857,14 @@ python: remote-control: # Enable remote control with unbound-control(8) here. # set up the keys and certificates with unbound-control-setup. - # control-enable: no + control-enable: $RC_ENABLE # what interfaces are listened to for remote control. # give 0.0.0.0 and ::0 to listen to all interfaces. # set to an absolute path to use a unix local name pipe, certificates # are not used for that, so key and cert files need not be present. # control-interface: 127.0.0.1 - # control-interface: ::1 + control-interface: $RC_INTERFACE # port number for remote control operations. # control-port: 8953 @@ -874,16 +874,16 @@ remote-control: # control-use-cert: "yes" # unbound server key file. - # server-key-file: "/unbound_server.key" + server-key-file: "$RC_SERVER_KEY_FILE" # unbound server certificate file. - # server-cert-file: "/unbound_server.pem" + server-cert-file: "$RC_SERVER_CERT_FILE" # unbound-control key file. - # control-key-file: "/unbound_control.key" + control-key-file: "$RC_CONTROL_KEY_FILE" # unbound-control certificate file. - # control-cert-file: "/unbound_control.pem" + control-cert-file: "$RC_CONTROL_CERT_FILE" # Stub zones. # Create entries like below, to make all queries for 'example.com' and diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote new file mode 100755 index 0000000..5d2bb4a --- /dev/null +++ b/type/__unbound/gencode-remote @@ -0,0 +1,8 @@ +#!/bin/sh + +UNBOUND_CERTS_DIR=/etc/unbound + +if [ -f "$__object/parameter/enable_rc" ]; then + echo "unbound-control-setup -d $UNBOUND_CERTS_DIR" + echo "chown unbound:unbound $UNBOUND_CERTS_DIR/*.pem $UNBOUND_CERTS_DIR/*.key" +fi diff --git a/type/__unbound/man.rst b/type/__unbound/man.rst index 6d113b0..679e601 100644 --- a/type/__unbound/man.rst +++ b/type/__unbound/man.rst @@ -31,6 +31,9 @@ access_control but localhost is refused by default), can be provided multiple times. The format is described in unbound.conf(5). +rc_interface + Address or path to socket used for remote control (see `--enable_control`. Defaults to `127.0.0.1`). + BOOLEAN PARAMETERS ------------------ disable-ip4 @@ -41,6 +44,9 @@ disable-ip6 Do not answer or issue queries over IPv6. Cannot be used alongside the `--disable-ip4` flag. +enable_rc + Enable remote control (see `unbound-control(8)`). + EXAMPLES -------- diff --git a/type/__unbound/manifest b/type/__unbound/manifest index a25cdec..d85ad66 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -49,6 +49,11 @@ if [ -f "$__object/parameter/access_control" ]; then export ACCESS_CONTROLS fi +if [ -f "$__object/parameter/rc_interface" ]; then + RC_INTERFACE=$(cat "$__object/parameter/rc_interface") + export RC_INTERFACE +fi + # Boolean parameters: if [ -f "$__object/parameter/disable_ip4" ] && \ [ -f "$__object/parameter/disable_ip6" ]; then @@ -68,6 +73,18 @@ else export DO_IP6='yes' fi +if [ -f "$__object/parameter/enable_rc" ]; then + export RC_ENABLE='yes' +else + export RC_ENABLE='no' +fi + +# Certs for remote control: +export RC_SERVER_KEY_FILE='/etc/unbound/unbound_server.key' +export RC_SERVER_CERT_FILE='/etc/unbound/unbound_server.pem' +export RC_CONTROL_KEY_FILE='/etc/unbound/unbound_control.key' +export RC_CONTROL_CERT_FILE='/etc/unbound/unbound_control.pem' + # Generate and deploy configuration files. source_file="$__object/files/unbound.conf" target_file="/etc/unbound/unbound.conf" diff --git a/type/__unbound/parameter/boolean b/type/__unbound/parameter/boolean index a2d5910..b6e53a1 100644 --- a/type/__unbound/parameter/boolean +++ b/type/__unbound/parameter/boolean @@ -1,2 +1,3 @@ disable_ip6 disable_ip4 +enable_rc diff --git a/type/__unbound/parameter/default/rc_interface b/type/__unbound/parameter/default/rc_interface new file mode 100644 index 0000000..7b9ad53 --- /dev/null +++ b/type/__unbound/parameter/default/rc_interface @@ -0,0 +1 @@ +127.0.0.1 diff --git a/type/__unbound/parameter/optional b/type/__unbound/parameter/optional new file mode 100644 index 0000000..0826b6e --- /dev/null +++ b/type/__unbound/parameter/optional @@ -0,0 +1 @@ +rc_interface From 1e45d85d4d3d171b211e957e099f0e1c1580fe57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:53:56 +0200 Subject: [PATCH 08/98] Add new __unbound_exporter type --- type/__unbound_exporter/files/openrc-service | 12 ++++ type/__unbound_exporter/gencode-remote | 46 ++++++++++++++ type/__unbound_exporter/man.rst | 63 ++++++++++++++++++++ type/__unbound_exporter/manifest | 44 ++++++++++++++ type/__unbound_exporter/parameter/required | 1 + type/__unbound_exporter/singleton | 0 6 files changed, 166 insertions(+) create mode 100644 type/__unbound_exporter/files/openrc-service create mode 100755 type/__unbound_exporter/gencode-remote create mode 100644 type/__unbound_exporter/man.rst create mode 100755 type/__unbound_exporter/manifest create mode 100644 type/__unbound_exporter/parameter/required create mode 100644 type/__unbound_exporter/singleton diff --git a/type/__unbound_exporter/files/openrc-service b/type/__unbound_exporter/files/openrc-service new file mode 100644 index 0000000..6caed5e --- /dev/null +++ b/type/__unbound_exporter/files/openrc-service @@ -0,0 +1,12 @@ +#!/sbin/openrc-run + +name=$RC_SVCNAME +command="/usr/local/bin/unbound_exporter" +command_args="" +command_user="unbound" +command_background="yes" +pidfile="/var/run/$RC_SVCNAME.pid" + +depend() { + need unbound +} diff --git a/type/__unbound_exporter/gencode-remote b/type/__unbound_exporter/gencode-remote new file mode 100755 index 0000000..ef7ef9b --- /dev/null +++ b/type/__unbound_exporter/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +upstream=https://github.com/wish/unbound_exporter/archive +version=$(cat $__object/parameter/version) +release="unbound_exporter-$version" + +cat << EOF +if command -v unbound_exporter +then + # already installed - ignoring. + echo "Nothing to do -" +else + # Initialize working directory + workdir=\$(mktemp -d) + cd \$workdir + + # Download and extract sources for requested release. + curl -L $upstream/v$version.tar.gz --output $release.tar.gz + tar xf $release.tar.gz + + # Build and install binary. + cd $release + go build + install -m755 unbound_exporter /usr/local/bin/ + + # Clean up! + rm -r \$workdir +fi +EOF diff --git a/type/__unbound_exporter/man.rst b/type/__unbound_exporter/man.rst new file mode 100644 index 0000000..934bdd7 --- /dev/null +++ b/type/__unbound_exporter/man.rst @@ -0,0 +1,63 @@ +cdist-type__unbound_exporter(7) +=============================== + +NAME +---- +cdist-type__unbound_exporter - A prometheus exporter for unbound + + +DESCRIPTION +----------- +Simple Prometheus metrics exporter for the Unbound DNS +resolver. It leverages the unbound remote control endpoint +and exposes metrics on port 9167. + + +REQUIRED PARAMETERS +------------------- +version + unbound_exporter release to be used. + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __unbound \ + --interface '::0' \ + --forward_addr '2a0a:e5c0:2:1::5' \ + --forward_addr '2a0a:e5c0:2:1::6' \ + --access_control '::0/0 deny' \ + --access_control '2a0a:e5c0::/29 allow' \ + --access_control '2a09:2940::/29 allow' \ + --disable_ip4 \ + --enable_rc \ + --rc_interface '::1' + + __unbound_exporter --version 0.1.3 + +SEE ALSO +-------- +:strong:`cdist-type__unbound(7)` + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__unbound_exporter/manifest b/type/__unbound_exporter/manifest new file mode 100755 index 0000000..9f71834 --- /dev/null +++ b/type/__unbound_exporter/manifest @@ -0,0 +1,44 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + alpine) + # Used in gencode-remote. + __package curl + __package tar + __package go + __package libc-dev + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +__file /etc/init.d/unbound_exporter \ + --source "$__type/files/openrc-service" \ + --mode 755 + +require="__file/etc/init.d/unbound_exporter" __service unbound_exporter --action start +require="__file/etc/init.d/unbound_exporter" __start_on_boot unbound_exporter diff --git a/type/__unbound_exporter/parameter/required b/type/__unbound_exporter/parameter/required new file mode 100644 index 0000000..088eda4 --- /dev/null +++ b/type/__unbound_exporter/parameter/required @@ -0,0 +1 @@ +version diff --git a/type/__unbound_exporter/singleton b/type/__unbound_exporter/singleton new file mode 100644 index 0000000..e69de29 From d14011c50b8dbfbddf1adfab9e7823e462ef042c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:55:34 +0200 Subject: [PATCH 09/98] Sync CHANGELOG with __unbound* changes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a61c82d..df26f7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # cdist-contrib changes +* 2020-06-07: New type: __unbound_exporter (Timothée Floure) +* 2020-06-07: Extended type: wire remote control configuration for __unbond (Timothée Floure) * 2020-06-03: New type: __unbound (Timothée Floure) * 2020-04-28: New type: __find_exec (Ander Punnar) From 17f6a4dd471193369e17b0287817fc9cd43895b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:57:51 +0200 Subject: [PATCH 10/98] Only shellcheck .sh files in type/*/files/ --- scripts/run-shellcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-shellcheck.sh b/scripts/run-shellcheck.sh index 769f853..475f426 100755 --- a/scripts/run-shellcheck.sh +++ b/scripts/run-shellcheck.sh @@ -15,7 +15,7 @@ check () { } check -path "*/explorer/*" -check -path "*/files/*" +check -path "*/files/*.sh" check -name manifest check -name gencode-local check -name gencode-remote From bffc38ad52a54d7c80354fac46bf9f556fb4e0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:58:14 +0200 Subject: [PATCH 11/98] __unbound_exporter: make shellcheck happy again --- type/__unbound_exporter/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__unbound_exporter/gencode-remote b/type/__unbound_exporter/gencode-remote index ef7ef9b..fcd85fb 100755 --- a/type/__unbound_exporter/gencode-remote +++ b/type/__unbound_exporter/gencode-remote @@ -18,7 +18,7 @@ # along with cdist. If not, see . upstream=https://github.com/wish/unbound_exporter/archive -version=$(cat $__object/parameter/version) +version=$(cat "$__object/parameter/version") release="unbound_exporter-$version" cat << EOF From 4ff703e6aa74e18b679302e4b02cd2b9fd41fe6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 10:34:35 +0200 Subject: [PATCH 12/98] Fine tuning prometheus exporter for unbound --- type/__unbound/gencode-remote | 8 ++++++++ type/__unbound/manifest | 3 --- type/__unbound_exporter/manifest | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote index 5d2bb4a..d12a395 100755 --- a/type/__unbound/gencode-remote +++ b/type/__unbound/gencode-remote @@ -6,3 +6,11 @@ if [ -f "$__object/parameter/enable_rc" ]; then echo "unbound-control-setup -d $UNBOUND_CERTS_DIR" echo "chown unbound:unbound $UNBOUND_CERTS_DIR/*.pem $UNBOUND_CERTS_DIR/*.key" fi + +cat << EOF +if pgrep unbound; then + service unbound reload +else + service unbounb start +fi +EOF diff --git a/type/__unbound/manifest b/type/__unbound/manifest index d85ad66..346f30c 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -95,6 +95,3 @@ require="__package/unbound" __file "$target_file" \ --source "$source_file" \ --owner root \ --mode 644 - -# Restart unbound server after reconfiguration. -require="__file/$target_file" __service unbound --action restart diff --git a/type/__unbound_exporter/manifest b/type/__unbound_exporter/manifest index 9f71834..3602e47 100755 --- a/type/__unbound_exporter/manifest +++ b/type/__unbound_exporter/manifest @@ -26,6 +26,7 @@ case "$os" in # Used in gencode-remote. __package curl __package tar + __package openssl __package go __package libc-dev ;; From f6a36a60c0f92ae212c2e1274cba64ffdc0c856c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 25 Jun 2020 11:15:23 +0200 Subject: [PATCH 13/98] Import __matrix_riot from ungleich's dot-cdist --- type/__matrix_riot/files/config.json.sh | 90 +++++++++++++++ type/__matrix_riot/gencode-remote | 73 +++++++++++++ type/__matrix_riot/man.rst | 87 +++++++++++++++ type/__matrix_riot/manifest | 103 ++++++++++++++++++ type/__matrix_riot/parameter/boolean | 1 + type/__matrix_riot/parameter/default/brand | 1 + .../parameter/default/cookie_policy_url | 1 + .../parameter/default/default_country_code | 1 + .../parameter/default/default_server_name | 1 + .../parameter/default/default_server_url | 1 + type/__matrix_riot/parameter/default/owner | 1 + .../parameter/default/privacy_policy_url | 1 + .../parameter/default/room_directory_servers | 1 + type/__matrix_riot/parameter/optional | 13 +++ .../__matrix_riot/parameter/optional_multiple | 1 + type/__matrix_riot/parameter/required | 2 + 16 files changed, 378 insertions(+) create mode 100755 type/__matrix_riot/files/config.json.sh create mode 100755 type/__matrix_riot/gencode-remote create mode 100644 type/__matrix_riot/man.rst create mode 100755 type/__matrix_riot/manifest create mode 100644 type/__matrix_riot/parameter/boolean create mode 100644 type/__matrix_riot/parameter/default/brand create mode 100644 type/__matrix_riot/parameter/default/cookie_policy_url create mode 100644 type/__matrix_riot/parameter/default/default_country_code create mode 100644 type/__matrix_riot/parameter/default/default_server_name create mode 100644 type/__matrix_riot/parameter/default/default_server_url create mode 100644 type/__matrix_riot/parameter/default/owner create mode 100644 type/__matrix_riot/parameter/default/privacy_policy_url create mode 100644 type/__matrix_riot/parameter/default/room_directory_servers create mode 100644 type/__matrix_riot/parameter/optional create mode 100644 type/__matrix_riot/parameter/optional_multiple create mode 100644 type/__matrix_riot/parameter/required diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_riot/files/config.json.sh new file mode 100755 index 0000000..e7fca72 --- /dev/null +++ b/type/__matrix_riot/files/config.json.sh @@ -0,0 +1,90 @@ +#!/bin/sh +# +# Upstream configuration guide/documentation: +# https://github.com/vector-im/riot-web/blob/develop/docs/config.md + +generate_embedded_pages () { + if [ $EMBED_HOMEPAGE ]; then + cat << EOF + "embeddedPages": { + "homeUrl": "home.html" + }, +EOF + fi +} + +generate_jitsi_config () { + if [ "$JITSI_DOMAIN" != "" ]; then + cat << EOF + "jitsi": { + "preferredDomain": "$JITSI_DOMAIN" + }, +EOF + fi +} + +generate_branding () { + echo '"branding": {' + + if [ "$BRANDING_AUTH_HEADER_LOGO_URL" != "" ]; then + cat << EOF + "authHeaderLogoUrl": "$BRANDING_AUTH_HEADER_LOGO_URL", +EOF + fi + + if [ "$BRANDING_AUTH_FOOTER_LINKS" != "" ]; then + cat << EOF + "authFooterLinks": "$BRANDING_AUTH_FOOTER_LINKS", +EOF + fi + + cat << EOF + "welcomeBackgroundUrl": "themes/riot/img/backgrounds/valley.jpg" +EOF + echo '},' +} + +cat << EOF +{ + "default_server_config": { + "m.homeserver": { + "base_url": "$DEFAULT_SERVER_URL", + "server_name": "$DEFAULT_SERVER_NAME" + }, + "m.identity_server": { + "base_url": "https://vector.im" + } + }, + "brand": "$BRAND", + $(generate_branding) + "defaultCountryCode": "$DEFAULT_COUNTRY_CODE", + "integrations_ui_url": "https://scalar.vector.im/", + "integrations_rest_url": "https://scalar.vector.im/api", + "integrations_widgets_urls": [ + "https://scalar.vector.im/_matrix/integrations/v1", + "https://scalar.vector.im/api", + "https://scalar-staging.vector.im/_matrix/integrations/v1", + "https://scalar-staging.vector.im/api", + "https://scalar-staging.riot.im/scalar/api" + ], + "bug_report_endpoint_url": "https://riot.im/bugreports/submit", + "roomDirectory": { + "servers": [ + $ROOM_DIRECTORY_SERVERS + ] + }, + "disable_custom_urls": "$DISABLE_CUSTOM_URLS", + $(generate_embedded_pages) + $(generate_jitsi_config) + "terms_and_conditions_links": [ + { + "url": "$PRIVACY_POLICY_URL", + "text": "Privacy Policy" + }, + { + "url": "$COOKIE_POLICY_URL", + "text": "Cookie Policy" + } + ] +} +EOF diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_riot/gencode-remote new file mode 100755 index 0000000..97c4777 --- /dev/null +++ b/type/__matrix_riot/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh -e +# +# 2019 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +VERSION=$(cat "$__object/parameter/version") +INSTALL_DIR=$(cat "$__object/parameter/install_dir") +OWNER=$(cat "$__object/parameter/owner") + +src="riot-v$VERSION" +archive="$src.tar.gz" +config='config.json' +homepage='home.html' +welcomepage='welcome.html' +welcomelogo='welcome-logo.svg' +url="https://github.com/vector-im/riot-web/releases/download/v$VERSION/$archive" + +# tar and curl are installed by the __matrix-riot manifest. mktemp is usually +# provided by coreutils and assumed installed. +cat << EOF +set -e + +# Ensure that coreutils is installed. +if [ ! -x \$(which mktemp) ]; then + echo "mktemp is not available on the remote host." >&2 + exit 1 +fi + +# Create temporary working directory. +tmpdir=\$(mktemp -d) +custom_files_dir="\$tmpdir/custom_files" +cd \$tmpdir + +# Download and extract sources. +curl -L '$url' > $archive +tar xf $archive + +# Backup files deployed by __matrix_riot. +mkdir -p \$custom_files_dir +for file in $INSTALL_DIR/cdist/*; do + cp "\$file" "\$custom_files_dir" +done + +# Deploy sources and restore configuration. +rm -r '$INSTALL_DIR' +mv '$src' '$INSTALL_DIR' + +for file in \$custom_files_dir/*; do + cp "\$file" '$INSTALL_DIR' +done + +# Chown deployed files to requested owner. +chown -R '$OWNER' '$INSTALL_DIR' + +# Remove temporary working directory. +cd / +rm -r \$tmpdir +EOF diff --git a/type/__matrix_riot/man.rst b/type/__matrix_riot/man.rst new file mode 100644 index 0000000..2fe8eae --- /dev/null +++ b/type/__matrix_riot/man.rst @@ -0,0 +1,87 @@ +cdist-type__matrix_riot(7) +====================== + +NAME +---- +cdist-type__matrix_riot - Install and configure Riot, a web Matrix client. + + +DESCRIPTION +----------- +This type install and configure the Riot web client. + + +REQUIRED PARAMETERS +------------------- +install_dir + Root directory of Riot's static files. + +version + Release of Riot to install. + +OPTIONAL PARAMETERS +------------------- +default_server_name + Name of matrix homeserver to connect to, defaults to 'matrix.org'. + +default_server_url + URL of matrix homeserver to connect to, defaults to 'https://matrix-client.matrix.org'. + +owner + Owner of the deployed files, passed to `chown`. Defaults to 'root'. + +brand + Web UI branding, defaults to 'Riot'. + +default_country_code + ISO 3166 alpha2 country code to use when showing country selectors, such as + phone number inputs. Defaults to GB. + +privacy_policy_url + Defaults to 'https://riot.im/privacy'. + +cookie_policy_url + Defaults to 'https://matrix.org/docs/guides/riot_im_cookie_policy'. + +jitsi_domain + Domain name of preferred Jitsi instance (default is jitsi.riot.im). This is + used whenever a user clicks on the voice/video call buttons. + +homepage + Path to custom homepage, displayed once logged in. + +welcomepage + Path to custom welcome (= login) page. + +custom_asset + Serve a file a the top-level directory (e.g. /my-custom-logo.svg). Can be specified multiple times. + +BOOLEAN PARAMETERS +------------------- +disable_custom_urls + Disallow the user to change the default homeserver when signing up or logging in. + +EXAMPLES +-------- + +.. code-block:: sh + + __matrix_riot my-riot --install_dir /var/www/riot-web --version 1.5.6 + + +SEE ALSO +-------- +- `cdist-type__matrix_synapse(7) `_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2019 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__matrix_riot/manifest b/type/__matrix_riot/manifest new file mode 100755 index 0000000..a843f4b --- /dev/null +++ b/type/__matrix_riot/manifest @@ -0,0 +1,103 @@ +#!/bin/sh -e +# +# 2019 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +INSTALL_DIR=$(cat "$__object/parameter/install_dir") + +export DEFAULT_SERVER_NAME=$(cat "$__object/parameter/default_server_name") +export DEFAULT_SERVER_URL=$(cat "$__object/parameter/default_server_url") +export BRAND=$(cat "$__object/parameter/brand") +export DEFAULT_COUNTRY_CODE=$(cat "$__object/parameter/default_country_code") +export ROOM_DIRECTORY_SERVERS=$(cat "$__object/parameter/room_directory_servers") +export PRIVACY_POLICY_URL=$(cat "$__object/parameter/privacy_policy_url") +export COOKIE_POLICY_URL=$(cat "$__object/parameter/cookie_policy_url") + +if [ -f "$__object/parameter/jitsi_domain" ]; then + export JITSI_DOMAIN=$(cat "$__object/parameter/jitsi_domain") +fi + +if [ -f "$__object/parameter/branding_auth_header_logo_url" ]; then + export BRANDING_AUTH_HEADER_LOGO_URL=$(cat "$__object/parameter/branding_auth_header_logo_url") +fi + +if [ -f "$__object/parameter/branding_auth_footer_links" ]; then + export BRANDING_AUTH_FOOTER_LINKS=$(cat "$__object/parameter/branding_auth_footer_links") +fi + +if [ -f "$__object/parameter/homepage" ]; then + export EMBED_HOMEPAGE=1 + homepage=$(cat "$__object/parameter/homepage") +fi + +if [ -f "$__object/parameter/welcomepage" ]; then + export EMBED_WELCOMEPAGE=1 + welcomepage=$(cat "$__object/parameter/welcomepage") +fi + +if [ -f "$__object/parameter/custom_asset" ]; then + for file in $(cat "$__object/parameter/custom_asset"); do + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename $file)" \ + --source "$file" \ + --mode 0664 \ + --state present + done +fi + +if [ -f "$__object/parameter/disable_custom_urls" ]; then + export DISABLE_CUSTOM_URLS='true' +else + export DISABLE_CUSTOM_URLS='false' +fi + +# Owner of the uploaded files. +owner=$(cat "$__object/parameter/owner") + +# Ensure that curl and tar are installed, as they will be required by the +# gencode-remote script. +__package curl --state present +__package tar --state present + +# Generate and deploy configuration file. +mkdir -p "$__object/files" +"$__type/files/config.json.sh" > "$__object/files/config.json" + +# Install the config.json configuration file. The application's sources are +# downloaded and deployed by gencode-remote. +__directory "$INSTALL_DIR/cdist" \ + --owner "$owner" --mode 0755 --parents \ + --state present + +require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/config.json" \ + --source "$__object/files/config.json" \ + --mode 0664 \ + --state present + +if [ $EMBED_HOMEPAGE ]; then + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/home.html" \ + --source "$homepage" \ + --mode 0664 \ + --state present +fi + +if [ $EMBED_WELCOMEPAGE ]; then + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/welcome.html" \ + --source "$welcomepage" \ + --mode 0664 \ + --state present +fi diff --git a/type/__matrix_riot/parameter/boolean b/type/__matrix_riot/parameter/boolean new file mode 100644 index 0000000..4d77768 --- /dev/null +++ b/type/__matrix_riot/parameter/boolean @@ -0,0 +1 @@ +disable_custom_urls diff --git a/type/__matrix_riot/parameter/default/brand b/type/__matrix_riot/parameter/default/brand new file mode 100644 index 0000000..e8095bb --- /dev/null +++ b/type/__matrix_riot/parameter/default/brand @@ -0,0 +1 @@ +Riot diff --git a/type/__matrix_riot/parameter/default/cookie_policy_url b/type/__matrix_riot/parameter/default/cookie_policy_url new file mode 100644 index 0000000..04e9c2b --- /dev/null +++ b/type/__matrix_riot/parameter/default/cookie_policy_url @@ -0,0 +1 @@ +https://matrix.org/docs/guides/riot_im_cookie_policy diff --git a/type/__matrix_riot/parameter/default/default_country_code b/type/__matrix_riot/parameter/default/default_country_code new file mode 100644 index 0000000..30ac4a3 --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_country_code @@ -0,0 +1 @@ +GB diff --git a/type/__matrix_riot/parameter/default/default_server_name b/type/__matrix_riot/parameter/default/default_server_name new file mode 100644 index 0000000..5528ffd --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_server_name @@ -0,0 +1 @@ +matrix.org diff --git a/type/__matrix_riot/parameter/default/default_server_url b/type/__matrix_riot/parameter/default/default_server_url new file mode 100644 index 0000000..2cb9227 --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_server_url @@ -0,0 +1 @@ +https://matrix-client.matrix.org diff --git a/type/__matrix_riot/parameter/default/owner b/type/__matrix_riot/parameter/default/owner new file mode 100644 index 0000000..d8649da --- /dev/null +++ b/type/__matrix_riot/parameter/default/owner @@ -0,0 +1 @@ +root diff --git a/type/__matrix_riot/parameter/default/privacy_policy_url b/type/__matrix_riot/parameter/default/privacy_policy_url new file mode 100644 index 0000000..4cdd12c --- /dev/null +++ b/type/__matrix_riot/parameter/default/privacy_policy_url @@ -0,0 +1 @@ +https://riot.im/privacy diff --git a/type/__matrix_riot/parameter/default/room_directory_servers b/type/__matrix_riot/parameter/default/room_directory_servers new file mode 100644 index 0000000..4ea73ad --- /dev/null +++ b/type/__matrix_riot/parameter/default/room_directory_servers @@ -0,0 +1 @@ +"matrix.org" diff --git a/type/__matrix_riot/parameter/optional b/type/__matrix_riot/parameter/optional new file mode 100644 index 0000000..21a2faf --- /dev/null +++ b/type/__matrix_riot/parameter/optional @@ -0,0 +1,13 @@ +default_server_url +default_server_name +brand +default_country_code +privacy_policy_url +cookie_policy_url +room_directory_servers +owner +homepage +welcomepage +jitsi_domain +branding_auth_header_logo_url +branding_auth_footer_links diff --git a/type/__matrix_riot/parameter/optional_multiple b/type/__matrix_riot/parameter/optional_multiple new file mode 100644 index 0000000..4c2ca54 --- /dev/null +++ b/type/__matrix_riot/parameter/optional_multiple @@ -0,0 +1 @@ +custom_asset diff --git a/type/__matrix_riot/parameter/required b/type/__matrix_riot/parameter/required new file mode 100644 index 0000000..a76477e --- /dev/null +++ b/type/__matrix_riot/parameter/required @@ -0,0 +1,2 @@ +version +install_dir From 331461219387b733d7978171aa0947954ecedbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 11 Jul 2020 09:21:37 +0200 Subject: [PATCH 14/98] __matrix_riot: improve friendship with shellcheck --- type/__matrix_riot/files/config.json.sh | 2 +- type/__matrix_riot/gencode-remote | 4 ---- type/__matrix_riot/manifest | 9 ++++++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_riot/files/config.json.sh index e7fca72..6cd1648 100755 --- a/type/__matrix_riot/files/config.json.sh +++ b/type/__matrix_riot/files/config.json.sh @@ -4,7 +4,7 @@ # https://github.com/vector-im/riot-web/blob/develop/docs/config.md generate_embedded_pages () { - if [ $EMBED_HOMEPAGE ]; then + if [ "$EMBED_HOMEPAGE" != "" ]; then cat << EOF "embeddedPages": { "homeUrl": "home.html" diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_riot/gencode-remote index 97c4777..dbc886c 100755 --- a/type/__matrix_riot/gencode-remote +++ b/type/__matrix_riot/gencode-remote @@ -24,10 +24,6 @@ OWNER=$(cat "$__object/parameter/owner") src="riot-v$VERSION" archive="$src.tar.gz" -config='config.json' -homepage='home.html' -welcomepage='welcome.html' -welcomelogo='welcome-logo.svg' url="https://github.com/vector-im/riot-web/releases/download/v$VERSION/$archive" # tar and curl are installed by the __matrix-riot manifest. mktemp is usually diff --git a/type/__matrix_riot/manifest b/type/__matrix_riot/manifest index a843f4b..544bd96 100755 --- a/type/__matrix_riot/manifest +++ b/type/__matrix_riot/manifest @@ -16,7 +16,10 @@ # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -# + +# Ignore "Declare and assign separately to avoid masking return values. [SC2155]" +# => not relevant for the type arguments. +# shellcheck disable=SC2155 INSTALL_DIR=$(cat "$__object/parameter/install_dir") @@ -51,8 +54,8 @@ if [ -f "$__object/parameter/welcomepage" ]; then fi if [ -f "$__object/parameter/custom_asset" ]; then - for file in $(cat "$__object/parameter/custom_asset"); do - require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename $file)" \ + "$__object/parameter/custom_asset" | while IFS= read -r file; do + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename "$file")" \ --source "$file" \ --mode 0664 \ --state present From 70047d10b1878e46a53c26a2bc8c50779c89f0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 20 Jul 2020 17:20:33 +0200 Subject: [PATCH 15/98] Initial __netbox type --- type/__netbox/files/configuration.py.sh | 258 ++++++++++++++++++++++++ type/__netbox/files/gunicorn.py | 16 ++ type/__netbox/files/netbox-rq.service | 21 ++ type/__netbox/files/netbox.service | 22 ++ type/__netbox/gencode-remote | 58 ++++++ type/__netbox/manifest | 47 +++++ type/__netbox/parameter/required | 5 + type/__netbox/singleton | 0 8 files changed, 427 insertions(+) create mode 100755 type/__netbox/files/configuration.py.sh create mode 100644 type/__netbox/files/gunicorn.py create mode 100644 type/__netbox/files/netbox-rq.service create mode 100644 type/__netbox/files/netbox.service create mode 100755 type/__netbox/gencode-remote create mode 100644 type/__netbox/manifest create mode 100644 type/__netbox/parameter/required create mode 100644 type/__netbox/singleton diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh new file mode 100755 index 0000000..b7e1597 --- /dev/null +++ b/type/__netbox/files/configuration.py.sh @@ -0,0 +1,258 @@ +#!/bin/sh + +cat << EOF +######################### +# # +# Required settings # +# # +######################### + +# This is a list of valid fully-qualified domain names (FQDNs) for the NetBox server. NetBox will not permit write +# access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. +# +# Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] +ALLOWED_HOSTS = [ '$ALLOWED_HOST' ] + +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases +DATABASE = { + 'NAME': '$DATABASE_NAME', # Database name + 'USER': '$DATABASE_USER', # PostgreSQL username + 'PASSWORD': '$DATABASE_PASSWORD', # PostgreSQL password + 'HOST': 'localhost', # Database server + 'PORT': '', # Database port (leave blank for default) + 'CONN_MAX_AGE': 300, # Max database connection age +} + +# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended +# to use two separate database IDs. +REDIS = { + 'tasks': { + 'HOST': 'localhost', + 'PORT': 6379, + # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} + +# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. +# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and +# symbols. NetBox will not run without this defined. For more information, see +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = '$SECRET_KEY' + + +######################### +# # +# Optional settings # +# # +######################### + +# Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +# application errors (assuming correct email settings are provided). +ADMINS = [ + # ['John Doe', 'jdoe@example.com'], +] + +# URL schemes that are allowed within links in NetBox +ALLOWED_URL_SCHEMES = ( + 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +) + +# Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same +# content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. +BANNER_TOP = '' +BANNER_BOTTOM = '' + +# Text to include on the login page above the login form. HTML is allowed. +BANNER_LOGIN = '' + +# Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: +# BASE_PATH = 'netbox/' +BASE_PATH = '' + +# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) +CACHE_TIMEOUT = 900 + +# Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) +CHANGELOG_RETENTION = 90 + +# API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be +# allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or +# CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers +CORS_ORIGIN_ALLOW_ALL = False +CORS_ORIGIN_WHITELIST = [ + # 'https://hostname.example.com', +] +CORS_ORIGIN_REGEX_WHITELIST = [ + # r'^(https?://)?(\w+\.)?example\.com$', +] + +# Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal +# sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging +# on a production system. +DEBUG = False + +# Email settings +EMAIL = { + 'SERVER': 'localhost', + 'PORT': 25, + 'USERNAME': '', + 'PASSWORD': '', + 'USE_SSL': False, + 'USE_TLS': False, + 'TIMEOUT': 10, # seconds + 'FROM_EMAIL': '', +} + +# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table +# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True. +ENFORCE_GLOBAL_UNIQUE = False + +# Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and +# by anonymous users. List models in the form \`.\`. Add '*' to this list to exempt all models. +EXEMPT_VIEW_PERMISSIONS = [ + # 'dcim.site', + # 'dcim.region', + # 'ipam.prefix', +] + +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +# HTTP_PROXIES = { +# 'http': 'http://10.10.1.10:3128', +# 'https': 'http://10.10.1.10:1080', +# } + +# IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing +# NetBox from an internal IP. +INTERNAL_IPS = ('127.0.0.1', '::1') + +# Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: +# https://docs.djangoproject.com/en/stable/topics/logging/ +LOGGING = {} + +# Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users +# are permitted to access most data in NetBox (excluding secrets) but not make any changes. +LOGIN_REQUIRED = False + +# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to +# re-authenticate. (Default: 1209600 [14 days]) +LOGIN_TIMEOUT = None + +# Setting this to True will display a "maintenance mode" banner at the top of every page. +MAINTENANCE_MODE = False + +# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. +# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request +# all objects by specifying "?limit=0". +MAX_PAGE_SIZE = 1000 + +# The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that +# the default value of this setting is derived from the installed location. +# MEDIA_ROOT = '/opt/netbox/netbox/media' + +# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } + +# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' +METRICS_ENABLED = False + +# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM. +NAPALM_USERNAME = '' +NAPALM_PASSWORD = '' + +# NAPALM timeout (in seconds). (Default: 30) +NAPALM_TIMEOUT = 30 + +# NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +# be provided as a dictionary. +NAPALM_ARGS = {} + +# Determine how many objects to display per page within a list. (Default: 50) +PAGINATE_COUNT = 50 + +# Enable installed plugins. Add the name of each plugin to the list. +PLUGINS = [] + +# Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# PLUGINS_CONFIG = { +# 'my_plugin': { +# 'foo': 'bar', +# 'buzz': 'bazz' +# } +# } + +# When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to +# prefer IPv4 instead. +PREFER_IPV4 = False + +# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1. +RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = 22 +RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 + +# Remote authentication support +REMOTE_AUTH_ENABLED = False +REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' +REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' +REMOTE_AUTH_AUTO_CREATE_USER = True +REMOTE_AUTH_DEFAULT_GROUPS = [] +REMOTE_AUTH_DEFAULT_PERMISSIONS = [] + +# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. +RELEASE_CHECK_TIMEOUT = 24 * 3600 + +# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the +# version check or use the URL below to check for release in the official NetBox repository. +RELEASE_CHECK_URL = None +# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +# The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +# REPORTS_ROOT = '/opt/netbox/netbox/reports' + +# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +# SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' + +# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use +# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only +# database access.) Note that the user as which NetBox runs must have read and write permissions to this path. +SESSION_FILE_PATH = None + +# Time zone (default: UTC) +TIME_ZONE = 'UTC' + +# Date/time formatting. See the following link for supported formats: +# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date +DATE_FORMAT = 'N j, Y' +SHORT_DATE_FORMAT = 'Y-m-d' +TIME_FORMAT = 'g:i a' +SHORT_TIME_FORMAT = 'H:i:s' +DATETIME_FORMAT = 'N j, Y g:i a' +SHORT_DATETIME_FORMAT = 'Y-m-d H:i' +EOF diff --git a/type/__netbox/files/gunicorn.py b/type/__netbox/files/gunicorn.py new file mode 100644 index 0000000..c582677 --- /dev/null +++ b/type/__netbox/files/gunicorn.py @@ -0,0 +1,16 @@ +# The IP address (typically localhost) and port that the Netbox WSGI process should listen on +bind = '127.0.0.1:8001' + +# Number of gunicorn workers to spawn. This should typically be 2n+1, where +# n is the number of CPU cores present. +workers = 3 + +# Number of threads per worker process +threads = 3 + +# Timeout (in seconds) for a request to complete +timeout = 120 + +# The maximum number of requests a worker can handle before being respawned +max_requests = 5000 +max_requests_jitter = 500 diff --git a/type/__netbox/files/netbox-rq.service b/type/__netbox/files/netbox-rq.service new file mode 100644 index 0000000..77d7091 --- /dev/null +++ b/type/__netbox/files/netbox-rq.service @@ -0,0 +1,21 @@ +[Unit] +Description=NetBox Request Queue Worker +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service new file mode 100644 index 0000000..18eb045 --- /dev/null +++ b/type/__netbox/files/netbox.service @@ -0,0 +1,22 @@ +[Unit] +Description=NetBox WSGI Service +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +PIDFile=/var/tmp/netbox.pid +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote new file mode 100755 index 0000000..a52b110 --- /dev/null +++ b/type/__netbox/gencode-remote @@ -0,0 +1,58 @@ +#!/bin/sh + +VERSION=$(cat "$__object/parameter/version") + +src="netbox-$VERSION" +archive="v$VERSION.tar.gz" +url="https://github.com/netbox-community/netbox/archive/$archive" +install_dir=/opt/netbox/netbox + +cat << EOF +set -e + +# Ensure that coreutils is installed. +if [ ! -x \$(which mktemp) ]; then + echo "mktemp is not available on the remote host." >&2 + exit 1 +fi + +# Create temporary working directory. +tmpdir=\$(mktemp -d) +cd "\$tmpdir" + +# Download and extract sources. +curl -L '$url' > '$archive' +tar xf '$archive' + +# Save cdist-upload configuration file. +mkdir -p "\$tmpdir" +cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" + +# Deploy sources and restore configuration. +rm -r '$install_dir' +cp -r '$src/netbox' '$install_dir' +cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' + +# Setup & enter python virtualenv. +virtualenv /opt/netbox/venv + +# Install python dependencies. +/opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" + +# Set final permissions. +chown -R netbox /opt/netbox + +# Run database migrations. +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate + +# Generate static assets. +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collectstatic --no-input + +# Remove temporary working directory. +cd / +rm -r "\$tmpdir" + +# Restart services. +service netbox restart +service netbox-rq restart +EOF diff --git a/type/__netbox/manifest b/type/__netbox/manifest new file mode 100644 index 0000000..64a0a55 --- /dev/null +++ b/type/__netbox/manifest @@ -0,0 +1,47 @@ +#!/bin/sh + +os=$(cat "$__global/explorer/os") + +case "$os" in + debian|ubuntu) + # Install netbox dependencies. + for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv; do + __package $pkg + done + + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +export DATABASE_NAME=$(cat "$__object/parameter/database") +export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") +export ALLOWED_HOST=$(cat "$__object/parameter/host") +export SECRET_KEY=$(cat "$__object/parameter/secret-key") + +# Create system user used to run netbox. +__user netbox --system --home /opt/netbox --create-home + +# Generate and upload netbox configuration. +mkdir -p "$__object/files" +"$__type/files/configuration.py.sh" > "$__object/files/configuration.py" + +require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents +require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ + --source "$__object/files/configuration.py" + +# Upload systemd units and gunicorn configuration. +for unit in netbox netbox-rq; do + __file /etc/systemd/system/$unit.service \ + --mode 644 --source "$__type/files/$unit.service" \ + --onchange 'systemctl daemon-reload' +done + +# Python worker configuration. +require="__user/netbox" __file /opt/netbox/gunicorn.py \ + --mode 644 --source "$__type/files/gunicorn.py" diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required new file mode 100644 index 0000000..388048a --- /dev/null +++ b/type/__netbox/parameter/required @@ -0,0 +1,5 @@ +version +database +database-password +secret-key +host diff --git a/type/__netbox/singleton b/type/__netbox/singleton new file mode 100644 index 0000000..e69de29 From ab2c826cc3347cc3ff2842480a20dce185cc89d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jul 2020 07:46:45 +0200 Subject: [PATCH 16/98] __netbox: add minimal LDAP support --- type/__netbox/files/ldap_config.py.sh | 27 +++++++++++++++++++++++++ type/__netbox/gencode-remote | 8 ++++++++ type/__netbox/manifest | 29 +++++++++++++++++++++++++++ type/__netbox/parameter/optional | 4 ++++ 4 files changed, 68 insertions(+) create mode 100755 type/__netbox/files/ldap_config.py.sh create mode 100644 type/__netbox/parameter/optional diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh new file mode 100755 index 0000000..88af4b9 --- /dev/null +++ b/type/__netbox/files/ldap_config.py.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +cat << EOF +############################## +# LDAP-backed authentication # +############################## + +import ldap +from django_auth_ldap.config import LDAPSearch + +# Server URI +AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" + +# Set the DN and password for the NetBox service account. +AUTH_LDAP_BIND_DN = "$LDAP_BIND_DN" +AUTH_LDAP_BIND_PASSWORD = "$LDAP_BIND_PASSWORD" + +# If a user's DN is producible from their username, we don't need to search. +AUTH_LDAP_USER_DN_TEMPLATE = "$LDAP_USER_DN_TEMPLATE" + +# You can map user attributes to Django attributes as so. +AUTH_LDAP_USER_ATTR_MAP = { + "first_name": "givenName", + "last_name": "sn", + "email": "mail" +} +EOF diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index a52b110..e3157a9 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -27,18 +27,26 @@ tar xf '$archive' # Save cdist-upload configuration file. mkdir -p "\$tmpdir" cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" +cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' +cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. virtualenv /opt/netbox/venv # Install python dependencies. /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" +EOF +if [ -f "$__object/parameter/ldap-server" ]; then + echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" +fi + +cat << EOF # Set final permissions. chown -R netbox /opt/netbox diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 64a0a55..2976686 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -10,6 +10,11 @@ case "$os" in __package $pkg done + if [ -f "$__object/parameter/ldap-server" ]; then + for pkg in libldap2-dev libsasl2-dev libssl-dev; do + __package $pkg + done + fi ;; *) printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 @@ -23,18 +28,42 @@ export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") export ALLOWED_HOST=$(cat "$__object/parameter/host") export SECRET_KEY=$(cat "$__object/parameter/secret-key") +if [ -f "$__object/parameter/ldap-server" ]; then + export LDAP_SERVER=$(cat "$__object/parameter/ldap-server") +fi + +if [ -f "$__object/parameter/ldap-bind-dn" ]; then + export LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") +fi + +if [ -f "$__object/parameter/ldap-bind-password" ]; then + export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") +fi + +if [ -f "$__object/parameter/ldap-user-dn-template" ]; then + export LDAP_USER_DN_TEMPLATE=$(cat "$__object/parameter/ldap-user-dn-template") +fi + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home # Generate and upload netbox configuration. mkdir -p "$__object/files" "$__type/files/configuration.py.sh" > "$__object/files/configuration.py" +"$__type/files/ldap_config.py.sh" > "$__object/files/ldap_config.py" require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents require="__directory/opt/netbox/netbox/cdist " __file \ /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ --source "$__object/files/configuration.py" +if [ -f "$__object/parameter/ldap-server" ]; then + require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + --source "$__object/files/ldap_config.py" +fi + + # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do __file /etc/systemd/system/$unit.service \ diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional new file mode 100644 index 0000000..c946d73 --- /dev/null +++ b/type/__netbox/parameter/optional @@ -0,0 +1,4 @@ +ldap-server +ldap-bind-dn +ldap-bind-password +ldap-user-dn-template From ef7a214dae108351e1e2ae66a51966a639ee90d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jul 2020 08:30:09 +0200 Subject: [PATCH 17/98] __netbox: support basic LDAP group mapping --- type/__netbox/files/ldap_config.py.sh | 40 +++++++++++++++++++++++++-- type/__netbox/manifest | 16 +++++++++-- type/__netbox/parameter/optional | 5 +++- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 88af4b9..901d6f5 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -6,7 +6,7 @@ cat << EOF ############################## import ldap -from django_auth_ldap.config import LDAPSearch +from django_auth_ldap.config import LDAPSearch, PosixGroupType # Server URI AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" @@ -15,8 +15,10 @@ AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" AUTH_LDAP_BIND_DN = "$LDAP_BIND_DN" AUTH_LDAP_BIND_PASSWORD = "$LDAP_BIND_PASSWORD" -# If a user's DN is producible from their username, we don't need to search. -AUTH_LDAP_USER_DN_TEMPLATE = "$LDAP_USER_DN_TEMPLATE" +# Search for user entry. +AUTH_LDAP_USER_SEARCH = LDAPSearch("$LDAP_USER_BASE", + ldap.SCOPE_SUBTREE, + "(uid=%(user)s)") # You can map user attributes to Django attributes as so. AUTH_LDAP_USER_ATTR_MAP = { @@ -25,3 +27,35 @@ AUTH_LDAP_USER_ATTR_MAP = { "email": "mail" } EOF + +if [ "$LDAP_GROUP_BASE" != "" ]; then + cat << EOF + +# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group +# hierarchy. +AUTH_LDAP_GROUP_SEARCH = LDAPSearch("$LDAP_GROUP_BASE", ldap.SCOPE_SUBTREE, + "(objectClass=posixGroup)") +AUTH_LDAP_GROUP_TYPE = PosixGroupType() + +# Mirror LDAP group assignments. +AUTH_LDAP_MIRROR_GROUPS = True +EOF + + if [ "$LDAP_REQUIRE_GROUP" != "" ]; then + cat << EOF + +# Define a group required to login. +AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" +EOF + fi + + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + cat << EOF + +# Define special user types using groups. Exercise great caution when assigning superuser status. +AUTH_LDAP_USER_FLAGS_BY_GROUP = { + "is_superuser": "$LDAP_SUPERUSER_GROUP", +} +EOF + fi +fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2976686..32c85e3 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -40,8 +40,20 @@ if [ -f "$__object/parameter/ldap-bind-password" ]; then export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") fi -if [ -f "$__object/parameter/ldap-user-dn-template" ]; then - export LDAP_USER_DN_TEMPLATE=$(cat "$__object/parameter/ldap-user-dn-template") +if [ -f "$__object/parameter/ldap-user-base" ]; then + export LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") +fi + +if [ -f "$__object/parameter/ldap-group-base" ]; then + export LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") +fi + +if [ -f "$__object/parameter/ldap-require-group" ]; then + export LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") +fi + +if [ -f "$__object/parameter/ldap-superuser-group" ]; then + export LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") fi # Create system user used to run netbox. diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index c946d73..290c98b 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,4 +1,7 @@ ldap-server ldap-bind-dn ldap-bind-password -ldap-user-dn-template +ldap-user-base +ldap-group-base +ldap-require-group +ldap-superuser-group From e2f01bbb3b3c297cd17400437eb8bd13f4697a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 22 Jul 2020 17:14:06 +0200 Subject: [PATCH 18/98] __matrix_riot: rename to __matrix_element --- .../files/config.json.sh | 2 +- .../gencode-remote | 2 +- .../man.rst | 20 +++++++++---------- .../manifest | 0 .../parameter/boolean | 0 type/__matrix_element/parameter/default/brand | 1 + .../parameter/default/cookie_policy_url | 0 .../parameter/default/default_country_code | 0 .../parameter/default/default_server_name | 0 .../parameter/default/default_server_url | 0 .../parameter/default/owner | 0 .../parameter/default/privacy_policy_url | 1 + .../parameter/default/room_directory_servers | 0 .../parameter/optional | 0 .../parameter/optional_multiple | 0 .../parameter/required | 0 type/__matrix_riot/parameter/default/brand | 1 - .../parameter/default/privacy_policy_url | 1 - 18 files changed, 14 insertions(+), 14 deletions(-) rename type/{__matrix_riot => __matrix_element}/files/config.json.sh (96%) rename type/{__matrix_riot => __matrix_element}/gencode-remote (97%) rename type/{__matrix_riot => __matrix_element}/man.rst (73%) rename type/{__matrix_riot => __matrix_element}/manifest (100%) rename type/{__matrix_riot => __matrix_element}/parameter/boolean (100%) create mode 100644 type/__matrix_element/parameter/default/brand rename type/{__matrix_riot => __matrix_element}/parameter/default/cookie_policy_url (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_country_code (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_server_name (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_server_url (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/owner (100%) create mode 100644 type/__matrix_element/parameter/default/privacy_policy_url rename type/{__matrix_riot => __matrix_element}/parameter/default/room_directory_servers (100%) rename type/{__matrix_riot => __matrix_element}/parameter/optional (100%) rename type/{__matrix_riot => __matrix_element}/parameter/optional_multiple (100%) rename type/{__matrix_riot => __matrix_element}/parameter/required (100%) delete mode 100644 type/__matrix_riot/parameter/default/brand delete mode 100644 type/__matrix_riot/parameter/default/privacy_policy_url diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_element/files/config.json.sh similarity index 96% rename from type/__matrix_riot/files/config.json.sh rename to type/__matrix_element/files/config.json.sh index 6cd1648..9791f38 100755 --- a/type/__matrix_riot/files/config.json.sh +++ b/type/__matrix_element/files/config.json.sh @@ -39,7 +39,7 @@ EOF fi cat << EOF - "welcomeBackgroundUrl": "themes/riot/img/backgrounds/valley.jpg" + "welcomeBackgroundUrl": "themes/element/img/backgrounds/lake.jpg" EOF echo '},' } diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_element/gencode-remote similarity index 97% rename from type/__matrix_riot/gencode-remote rename to type/__matrix_element/gencode-remote index dbc886c..e643976 100755 --- a/type/__matrix_riot/gencode-remote +++ b/type/__matrix_element/gencode-remote @@ -46,7 +46,7 @@ cd \$tmpdir curl -L '$url' > $archive tar xf $archive -# Backup files deployed by __matrix_riot. +# Backup files deployed by __matrix_element. mkdir -p \$custom_files_dir for file in $INSTALL_DIR/cdist/*; do cp "\$file" "\$custom_files_dir" diff --git a/type/__matrix_riot/man.rst b/type/__matrix_element/man.rst similarity index 73% rename from type/__matrix_riot/man.rst rename to type/__matrix_element/man.rst index 2fe8eae..c3e0d5a 100644 --- a/type/__matrix_riot/man.rst +++ b/type/__matrix_element/man.rst @@ -1,23 +1,23 @@ -cdist-type__matrix_riot(7) +cdist-type__matrix_element(7) ====================== NAME ---- -cdist-type__matrix_riot - Install and configure Riot, a web Matrix client. +cdist-type__matrix_element - Install and configure Element, a web Matrix client. DESCRIPTION ----------- -This type install and configure the Riot web client. +This type install and configure the Element web client. REQUIRED PARAMETERS ------------------- install_dir - Root directory of Riot's static files. + Root directory of Element's static files. version - Release of Riot to install. + Release of Element to install. OPTIONAL PARAMETERS ------------------- @@ -31,20 +31,20 @@ owner Owner of the deployed files, passed to `chown`. Defaults to 'root'. brand - Web UI branding, defaults to 'Riot'. + Web UI branding, defaults to 'Element'. default_country_code ISO 3166 alpha2 country code to use when showing country selectors, such as phone number inputs. Defaults to GB. privacy_policy_url - Defaults to 'https://riot.im/privacy'. + Defaults to 'https://element.io/privacy'. cookie_policy_url - Defaults to 'https://matrix.org/docs/guides/riot_im_cookie_policy'. + Defaults to 'https://matrix.org/docs/guides/element_im_cookie_policy'. jitsi_domain - Domain name of preferred Jitsi instance (default is jitsi.riot.im). This is + Domain name of preferred Jitsi instance (default is jitsi.element.im). This is used whenever a user clicks on the voice/video call buttons. homepage @@ -66,7 +66,7 @@ EXAMPLES .. code-block:: sh - __matrix_riot my-riot --install_dir /var/www/riot-web --version 1.5.6 + __matrix_element my-element --install_dir /var/www/element-web --version 1.5.6 SEE ALSO diff --git a/type/__matrix_riot/manifest b/type/__matrix_element/manifest similarity index 100% rename from type/__matrix_riot/manifest rename to type/__matrix_element/manifest diff --git a/type/__matrix_riot/parameter/boolean b/type/__matrix_element/parameter/boolean similarity index 100% rename from type/__matrix_riot/parameter/boolean rename to type/__matrix_element/parameter/boolean diff --git a/type/__matrix_element/parameter/default/brand b/type/__matrix_element/parameter/default/brand new file mode 100644 index 0000000..907f907 --- /dev/null +++ b/type/__matrix_element/parameter/default/brand @@ -0,0 +1 @@ +Element diff --git a/type/__matrix_riot/parameter/default/cookie_policy_url b/type/__matrix_element/parameter/default/cookie_policy_url similarity index 100% rename from type/__matrix_riot/parameter/default/cookie_policy_url rename to type/__matrix_element/parameter/default/cookie_policy_url diff --git a/type/__matrix_riot/parameter/default/default_country_code b/type/__matrix_element/parameter/default/default_country_code similarity index 100% rename from type/__matrix_riot/parameter/default/default_country_code rename to type/__matrix_element/parameter/default/default_country_code diff --git a/type/__matrix_riot/parameter/default/default_server_name b/type/__matrix_element/parameter/default/default_server_name similarity index 100% rename from type/__matrix_riot/parameter/default/default_server_name rename to type/__matrix_element/parameter/default/default_server_name diff --git a/type/__matrix_riot/parameter/default/default_server_url b/type/__matrix_element/parameter/default/default_server_url similarity index 100% rename from type/__matrix_riot/parameter/default/default_server_url rename to type/__matrix_element/parameter/default/default_server_url diff --git a/type/__matrix_riot/parameter/default/owner b/type/__matrix_element/parameter/default/owner similarity index 100% rename from type/__matrix_riot/parameter/default/owner rename to type/__matrix_element/parameter/default/owner diff --git a/type/__matrix_element/parameter/default/privacy_policy_url b/type/__matrix_element/parameter/default/privacy_policy_url new file mode 100644 index 0000000..37fa4bc --- /dev/null +++ b/type/__matrix_element/parameter/default/privacy_policy_url @@ -0,0 +1 @@ +https://element.io/privacy diff --git a/type/__matrix_riot/parameter/default/room_directory_servers b/type/__matrix_element/parameter/default/room_directory_servers similarity index 100% rename from type/__matrix_riot/parameter/default/room_directory_servers rename to type/__matrix_element/parameter/default/room_directory_servers diff --git a/type/__matrix_riot/parameter/optional b/type/__matrix_element/parameter/optional similarity index 100% rename from type/__matrix_riot/parameter/optional rename to type/__matrix_element/parameter/optional diff --git a/type/__matrix_riot/parameter/optional_multiple b/type/__matrix_element/parameter/optional_multiple similarity index 100% rename from type/__matrix_riot/parameter/optional_multiple rename to type/__matrix_element/parameter/optional_multiple diff --git a/type/__matrix_riot/parameter/required b/type/__matrix_element/parameter/required similarity index 100% rename from type/__matrix_riot/parameter/required rename to type/__matrix_element/parameter/required diff --git a/type/__matrix_riot/parameter/default/brand b/type/__matrix_riot/parameter/default/brand deleted file mode 100644 index e8095bb..0000000 --- a/type/__matrix_riot/parameter/default/brand +++ /dev/null @@ -1 +0,0 @@ -Riot diff --git a/type/__matrix_riot/parameter/default/privacy_policy_url b/type/__matrix_riot/parameter/default/privacy_policy_url deleted file mode 100644 index 4cdd12c..0000000 --- a/type/__matrix_riot/parameter/default/privacy_policy_url +++ /dev/null @@ -1 +0,0 @@ -https://riot.im/privacy From 5960356e7b0e540ed49af17ec05a0ba608dfaae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 10 Jul 2020 10:41:20 +0200 Subject: [PATCH 19/98] __unbound: add --local_data flag --- type/__unbound/files/unbound.conf.sh | 9 +++++++++ type/__unbound/man.rst | 4 ++++ type/__unbound/manifest | 5 +++++ type/__unbound/parameter/optional_multiple | 1 + 4 files changed, 19 insertions(+) diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index 46d2824..2f7ecff 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -18,6 +18,14 @@ generate_forward_addr() { done } +generate_local_data() { + IFS=' +' + for entry in $LOCAL_DATA; do + echo " local-data: \"$entry\"" + done +} + cat << EOF # # THIS FILE HAS BEEN GENERATED BY CDIST, DO NOT EDIT BY HAND. @@ -720,6 +728,7 @@ $(generate_access_control) # (this makes example.com, www.example.com, etc, all go to 192.0.2.3) # local-zone: "example.com" redirect # local-data: "example.com A 192.0.2.3" +$(generate_local_data) # # Shorthand to make PTR records, "IPv4 name" or "IPv6 name". # You can also add PTR records using local-data directly, but then diff --git a/type/__unbound/man.rst b/type/__unbound/man.rst index 679e601..316d011 100644 --- a/type/__unbound/man.rst +++ b/type/__unbound/man.rst @@ -34,6 +34,10 @@ access_control rc_interface Address or path to socket used for remote control (see `--enable_control`. Defaults to `127.0.0.1`). +local_data + Configure local data, which is served in reply to queries for it. Can be + specified multiple times. + BOOLEAN PARAMETERS ------------------ disable-ip4 diff --git a/type/__unbound/manifest b/type/__unbound/manifest index 346f30c..5d6b50f 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -54,6 +54,11 @@ if [ -f "$__object/parameter/rc_interface" ]; then export RC_INTERFACE fi +if [ -f "$__object/parameter/local_data" ]; then + LOCAL_DATA=$(cat "$__object/parameter/local_data") + export LOCAL_DATA +fi + # Boolean parameters: if [ -f "$__object/parameter/disable_ip4" ] && \ [ -f "$__object/parameter/disable_ip6" ]; then diff --git a/type/__unbound/parameter/optional_multiple b/type/__unbound/parameter/optional_multiple index ecacd1d..3fe7eca 100644 --- a/type/__unbound/parameter/optional_multiple +++ b/type/__unbound/parameter/optional_multiple @@ -1,2 +1,3 @@ access_control +local_data interface From 7eecc99c73f2304c5f4837dd9324a5d925256782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 27 Jul 2020 14:53:45 +0200 Subject: [PATCH 20/98] __unbound: fix typo in gencode-remote / service start --- type/__unbound/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote index d12a395..ba6d92b 100755 --- a/type/__unbound/gencode-remote +++ b/type/__unbound/gencode-remote @@ -11,6 +11,6 @@ cat << EOF if pgrep unbound; then service unbound reload else - service unbounb start + service unbound start fi EOF From f73bf49e4b510b426dce4c04d1d63e08455f1ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jul 2020 14:23:30 +0200 Subject: [PATCH 21/98] Remove manual / outdated CHANGELOG --- CHANGELOG.md | 6 ------ README.md | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index df26f7e..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,6 +0,0 @@ -# cdist-contrib changes - -* 2020-06-07: New type: __unbound_exporter (Timothée Floure) -* 2020-06-07: Extended type: wire remote control configuration for __unbond (Timothée Floure) -* 2020-06-03: New type: __unbound (Timothée Floure) -* 2020-04-28: New type: __find_exec (Ander Punnar) diff --git a/README.md b/README.md index ef4b2c3..4231de2 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ tool with community-maitained types which are either too specific to fit/be maintained in cdist itself or were not accepted in code cdist but could still be useful. -This project does not have releases and is continously updated: see -`CHANGELOG.md` for details. +This project does not have releases and is continously updated: see git history +for change log. ## Using cdist-contrib From bf5f85068d8614f97e190e5115e34f5442ba7924 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 7 Aug 2020 10:39:38 +0200 Subject: [PATCH 22/98] Add a type to create a borg repository. --- type/__borg_repo/gencode-remote | 36 ++++++++++++++++ type/__borg_repo/man.rst | 43 +++++++++++++++++++ type/__borg_repo/manifest | 14 ++++++ type/__borg_repo/parameter/boolean | 1 + type/__borg_repo/parameter/default/encryption | 1 + type/__borg_repo/parameter/optional | 1 + type/__borg_repo/parameter/required | 1 + 7 files changed, 97 insertions(+) create mode 100644 type/__borg_repo/gencode-remote create mode 100644 type/__borg_repo/man.rst create mode 100644 type/__borg_repo/manifest create mode 100644 type/__borg_repo/parameter/boolean create mode 100644 type/__borg_repo/parameter/default/encryption create mode 100644 type/__borg_repo/parameter/optional create mode 100644 type/__borg_repo/parameter/required diff --git a/type/__borg_repo/gencode-remote b/type/__borg_repo/gencode-remote new file mode 100644 index 0000000..542bf5f --- /dev/null +++ b/type/__borg_repo/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh + +passphrase= +appendonly= + +case "$(cat "${__object:?}/parameter/encryption")" in + none) + enc=none + ;; + repokey) + enc=repokey + if [ -f "${__object:?}/parameter/passphrase" ]; + then + passphrase="$(cat "${__object:?}/parameter/passphrase")" + else + echo "__borg_repo cannot use repokey encryption with no passphrase. Aborting." >&2; + exit 1; + fi + ;; + *) + echo "$enc is not a known encryption mode for __borg_repo. Aborting." >&2 + exit 1; +esac + +if [ -f "${__object:?}/parameter/append-only" ]; +then + appendonly='--append-only' +fi + +cat <<- EOF + if ! borg check --repository-only 1>&2 2>/dev/null "/${__object_id:?}"; + then + BORG_NEW_PASSPHRASE=$passphrase borg init -e ${enc:?} $appendonly /${__object_id:?} + fi +EOF + diff --git a/type/__borg_repo/man.rst b/type/__borg_repo/man.rst new file mode 100644 index 0000000..38ab0c9 --- /dev/null +++ b/type/__borg_repo/man.rst @@ -0,0 +1,43 @@ +cdist-type__borg_repo(7) +======================== + +NAME +---- +cdist-type__borg_repo - Configure a borg repository on host + + +DESCRIPTION +----------- + +Initializes a borg repository at the location specified in the +`${__object_id}`. Nothing is done if the repository already exists. + +Currently, only `none` and `repokey` are supported as encryption modes; +`repokey` requires the `passphrase` argument to be given. The default is +`none`. + +REQUIRED PARAMETERS +------------------- +encryption + The encryption to use. + +OPTIONAL PARAMETERS +------------------- +passphrase + The passphrase to encrypt the keyfile with. + +BOOLEAN PARAMETERS +------------------ +append-only + If the repository is append-only + +AUTHORS +------- +Joachim Desroches + +COPYING +------- +Copyright \(C) 2020 Joachim Desroches. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__borg_repo/manifest b/type/__borg_repo/manifest new file mode 100644 index 0000000..fe18c9c --- /dev/null +++ b/type/__borg_repo/manifest @@ -0,0 +1,14 @@ +#!/bin/sh + +os="$(cat "${__global:?}"/explorer/os)" + +case "$os" in + "alpine") + borg_package=borgbackup + ;; + *) + echo "__borg_repo is not yet implemented for os $os. Aborting." >&2; + exit 1; +esac + +__package "$borg_package" diff --git a/type/__borg_repo/parameter/boolean b/type/__borg_repo/parameter/boolean new file mode 100644 index 0000000..f8ee7c6 --- /dev/null +++ b/type/__borg_repo/parameter/boolean @@ -0,0 +1 @@ +append-only diff --git a/type/__borg_repo/parameter/default/encryption b/type/__borg_repo/parameter/default/encryption new file mode 100644 index 0000000..621e94f --- /dev/null +++ b/type/__borg_repo/parameter/default/encryption @@ -0,0 +1 @@ +none diff --git a/type/__borg_repo/parameter/optional b/type/__borg_repo/parameter/optional new file mode 100644 index 0000000..f63b25b --- /dev/null +++ b/type/__borg_repo/parameter/optional @@ -0,0 +1 @@ +passphrase diff --git a/type/__borg_repo/parameter/required b/type/__borg_repo/parameter/required new file mode 100644 index 0000000..a5465f8 --- /dev/null +++ b/type/__borg_repo/parameter/required @@ -0,0 +1 @@ +encryption From 47584d6da64b86a266d950a61879ccb82268d2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 08:58:29 +0200 Subject: [PATCH 23/98] __netbox: use __systemd_unit, do not recreate existing directory --- type/__netbox/gencode-remote | 1 - type/__netbox/manifest | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index e3157a9..122df96 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -25,7 +25,6 @@ curl -L '$url' > '$archive' tar xf '$archive' # Save cdist-upload configuration file. -mkdir -p "\$tmpdir" cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 32c85e3..2503bb4 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -78,9 +78,9 @@ fi # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do - __file /etc/systemd/system/$unit.service \ - --mode 644 --source "$__type/files/$unit.service" \ - --onchange 'systemctl daemon-reload' + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled done # Python worker configuration. From 240b8f1615d292ddca081304ce930792e330fc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 09:01:05 +0200 Subject: [PATCH 24/98] __netbox: make shellcheck happy --- type/__netbox/manifest | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2503bb4..fc9192c 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -23,37 +23,48 @@ case "$os" in ;; esac -export DATABASE_NAME=$(cat "$__object/parameter/database") -export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") -export ALLOWED_HOST=$(cat "$__object/parameter/host") -export SECRET_KEY=$(cat "$__object/parameter/secret-key") +DATABASE_NAME=$(cat "$__object/parameter/database") +export DATABASE_NAME +DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") +export DATABASE_PASSWORD +ALLOWED_HOST=$(cat "$__object/parameter/host") +export ALLOWED_HOST +SECRET_KEY=$(cat "$__object/parameter/secret-key") +export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then - export LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + export LDAP_SERVER fi if [ -f "$__object/parameter/ldap-bind-dn" ]; then - export LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + export LDAP_BIND_DN fi if [ -f "$__object/parameter/ldap-bind-password" ]; then - export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + export LDAP_BIND_PASSWORD fi if [ -f "$__object/parameter/ldap-user-base" ]; then - export LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + export LDAP_USER_BASE fi if [ -f "$__object/parameter/ldap-group-base" ]; then - export LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + export LDAP_GROUP_BASE fi if [ -f "$__object/parameter/ldap-require-group" ]; then - export LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + export LDAP_REQUIRE_GROUP fi if [ -f "$__object/parameter/ldap-superuser-group" ]; then - export LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + export LDAP_SUPERUSER_GROUP fi # Create system user used to run netbox. From dfd9f108b1ba11b91c3e198fde60f86e5bf34006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 09:15:18 +0200 Subject: [PATCH 25/98] __netbox: add minimal manpage --- type/__netbox/man.rst | 94 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 type/__netbox/man.rst diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst new file mode 100644 index 0000000..25aeb26 --- /dev/null +++ b/type/__netbox/man.rst @@ -0,0 +1,94 @@ +cdist-type__netbox(7) +===================== + +NAME +---- +cdist-type__netbox - Install and configure NetBox + + +DESCRIPTION +----------- +This (singleton) type installs and configures a NetBox instance, a web +application to help manage and document computer networks + + +REQUIRED PARAMETERS +------------------- +version + NetBox version to be installed. + +database + PostgreSQL database name. + +database-password + PostgreSQL database password. + +secret-key + Random secret key of at least 50 alphanumeric characters. This key must be + unique to this installation and must not be shared outside the local + system. + +host + Hostname (domain or IP address) on which the application is served. + +OPTIONAL PARAMETERS +------------------- +ldap-server + LDAP server URI. Enables LDAP-backed authentication if specified. + +ldap-bind-dn + DN for the NetBox service account. Required for LDAP authentication. + +ldap-bind-password + Password for the NetBox service account. Required for LDAP authentication. + +ldap-user-base + Base used for searching user entries. Required for LDAP authentication. + +ldap-group-base + Base used for searching group entries. + +ldap-require-group + Group required to login. + +ldap-superuser-group + Make members of this groups superusers. + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __netbox --version 2.8.7 --database netbox \ + --database-password "secretsecretsecret" \ + --secret-key "secretsecretsecret" \ + --host "${__target_host:?}" \ + --ldap-server "ldaps://ldap.domain.tld" \ + --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ + --ldap-bind-password "secretsecretsecret" \ + --ldap-user-base "ou=users,dc=domain,dc=tld" \ + --ldap-group-base "ou=groups,dc=domain,dc=tld" \ + --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ + --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + + +SEE ALSO +-------- +- `NetBox documentation `_ + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. From efab6ac23ec52e8d5dad8cda9df9b050d66a23f1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 16 Aug 2020 15:18:47 +0200 Subject: [PATCH 26/98] __netbox: add version memorisation This saves the installed version and checks if a new version must be installed. If no update required, it updates the configuration files if required. --- type/__netbox/explorer/version | 5 ++++ type/__netbox/gencode-remote | 42 ++++++++++++++++++++++++++++------ type/__netbox/man.rst | 10 ++++++++ type/__netbox/manifest | 0 4 files changed, 50 insertions(+), 7 deletions(-) create mode 100755 type/__netbox/explorer/version mode change 100644 => 100755 type/__netbox/manifest diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version new file mode 100755 index 0000000..a9ed590 --- /dev/null +++ b/type/__netbox/explorer/version @@ -0,0 +1,5 @@ +#!/bin/sh + +# output version if exist +version_path="/opt/netbox/netbox/cdist/version" +if [ -f "$version_path" ]; then cat "$version_path"; fi diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 122df96..48463f7 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -1,5 +1,8 @@ #!/bin/sh +echo "set -e" + +old_version="$(cat "$__object/explorer/version")" VERSION=$(cat "$__object/parameter/version") src="netbox-$VERSION" @@ -7,9 +10,8 @@ archive="v$VERSION.tar.gz" url="https://github.com/netbox-community/netbox/archive/$archive" install_dir=/opt/netbox/netbox -cat << EOF -set -e - +if [ "$VERSION" != "$old_version" ]; then + cat << EOF # Ensure that coreutils is installed. if [ ! -x \$(which mktemp) ]; then echo "mktemp is not available on the remote host." >&2 @@ -41,11 +43,11 @@ virtualenv /opt/netbox/venv /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" EOF -if [ -f "$__object/parameter/ldap-server" ]; then - echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" -fi + if [ -f "$__object/parameter/ldap-server" ]; then + echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + fi -cat << EOF + cat << EOF # Set final permissions. chown -R netbox /opt/netbox @@ -59,7 +61,33 @@ sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collect cd / rm -r "\$tmpdir" +# Save version after successful installation +printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' +EOF + + # meta + printf "%s %s\n" "installed" "$VERSION" >> "$__messages_out" + changes=yes + +# check if configuration changed +elif grep -q "^__file/opt/netbox/" "$__messages_in"; then + cat <> "$__messages_out" + changes=yes +fi + + +# check for changes +if [ "$changes" = "yes" ]; then + cat < Date: Sun, 16 Aug 2020 15:31:38 +0200 Subject: [PATCH 27/98] __netbox: add manpage explanation of the type --- type/__netbox/man.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 84f1681..e41b099 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -9,7 +9,11 @@ cdist-type__netbox - Install and configure NetBox DESCRIPTION ----------- This (singleton) type installs and configures a NetBox instance, a web -application to help manage and document computer networks +application to help manage and document computer networks. + +It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. +Netbox will be run via `gnuicorn` as WSGI service. It setup systemd unit files +for the services `netbox` and `netbox-rq`. REQUIRED PARAMETERS From 0281b2c8046832d067dfa3be0da8702ef590faf4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 20 Aug 2020 17:53:25 +0200 Subject: [PATCH 28/98] __netbox: no ldap config if no ldap parameters The ldap config file will be empty (only comments) if no parameter name set matches `^ldap-*`. It still generate a ldap config if one or more ldap parameters are set, but not all basic parameters. --- type/__netbox/files/ldap_config.py.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 901d6f5..3e7aaa6 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,5 +1,19 @@ #!/bin/sh +# no configuration if there are no ldap parameters +if ls -1qA "$__object/parameter"/ldap-* | grep -q .; then + # skip + cat << EOF +############################## +# LDAP-backed authentication # +############################## + +# no options set +EOF + exit 0 +fi + + cat << EOF ############################## # LDAP-backed authentication # From a05cb214e17303068649aab3749099e4a06ceef1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 20 Aug 2020 19:37:09 +0200 Subject: [PATCH 29/98] __netbox: fix shellcheck by changing parameter checks The working `ls | grep .` like solution was dropped based on the shellcheck warning. Rather a solution with `find(1)` is used. As alternative, it could be simply checked if the glob succeeds or not by test all given arguments if there are existing files. It was dropped in question if it's really better than the `find(1)` solution. --- type/__netbox/files/ldap_config.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 3e7aaa6..bc494fb 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if ls -1qA "$__object/parameter"/ldap-* | grep -q .; then +if [ "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then # skip cat << EOF ############################## From e6e6dbcd0029639539ce7fe2d8a6fb7d753ad727 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 21 Aug 2020 23:50:24 +0200 Subject: [PATCH 30/98] __netbox: added some netbox configuration options Those who might be useful are added .. --- type/__netbox/files/configuration.py.sh | 64 ++++++++++++------ type/__netbox/man.rst | 65 ++++++++++++++++++- type/__netbox/manifest | 65 +++++++++++++++++++ type/__netbox/parameter/boolean | 4 ++ type/__netbox/parameter/default/basepath | 1 + .../parameter/default/redis-dbid-offset | 1 + type/__netbox/parameter/default/redis-host | 1 + .../__netbox/parameter/default/redis-password | 1 + type/__netbox/parameter/default/redis-port | 1 + .../parameter/default/smtp-from-email | 1 + type/__netbox/parameter/default/smtp-host | 1 + type/__netbox/parameter/default/smtp-password | 1 + type/__netbox/parameter/default/smtp-port | 1 + type/__netbox/parameter/default/smtp-user | 1 + type/__netbox/parameter/optional | 12 ++++ 15 files changed, 200 insertions(+), 20 deletions(-) create mode 100644 type/__netbox/parameter/boolean create mode 100644 type/__netbox/parameter/default/basepath create mode 100644 type/__netbox/parameter/default/redis-dbid-offset create mode 100644 type/__netbox/parameter/default/redis-host create mode 100644 type/__netbox/parameter/default/redis-password create mode 100644 type/__netbox/parameter/default/redis-port create mode 100644 type/__netbox/parameter/default/smtp-from-email create mode 100644 type/__netbox/parameter/default/smtp-host create mode 100644 type/__netbox/parameter/default/smtp-password create mode 100644 type/__netbox/parameter/default/smtp-port create mode 100644 type/__netbox/parameter/default/smtp-user diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index b7e1597..e621845 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -29,26 +29,26 @@ DATABASE = { # to use two separate database IDs. REDIS = { 'tasks': { - 'HOST': 'localhost', - 'PORT': 6379, + 'HOST': '$REDIS_HOST', + 'PORT': $REDIS_PORT, # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], # 'SENTINEL_SERVICE': 'netbox', - 'PASSWORD': '', - 'DATABASE': 0, + 'PASSWORD': '$REDIS_PASSWORD', + 'DATABASE': $((REDIS_DBID_OFFSET + 0)), 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'SSL': $REDIS_SSL, }, 'caching': { - 'HOST': 'localhost', - 'PORT': 6379, + 'HOST': '$REDIS_HOST', + 'PORT': $REDIS_PORT, # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], # 'SENTINEL_SERVICE': 'netbox', - 'PASSWORD': '', - 'DATABASE': 1, + 'PASSWORD': '$REDIS_PASSWORD', + 'DATABASE': $((REDIS_DBID_OFFSET + 1)), 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'SSL': $REDIS_SSL, } } @@ -86,7 +86,7 @@ BANNER_LOGIN = '' # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: # BASE_PATH = 'netbox/' -BASE_PATH = '' +BASE_PATH = '$BASEPATH' # Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) CACHE_TIMEOUT = 900 @@ -112,14 +112,14 @@ DEBUG = False # Email settings EMAIL = { - 'SERVER': 'localhost', - 'PORT': 25, - 'USERNAME': '', - 'PASSWORD': '', - 'USE_SSL': False, - 'USE_TLS': False, + 'SERVER': '$SMTP_HOST', + 'PORT': $SMTP_PORT, + 'USERNAME': '$SMTP_USER', + 'PASSWORD': '$SMTP_PASSWORD', + 'USE_SSL': $SMTP_USE_SSL, + 'USE_TLS': $SMTP_USE_TLS, 'TIMEOUT': 10, # seconds - 'FROM_EMAIL': '', + 'FROM_EMAIL': '$SMTP_FROM_EMAIL', } # Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table @@ -134,12 +134,38 @@ EXEMPT_VIEW_PERMISSIONS = [ # 'ipam.prefix', ] +EOF + +if [ "$HTTP_PROXY" != "" ] || [ "$HTTPS_PROXY" != "" ]; then + cat << EOF +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +HTTP_PROXIES = { +EOF + if [ "$HTTP_PROXY" != "" ]; then + cat << EOF + 'http': '$HTTP_PROXY', +EOF + fi + if [ "$HTTPS_PROXY" != "" ]; then + cat << EOF + 'https': '$HTTPS_PROXY', +EOF + fi + cat << EOF +} +EOF + +else + cat << EOF # HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). # HTTP_PROXIES = { # 'http': 'http://10.10.1.10:3128', # 'https': 'http://10.10.1.10:1080', # } +EOF +fi +cat << EOF # IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing # NetBox from an internal IP. INTERNAL_IPS = ('127.0.0.1', '::1') @@ -150,7 +176,7 @@ LOGGING = {} # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users # are permitted to access most data in NetBox (excluding secrets) but not make any changes. -LOGIN_REQUIRED = False +LOGIN_REQUIRED = $LOGIN_REQUIRED # The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to # re-authenticate. (Default: 1209600 [14 days]) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index e41b099..3e607b5 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -58,9 +58,62 @@ ldap-require-group ldap-superuser-group Make members of this groups superusers. +redis-host + Redis database hostname. Defaults to ``localhost``. + +redis-port + Redis database port. Defaults to ``6379``. + +redis-password + Redis password. Defaults to empty password. + +redis-dbid-offset + Offset to set the redis database id's. The `tasks` database id is `offset + 0` + and `caching` is `offset + 1`. The offset defaults to ``0``. + +smtp-host + Host of the SMTP email server. Defaults to ``localhost``. + +smtp-port + Port of the SMTP email server. Defaults to ``25``. + +smtp-user + Username to access the SMTP email server. Defaults to empty. + +smtp-password + Password to access the SMTP email server. Defaults to empty. + +smtp-from-email + Email from which NetBox will be sent of. Defaults to empty. + +basepath + Base URL path if accessing netbox within a directory instead of directly the + webroot ``/``. For example, if installed at https://example.com/netbox/, set + the value ``netbox/``. + +http-proxy +https-proxy + Proxy which will be used with any HTTP request like webhooks. + BOOLEAN PARAMETERS ------------------ -None. +redis-ssl + Enables a secure TLS/SSL connection to the redis database. By default, ssl + is disabled. + +smtp-use-tls + Uses TLS to connect to the SMTP email server. `See documentation + ` + SEE ALSO -------- - `NetBox documentation `_ diff --git a/type/__netbox/manifest b/type/__netbox/manifest index fc9192c..1f7f8ed 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -23,6 +23,7 @@ case "$os" in ;; esac + DATABASE_NAME=$(cat "$__object/parameter/database") export DATABASE_NAME DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") @@ -67,6 +68,70 @@ if [ -f "$__object/parameter/ldap-superuser-group" ]; then export LDAP_SUPERUSER_GROUP fi +# have default values +REDIS_HOST="$(cat "$__object/parameter/redis-host")" +export REDIS_HOST +REDIS_PORT="$(cat "$__object/parameter/redis-port")" +export REDIS_PORT +REDIS_PASSWORD="$(cat "$__object/parameter/redis-password")" +export REDIS_PASSWORD +REDIS_DBID_OFFSET="$(cat "$__object/parameter/redis-dbid-offset")" +export REDIS_DBID_OFFSET +if [ -f "$__object/parameter/redis-ssl" ]; then + REDIS_SSL="True" +else + REDIS_SSL="False" +fi +export REDIS_SSL + +SMTP_HOST="$(cat "$__object/parameter/smtp-host")" +export SMTP_HOST +SMTP_PORT="$(cat "$__object/parameter/smtp-port")" +export SMTP_PORT +SMTP_USER="$(cat "$__object/parameter/smtp-user")" +export SMTP_USER +SMTP_PASSWORD="$(cat "$__object/parameter/smtp-password")" +export SMTP_PASSWORD +SMTP_FROM_EMAIL="$(cat "$__object/parameter/smtp-from-email")" +export SMTP_FROM_EMAIL + +if [ -f "$__object/parameter/smtp-use-ssl" ]; then + SMTP_USE_SSL="True" +else + SMTP_USE_SSL="False" +fi +export SMTP_USE_SSL +if [ -f "$__object/parameter/smtp-use-tls" ]; then + if [ "$SMTP_USE_SSL" = "True" ]; then + echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" + exit 2 + fi + SMTP_USE_TLS="True" +else + SMTP_USE_TLS="False" +fi +export SMTP_USE_TLS + +BASEPATH="$(cat "$__object/parameter/basepath")" +export BASEPATH + +if [ -f "$__object/parameter/http-proxy" ]; then + HTTP_PROXY=$(cat "$__object/parameter/http-proxy") + export HTTP_PROXY +fi +if [ -f "$__object/parameter/https-proxy" ]; then + HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") + export HTTPS_PROXY +fi + +if [ -f "$__object/parameter/login-required" ]; then + LOGIN_REQUIRED="True" +else + LOGIN_REQUIRED="False" +fi +export LOGIN_REQUIRED + + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/boolean b/type/__netbox/parameter/boolean new file mode 100644 index 0000000..a94808c --- /dev/null +++ b/type/__netbox/parameter/boolean @@ -0,0 +1,4 @@ +redis-ssl +smtp-use-ssl +smtp-use-tls +login-required diff --git a/type/__netbox/parameter/default/basepath b/type/__netbox/parameter/default/basepath new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/basepath @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/redis-dbid-offset b/type/__netbox/parameter/default/redis-dbid-offset new file mode 100644 index 0000000..573541a --- /dev/null +++ b/type/__netbox/parameter/default/redis-dbid-offset @@ -0,0 +1 @@ +0 diff --git a/type/__netbox/parameter/default/redis-host b/type/__netbox/parameter/default/redis-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/redis-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/redis-password b/type/__netbox/parameter/default/redis-password new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/redis-password @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/redis-port b/type/__netbox/parameter/default/redis-port new file mode 100644 index 0000000..a8c4b8e --- /dev/null +++ b/type/__netbox/parameter/default/redis-port @@ -0,0 +1 @@ +6379 diff --git a/type/__netbox/parameter/default/smtp-from-email b/type/__netbox/parameter/default/smtp-from-email new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-from-email @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/smtp-host b/type/__netbox/parameter/default/smtp-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/smtp-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/smtp-password b/type/__netbox/parameter/default/smtp-password new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-password @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/smtp-port b/type/__netbox/parameter/default/smtp-port new file mode 100644 index 0000000..7273c0f --- /dev/null +++ b/type/__netbox/parameter/default/smtp-port @@ -0,0 +1 @@ +25 diff --git a/type/__netbox/parameter/default/smtp-user b/type/__netbox/parameter/default/smtp-user new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-user @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 290c98b..159c813 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -5,3 +5,15 @@ ldap-user-base ldap-group-base ldap-require-group ldap-superuser-group +redis-host +redis-port +redis-password +redis-dbid-offset +smtp-host +smtp-port +smtp-user +smtp-password +smtp-from-email +basepath +http-proxy +https-proxy From 855843ea34ff896ccc23e08f5e84b6f0bdfb49b5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 09:19:12 +0200 Subject: [PATCH 31/98] __netbox: small code and config handling improvements fixed codestyle and detection if config files need to be copied. --- type/__netbox/gencode-remote | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 48463f7..8533f4a 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -66,26 +66,29 @@ printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' EOF # meta - printf "%s %s\n" "installed" "$VERSION" >> "$__messages_out" + printf "installed %s\n" "$VERSION" >> "$__messages_out" changes=yes # check if configuration changed elif grep -q "^__file/opt/netbox/" "$__messages_in"; then - cat <> "$__messages_out" + printf "configuration\n" >> "$__messages_out" changes=yes fi # check for changes if [ "$changes" = "yes" ]; then - cat < Date: Sat, 22 Aug 2020 14:40:07 +0200 Subject: [PATCH 32/98] __netbox: note + link to find NetBox versions --- type/__netbox/man.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 3e607b5..d91c6e8 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -19,7 +19,9 @@ for the services `netbox` and `netbox-rq`. REQUIRED PARAMETERS ------------------- version - NetBox version to be installed. + NetBox version to be installed. You can find the correct and newest version + on GitHub at the NetBox project page under + "`Releases `_". database PostgreSQL database name. From 987393942b9acc10e743c60b264f08dc203877bb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 15:53:24 +0200 Subject: [PATCH 33/98] __netbox: add more database parameters --- type/__netbox/files/configuration.py.sh | 4 ++-- type/__netbox/man.rst | 9 +++++++++ type/__netbox/manifest | 7 +++++++ type/__netbox/parameter/default/database-host | 1 + type/__netbox/parameter/default/database-port | 1 + type/__netbox/parameter/optional | 2 ++ type/__netbox/parameter/required | 1 + 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 type/__netbox/parameter/default/database-host create mode 100644 type/__netbox/parameter/default/database-port diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index e621845..65b00f5 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -19,8 +19,8 @@ DATABASE = { 'NAME': '$DATABASE_NAME', # Database name 'USER': '$DATABASE_USER', # PostgreSQL username 'PASSWORD': '$DATABASE_PASSWORD', # PostgreSQL password - 'HOST': 'localhost', # Database server - 'PORT': '', # Database port (leave blank for default) + 'HOST': '$DATABASE_HOST', # Database server + 'PORT': '$DATABASE_PORT', # Database port (leave blank for default) 'CONN_MAX_AGE': 300, # Max database connection age } diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index d91c6e8..a09b13a 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -26,6 +26,9 @@ version database PostgreSQL database name. +database-user + PostgreSQL database user. + database-password PostgreSQL database password. @@ -39,6 +42,12 @@ host OPTIONAL PARAMETERS ------------------- +database-host + PostgreSQL database hostname. Defaults to ``localhost``. + +database-port + PostgreSQL database port. Defaults to empty (uses the default port). + ldap-server LDAP server URI. Enables LDAP-backed authentication if specified. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 1f7f8ed..45c8afc 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -26,8 +26,15 @@ esac DATABASE_NAME=$(cat "$__object/parameter/database") export DATABASE_NAME +DATABASE_USER="$(cat "$__object/parameter/database-user")" +export DATABASE_USER DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") export DATABASE_PASSWORD +DATABASE_HOST="$(cat "$__object/parameter/database-host")" +export DATABASE_HOST +DATABASE_PORT="$(cat "$__object/parameter/database-port")" +export DATABASE_PORT + ALLOWED_HOST=$(cat "$__object/parameter/host") export ALLOWED_HOST SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox/parameter/default/database-host b/type/__netbox/parameter/default/database-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/database-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/database-port b/type/__netbox/parameter/default/database-port new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/database-port @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 159c813..d97f961 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,3 +1,5 @@ +database-host +database-port ldap-server ldap-bind-dn ldap-bind-password diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 388048a..84e8e4b 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -1,5 +1,6 @@ version database +database-user database-password secret-key host From 9fb3c7f52ef2a27e7d60d850d1918083c90d84c5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 16:00:15 +0200 Subject: [PATCH 34/98] __netbox: fix manpage indents --- type/__netbox/man.rst | 67 ++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index a09b13a..2e43bbd 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -43,88 +43,89 @@ host OPTIONAL PARAMETERS ------------------- database-host - PostgreSQL database hostname. Defaults to ``localhost``. + PostgreSQL database hostname. Defaults to ``localhost``. database-port - PostgreSQL database port. Defaults to empty (uses the default port). + PostgreSQL database port. Defaults to empty (uses the default port). ldap-server - LDAP server URI. Enables LDAP-backed authentication if specified. + LDAP server URI. Enables LDAP-backed authentication if specified. ldap-bind-dn - DN for the NetBox service account. Required for LDAP authentication. + DN for the NetBox service account. Required for LDAP authentication. ldap-bind-password - Password for the NetBox service account. Required for LDAP authentication. + Password for the NetBox service account. Required for LDAP authentication. ldap-user-base - Base used for searching user entries. Required for LDAP authentication. + Base used for searching user entries. Required for LDAP authentication. ldap-group-base - Base used for searching group entries. + Base used for searching group entries. ldap-require-group - Group required to login. + Group required to login. ldap-superuser-group - Make members of this groups superusers. + Make members of this groups superusers. redis-host - Redis database hostname. Defaults to ``localhost``. + Redis database hostname. Defaults to ``localhost``. redis-port - Redis database port. Defaults to ``6379``. + Redis database port. Defaults to ``6379``. redis-password - Redis password. Defaults to empty password. + Redis password. Defaults to empty password. redis-dbid-offset - Offset to set the redis database id's. The `tasks` database id is `offset + 0` - and `caching` is `offset + 1`. The offset defaults to ``0``. + Offset to set the redis database id's. The `tasks` database id is + `offset + 0` and `caching` is `offset + 1`. The offset defaults + to ``0``. smtp-host - Host of the SMTP email server. Defaults to ``localhost``. + Host of the SMTP email server. Defaults to ``localhost``. smtp-port - Port of the SMTP email server. Defaults to ``25``. + Port of the SMTP email server. Defaults to ``25``. smtp-user - Username to access the SMTP email server. Defaults to empty. + Username to access the SMTP email server. Defaults to empty. smtp-password - Password to access the SMTP email server. Defaults to empty. + Password to access the SMTP email server. Defaults to empty. smtp-from-email - Email from which NetBox will be sent of. Defaults to empty. + Email from which NetBox will be sent of. Defaults to empty. basepath - Base URL path if accessing netbox within a directory instead of directly the - webroot ``/``. For example, if installed at https://example.com/netbox/, set - the value ``netbox/``. + Base URL path if accessing netbox within a directory instead of directly the + webroot ``/``. For example, if installed at https://example.com/netbox/, set + the value ``netbox/``. http-proxy https-proxy - Proxy which will be used with any HTTP request like webhooks. + Proxy which will be used with any HTTP request like webhooks. BOOLEAN PARAMETERS ------------------ redis-ssl - Enables a secure TLS/SSL connection to the redis database. By default, ssl - is disabled. + Enables a secure TLS/SSL connection to the redis database. By default, ssl + is disabled. smtp-use-tls - Uses TLS to connect to the SMTP email server. `See documentation - Date: Sat, 22 Aug 2020 20:55:35 +0200 Subject: [PATCH 35/98] __netbox: auto-install sudo as dependency There are may be systems does not have `sudo` preinstalled like debian-minimal. Else, `su` could be used as alternative. --- type/__netbox/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 45c8afc..3033c4d 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -6,7 +6,7 @@ case "$os" in debian|ubuntu) # Install netbox dependencies. for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv; do + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do __package $pkg done From ae7ffac6bbf17e64b6ed79768f6db944395672dd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:29:30 +0200 Subject: [PATCH 36/98] __netbox: force virtualenv to be python3 Set a flag to be sure it is python3. Else, you may get an ugly error if python2 is not installed (and netbox requires python3.6 at minimum). --- type/__netbox/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 8533f4a..9359252 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -37,7 +37,8 @@ cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. -virtualenv /opt/netbox/venv +# forcing python3 to be sure (till python4 gets released ..) +virtualenv -p python3 /opt/netbox/venv # Install python dependencies. /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" From ce47cec2e7883723d8b11efda1f97090ac95cd8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:50:54 +0200 Subject: [PATCH 37/98] __netbox: silence curl progress bar .. but still showing errors if any. --- type/__netbox/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 9359252..e0eb09d 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -23,7 +23,7 @@ tmpdir=\$(mktemp -d) cd "\$tmpdir" # Download and extract sources. -curl -L '$url' > '$archive' +curl -sS -L '$url' > '$archive' tar xf '$archive' # Save cdist-upload configuration file. From 4a04a2c4728ef0625bc49a039ece3240058159f0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:58:44 +0200 Subject: [PATCH 38/98] __netbox: repalace tabs with whitespaces Doing `s/\t/ /g` to have all times the same indents. --- type/__netbox/files/ldap_config.py.sh | 14 ++--- type/__netbox/man.rst | 20 +++--- type/__netbox/manifest | 88 +++++++++++++-------------- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index bc494fb..cf4ed92 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -43,7 +43,7 @@ AUTH_LDAP_USER_ATTR_MAP = { EOF if [ "$LDAP_GROUP_BASE" != "" ]; then - cat << EOF + cat << EOF # This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group # hierarchy. @@ -55,21 +55,21 @@ AUTH_LDAP_GROUP_TYPE = PosixGroupType() AUTH_LDAP_MIRROR_GROUPS = True EOF - if [ "$LDAP_REQUIRE_GROUP" != "" ]; then - cat << EOF + if [ "$LDAP_REQUIRE_GROUP" != "" ]; then + cat << EOF # Define a group required to login. AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" EOF - fi + fi - if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then - cat << EOF + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + cat << EOF # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { "is_superuser": "$LDAP_SUPERUSER_GROUP", } EOF - fi + fi fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 2e43bbd..4e63ccc 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -144,16 +144,16 @@ EXAMPLES .. code-block:: sh __netbox --version 2.8.7 --database netbox \ - --database-password "secretsecretsecret" \ - --secret-key "secretsecretsecret" \ - --host "${__target_host:?}" \ - --ldap-server "ldaps://ldap.domain.tld" \ - --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ - --ldap-bind-password "secretsecretsecret" \ - --ldap-user-base "ou=users,dc=domain,dc=tld" \ - --ldap-group-base "ou=groups,dc=domain,dc=tld" \ - --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ - --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + --database-password "secretsecretsecret" \ + --secret-key "secretsecretsecret" \ + --host "${__target_host:?}" \ + --ldap-server "ldaps://ldap.domain.tld" \ + --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ + --ldap-bind-password "secretsecretsecret" \ + --ldap-user-base "ou=users,dc=domain,dc=tld" \ + --ldap-group-base "ou=groups,dc=domain,dc=tld" \ + --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ + --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" NOTES diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 3033c4d..21a9bc6 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -3,24 +3,24 @@ os=$(cat "$__global/explorer/os") case "$os" in - debian|ubuntu) - # Install netbox dependencies. - for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do - __package $pkg - done + debian|ubuntu) + # Install netbox dependencies. + for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do + __package $pkg + done - if [ -f "$__object/parameter/ldap-server" ]; then - for pkg in libldap2-dev libsasl2-dev libssl-dev; do - __package $pkg - done - fi - ;; - *) - printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; + if [ -f "$__object/parameter/ldap-server" ]; then + for pkg in libldap2-dev libsasl2-dev libssl-dev; do + __package $pkg + done + fi + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; esac @@ -41,38 +41,38 @@ SECRET_KEY=$(cat "$__object/parameter/secret-key") export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then - LDAP_SERVER=$(cat "$__object/parameter/ldap-server") - export LDAP_SERVER + LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + export LDAP_SERVER fi if [ -f "$__object/parameter/ldap-bind-dn" ]; then - LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") - export LDAP_BIND_DN + LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + export LDAP_BIND_DN fi if [ -f "$__object/parameter/ldap-bind-password" ]; then - LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") - export LDAP_BIND_PASSWORD + LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + export LDAP_BIND_PASSWORD fi if [ -f "$__object/parameter/ldap-user-base" ]; then - LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") - export LDAP_USER_BASE + LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + export LDAP_USER_BASE fi if [ -f "$__object/parameter/ldap-group-base" ]; then - LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") - export LDAP_GROUP_BASE + LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + export LDAP_GROUP_BASE fi if [ -f "$__object/parameter/ldap-require-group" ]; then - LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") - export LDAP_REQUIRE_GROUP + LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + export LDAP_REQUIRE_GROUP fi if [ -f "$__object/parameter/ldap-superuser-group" ]; then - LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") - export LDAP_SUPERUSER_GROUP + LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + export LDAP_SUPERUSER_GROUP fi # have default values @@ -123,12 +123,12 @@ BASEPATH="$(cat "$__object/parameter/basepath")" export BASEPATH if [ -f "$__object/parameter/http-proxy" ]; then - HTTP_PROXY=$(cat "$__object/parameter/http-proxy") - export HTTP_PROXY + HTTP_PROXY=$(cat "$__object/parameter/http-proxy") + export HTTP_PROXY fi if [ -f "$__object/parameter/https-proxy" ]; then - HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") - export HTTPS_PROXY + HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") + export HTTPS_PROXY fi if [ -f "$__object/parameter/login-required" ]; then @@ -149,23 +149,23 @@ mkdir -p "$__object/files" require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ - --source "$__object/files/configuration.py" + /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ + --source "$__object/files/configuration.py" if [ -f "$__object/parameter/ldap-server" ]; then - require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ - --source "$__object/files/ldap_config.py" + require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + --source "$__object/files/ldap_config.py" fi # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled done # Python worker configuration. require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" + --mode 644 --source "$__type/files/gunicorn.py" From 0b261070d362fed393c9e021654917bb525f8b7a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:04:33 +0200 Subject: [PATCH 39/98] __netbox: man note regards creating a superuser --- type/__netbox/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 4e63ccc..91313be 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -166,6 +166,11 @@ you are welcome to contribute! - `Possible optional settings ` +If you not setup ldap authentification, you may be interested into how to +`setting up a super user +` +directly on the machine to be able to access and use NetBox. + SEE ALSO -------- - `NetBox documentation `_ From 584e0bbda85669de99e907784c17a4d176aae996 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:23:36 +0200 Subject: [PATCH 40/98] __netbox: fix netbox version save The cdist directory needs to be created to write there. This is done directly after the new tarball is installed. --- type/__netbox/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index e0eb09d..d2ec5f9 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -33,6 +33,8 @@ cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' +mkdir '$install_dir/cdist' + cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' From 88ddebd0404a5b0d4adf597ec6b42c61c7240ea5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:47:15 +0200 Subject: [PATCH 41/98] __netbox: fix deprecated configuration option Fixes error message `/opt/netbox/netbox/netbox/settings.py:141: UserWarning: REMOTE_AUTH_DEFAULT_PERMISSIONS should be a dictionary. Backward compatibility will be removed in v2.10.` --- type/__netbox/files/configuration.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 65b00f5..8679437 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -247,7 +247,7 @@ REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] -REMOTE_AUTH_DEFAULT_PERMISSIONS = [] +REMOTE_AUTH_DEFAULT_PERMISSIONS = {} # This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. RELEASE_CHECK_TIMEOUT = 24 * 3600 From f5c8c26db6436777430a31c410c1c442fce184c7 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 11:36:35 +0200 Subject: [PATCH 42/98] __netbox: new config parameters for data paths .. to move them to an other partition. --- type/__netbox/files/configuration.py.sh | 36 ++++++++++++++++--------- type/__netbox/man.rst | 33 +++++++++++++++++++++++ type/__netbox/manifest | 19 +++++++++++++ type/__netbox/parameter/optional | 4 +++ 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 8679437..af19153 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -154,15 +154,6 @@ EOF cat << EOF } EOF - -else - cat << EOF -# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). -# HTTP_PROXIES = { -# 'http': 'http://10.10.1.10:3128', -# 'https': 'http://10.10.1.10:1080', -# } -EOF fi cat << EOF @@ -190,10 +181,18 @@ MAINTENANCE_MODE = False # all objects by specifying "?limit=0". MAX_PAGE_SIZE = 1000 +EOF + +if [ "$MEDIA_ROOT" != "" ]; then + cat << EOF # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that # the default value of this setting is derived from the installed location. -# MEDIA_ROOT = '/opt/netbox/netbox/media' +MEDIA_ROOT = '$MEDIA_ROOT' +EOF +fi + +cat << EOF # By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the # class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: # STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' @@ -257,14 +256,27 @@ RELEASE_CHECK_TIMEOUT = 24 * 3600 RELEASE_CHECK_URL = None # RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' +EOF + +if [ "$REPORTS_ROOT" != "" ]; then + cat << EOF # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -# REPORTS_ROOT = '/opt/netbox/netbox/reports' +REPORTS_ROOT = '$REPORTS_ROOT' +EOF +fi + +if [ "$SCRIPTS_ROOT" != "" ]; then + cat << EOF # The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -# SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' +SCRIPTS_ROOT = '$SCRIPTS_ROOT' +EOF +fi + +cat << EOF # By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use # local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 91313be..c910d5c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -107,6 +107,39 @@ http-proxy https-proxy Proxy which will be used with any HTTP request like webhooks. +data-root + This parameter set's the media, reports and scripts root to subdirectories + of the given directory. Values can be overwritten by special parameters like + `--media-root` for example. Use this option if you want to store persistant + data of netbox on an other partition. A trailing shlash is not needed. + + The data directories have following predefined sub-directory names: + + media root: + ``$data_root/media`` + reports root: + ``$data_root/reports`` + scripts root: + ``$data_root/scripts`` + +media-root + The file path to where media files (like image attachments) are stored. + Change this path if you require to store data on an other partiotion. + A trailing slash is not needed. By default, it will be stored into the + installation directory (``/opt/netbox/netbox/netbox/media``). + +reports-root + The file path of where custom reports are kept. Change this path if you + require to store data on an other partition. A trailing slash is not + needed. By default, it will be stored into the installation directory + (``/opt/netbox/netbox/netbox/reports``). + +scripts-root + The file path of where custom scripts are kept. Change this path if you + require to store data on an other partition. A trailing slash is not + needed. By default, it will be stored into the installation directory + (``/opt/netbox/netbox/netbox/scripts``). + BOOLEAN PARAMETERS ------------------ redis-ssl diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 21a9bc6..133b527 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -138,6 +138,25 @@ else fi export LOGIN_REQUIRED +if [ -f "$__object/parameter/data-root" ]; then + data_root="$(cat "$__object/parameter/data-root")" + MEDIA_ROOT="$data_root/media" + REPORTS_ROOT="$data_root/reports" + SCRIPTS_ROOT="$data_root/scripts" +fi +if [ -f "$__object/parameter/media-root" ]; then + MEDIA_ROOT="$(cat "$__object/parameter/media-root")" +fi +export MEDIA_ROOT +if [ -f "$__object/parameter/reports-root" ]; then + REPORTS_ROOT="$(cat "$__object/parameter/reports-root")" +fi +export REPORTS_ROOT +if [ -f "$__object/parameter/scripts-root" ]; then + SCRIPTS_ROOT="$(cat "$__object/parameter/scripts-root")" +fi +export SCRIPTS_ROOT + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index d97f961..21fea5f 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -19,3 +19,7 @@ smtp-from-email basepath http-proxy https-proxy +data-root +media-root +reports-root +scripts-root From 49bb527dea8752fe14155516581bd7ee9049143a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 16:52:02 +0200 Subject: [PATCH 43/98] __netbox: autogen secretkey and store at remote The secret key is generated if it is not set via parameter and the explorer does not return any. It will be saved in the netbox home directory to easily read the key for the config generation. --- type/__netbox/explorer/secretkey | 8 ++++++++ type/__netbox/explorer/version | 2 +- type/__netbox/man.rst | 16 +++++++++++----- type/__netbox/manifest | 19 ++++++++++++++++++- type/__netbox/parameter/optional | 1 + type/__netbox/parameter/required | 1 - 6 files changed, 39 insertions(+), 8 deletions(-) create mode 100755 type/__netbox/explorer/secretkey diff --git a/type/__netbox/explorer/secretkey b/type/__netbox/explorer/secretkey new file mode 100755 index 0000000..9f0331c --- /dev/null +++ b/type/__netbox/explorer/secretkey @@ -0,0 +1,8 @@ +#!/bin/sh -e + +# Explorer will output the key if he exists. + +secretkey="/opt/netbox/.secretkey" +if [ -f "$secretkey" ]; then + cat "$secretkey" +fi diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version index a9ed590..5231e16 100755 --- a/type/__netbox/explorer/version +++ b/type/__netbox/explorer/version @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # output version if exist version_path="/opt/netbox/netbox/cdist/version" diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index c910d5c..77b47d0 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -32,16 +32,22 @@ database-user database-password PostgreSQL database password. -secret-key - Random secret key of at least 50 alphanumeric characters. This key must be - unique to this installation and must not be shared outside the local - system. - host Hostname (domain or IP address) on which the application is served. OPTIONAL PARAMETERS ------------------- +secret-key + Random secret key of at least 50 alphanumeric characters and symbols. This + key must be unique to this installation and must not be shared outside the + local system. If no secret key is given, the type generates an own 50 chars + long key and saves it on the remote host to remember it for the next run. + + The secret, random string is used to assist in the creation new + cryptographic hashes for passwords and HTTP cookies. It is not directly + used for hasing user passwords or for encrpted storage. It can be changed + at any time, but will invalidate all existing sessions. + database-host PostgreSQL database hostname. Defaults to ``localhost``. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 133b527..cd47d38 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -37,7 +37,18 @@ export DATABASE_PORT ALLOWED_HOST=$(cat "$__object/parameter/host") export ALLOWED_HOST -SECRET_KEY=$(cat "$__object/parameter/secret-key") + +if [ -f "$__object/parameter/secret-key" ]; then + SECRET_KEY=$(cat "$__object/parameter/secret-key") +elif [ -s "$__object/explorer/secretkey" ]; then + # Can be done over netbox/generate_secret_key.py too, but it's to + # complicated with the variable setup (can't generated right now!). + # Generates a 50-character long key (without ' cause of python quotes) + SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" +else + # take the key that is already used + SECRET_KEY="$(cat "$__object/explorer/secretkey")" +fi export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then @@ -177,6 +188,12 @@ if [ -f "$__object/parameter/ldap-server" ]; then --source "$__object/files/ldap_config.py" fi +# save secret +require="__user/netbox" __file /opt/netbox/.secretkey --mode 400 \ + --owner netbox --source - << SECRET +$SECRET_KEY +SECRET + # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 21fea5f..47722e6 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,3 +1,4 @@ +secret-key database-host database-port ldap-server diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 84e8e4b..3d03e5a 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -2,5 +2,4 @@ version database database-user database-password -secret-key host From 4e90c93eef881a228272167f2e328de9a16bf892 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 17:15:32 +0200 Subject: [PATCH 44/98] __netbox: updated copyright Added me to the copyright after there are some more changes :-) --- type/__netbox/man.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 77b47d0..10485a1 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -217,11 +217,13 @@ SEE ALSO AUTHORS ------- Timothée Floure +Matthias Stecher COPYING ------- -Copyright \(C) 2020 Timothée Floure. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2020 Timothée Floure. +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. From 863c95059e7d5b10607b97b88aa49b402c0561d3 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 19:48:56 +0200 Subject: [PATCH 45/98] __netbox: fix REMOTE_AUTH_BACKEND module not found Updated with the backend from the current upstream configuration. --- type/__netbox/files/configuration.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index af19153..0b8304d 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -242,7 +242,7 @@ RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 # Remote authentication support REMOTE_AUTH_ENABLED = False -REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' +REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] From 95fd7d872ed1c649742008604d4becc88be010a4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 24 Aug 2020 18:57:27 +0200 Subject: [PATCH 46/98] __netbox: fix ldap config short-circuit Forgot to negotate if statement .. works now :/ --- type/__netbox/files/ldap_config.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index cf4ed92..c31f46a 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if [ "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then +if [ -z "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then # skip cat << EOF ############################## From 7d694342fffe1dd2bd850ac38c5fa7580da8fedb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 24 Aug 2020 19:31:49 +0200 Subject: [PATCH 47/98] __netbox: new parameter for automatic release checks --- type/__netbox/files/configuration.py.sh | 14 ++++++++++++-- type/__netbox/man.rst | 8 ++++++-- type/__netbox/manifest | 5 +++++ type/__netbox/parameter/boolean | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 0b8304d..21104b4 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -253,11 +253,21 @@ RELEASE_CHECK_TIMEOUT = 24 * 3600 # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the # version check or use the URL below to check for release in the official NetBox repository. -RELEASE_CHECK_URL = None -# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' EOF +if [ "$UPDATE_CHECK" != "" ]; then + cat << EOF +RELEASE_CHECK_URL = None + +EOF +else + cat << EOF +RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +EOF +fi + if [ "$REPORTS_ROOT" != "" ]; then cat << EOF # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 10485a1..761494a 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -163,8 +163,12 @@ smtp-use-ssl for more information. login-required - Sets if a login is required to access all sites. By default, anounymous users - can see most data (excluding secrets) but not make any changes. + Sets if a login is required to access all sites. By default, anonymous + users can see most data (excluding secrets) but not make any changes. + +update-notify + Enables the NetBox version check for new upstream updates. It checks every + 24 hours for new releases and notify the admin users in the gui if any. MESSAGES -------- diff --git a/type/__netbox/manifest b/type/__netbox/manifest index cd47d38..62da0a2 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -168,6 +168,11 @@ if [ -f "$__object/parameter/scripts-root" ]; then fi export SCRIPTS_ROOT +if [ -f "$__object/parameter/update-notify" ]; then + UPDATE_CHECK="yes" + export UPDATE_CHECK +fi + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/boolean b/type/__netbox/parameter/boolean index a94808c..d568037 100644 --- a/type/__netbox/parameter/boolean +++ b/type/__netbox/parameter/boolean @@ -2,3 +2,4 @@ redis-ssl smtp-use-ssl smtp-use-tls login-required +update-notify From 8ff7a4616dcedac333b5e93d994cbff2887ffbe9 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 25 Aug 2020 19:30:32 +0200 Subject: [PATCH 48/98] __netbox: allow multiple hostnames --- type/__netbox/files/configuration.py.sh | 2 +- type/__netbox/man.rst | 2 ++ type/__netbox/manifest | 8 ++++++-- type/__netbox/parameter/required | 1 - type/__netbox/parameter/required_multiple | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 type/__netbox/parameter/required_multiple diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 21104b4..4b7be11 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -11,7 +11,7 @@ cat << EOF # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. # # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] -ALLOWED_HOSTS = [ '$ALLOWED_HOST' ] +ALLOWED_HOSTS = [$ALLOWED_HOSTS ] # PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: # https://docs.djangoproject.com/en/stable/ref/settings/#databases diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 761494a..2af2bef 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -34,6 +34,7 @@ database-password host Hostname (domain or IP address) on which the application is served. + Multiple hostnames are possible; given as multiple arguments. OPTIONAL PARAMETERS ------------------- @@ -190,6 +191,7 @@ EXAMPLES --database-password "secretsecretsecret" \ --secret-key "secretsecretsecret" \ --host "${__target_host:?}" \ + --host "cool-netbox.xyz" \ --ldap-server "ldaps://ldap.domain.tld" \ --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ --ldap-bind-password "secretsecretsecret" \ diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 62da0a2..2f5cf4e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -35,8 +35,12 @@ export DATABASE_HOST DATABASE_PORT="$(cat "$__object/parameter/database-port")" export DATABASE_PORT -ALLOWED_HOST=$(cat "$__object/parameter/host") -export ALLOWED_HOST +# list of hosts +ALLOWED_HOSTS="" +while read -r hostname; do + ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," +done < "$__object/parameter/host" +export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 3d03e5a..02fca9f 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -2,4 +2,3 @@ version database database-user database-password -host diff --git a/type/__netbox/parameter/required_multiple b/type/__netbox/parameter/required_multiple new file mode 100644 index 0000000..c70dc2d --- /dev/null +++ b/type/__netbox/parameter/required_multiple @@ -0,0 +1 @@ +host From eaf639790b00bdb5d7d9197939d0e6fa7963f3a2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 17:39:19 +0200 Subject: [PATCH 49/98] __netbox: remove ldap pip3 package if not required --- type/__netbox/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d2ec5f9..abe7ffa 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -48,6 +48,8 @@ EOF if [ -f "$__object/parameter/ldap-server" ]; then echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + else + echo "/opt/netbox/venv/bin/pip3 uninstall django-auth-ldap" fi cat << EOF From e0a1b4f66343415b002c6d1c86114da3c9d402f8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 20:01:23 +0200 Subject: [PATCH 50/98] __netbox_gunicorn: extract gunicorn setup from __netbox Because someone *want* to use something other than just gunicorn, it was extracted to a own type. Because gunicorn is a bit deep in the netbox installation process, it's a bit harder to isolate it. `__netbox_uwsgi` will come, too. --- type/__netbox/gencode-remote | 8 +- type/__netbox/manifest | 14 +--- type/__netbox_gunicorn/explorer/installed | 4 + .../explorer/should_installed | 3 + .../files/gunicorn.py.sh} | 19 ++++- .../files/netbox.service | 3 +- type/__netbox_gunicorn/gencode-remote | 36 +++++++++ type/__netbox_gunicorn/man.rst | 75 +++++++++++++++++++ type/__netbox_gunicorn/manifest | 20 +++++ .../parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + type/__netbox_gunicorn/singleton | 0 12 files changed, 168 insertions(+), 16 deletions(-) create mode 100755 type/__netbox_gunicorn/explorer/installed create mode 100755 type/__netbox_gunicorn/explorer/should_installed rename type/{__netbox/files/gunicorn.py => __netbox_gunicorn/files/gunicorn.py.sh} (56%) mode change 100644 => 100755 rename type/{__netbox => __netbox_gunicorn}/files/netbox.service (88%) create mode 100755 type/__netbox_gunicorn/gencode-remote create mode 100644 type/__netbox_gunicorn/man.rst create mode 100755 type/__netbox_gunicorn/manifest create mode 100644 type/__netbox_gunicorn/parameter/default/bind-to create mode 100644 type/__netbox_gunicorn/parameter/optional_multiple create mode 100644 type/__netbox_gunicorn/singleton diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index abe7ffa..af4d8c8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -35,6 +35,7 @@ rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' mkdir '$install_dir/cdist' +cp '$src/requirements.txt' /opt/netbox/ # backup dependency info cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' @@ -43,7 +44,9 @@ cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Install python dependencies. -/opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" +# avoid gunicorn, because it will be done in an other type +grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ + | xargs /opt/netbox/venv/bin/pip3 install EOF if [ -f "$__object/parameter/ldap-server" ]; then @@ -94,8 +97,7 @@ fi # check for changes if [ "$changes" = "yes" ]; then cat << EOF -# Restart services. -service netbox restart +# Restart service. service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2f5cf4e..f0da2a3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,13 +204,7 @@ $SECRET_KEY SECRET -# Upload systemd units and gunicorn configuration. -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done - -# Python worker configuration. -require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" +# Upload systemd unit for worker service +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled diff --git a/type/__netbox_gunicorn/explorer/installed b/type/__netbox_gunicorn/explorer/installed new file mode 100755 index 0000000..c6f5d87 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show gunicorn | awk '/Version:/{print $2}' diff --git a/type/__netbox_gunicorn/explorer/should_installed b/type/__netbox_gunicorn/explorer/should_installed new file mode 100755 index 0000000..073be92 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/should_installed @@ -0,0 +1,3 @@ +#!/bin/sh -e + +awk -v FS="==" '$1 ~ /gunicorn/{print $2}' /opt/netbox/requirements.txt diff --git a/type/__netbox/files/gunicorn.py b/type/__netbox_gunicorn/files/gunicorn.py.sh old mode 100644 new mode 100755 similarity index 56% rename from type/__netbox/files/gunicorn.py rename to type/__netbox_gunicorn/files/gunicorn.py.sh index c582677..862ba06 --- a/type/__netbox/files/gunicorn.py +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -1,9 +1,23 @@ +#!/bin/sh -e + +# Generates gunicorn config +# see https://docs.gunicorn.org/en/stable/settings.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = '127.0.0.1:8001' +bind = [$HOST ] # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. -workers = 3 +workers = $(( 2*cores + 1 )) # Number of threads per worker process threads = 3 @@ -14,3 +28,4 @@ timeout = 120 # The maximum number of requests a worker can handle before being respawned max_requests = 5000 max_requests_jitter = 500 +EOF diff --git a/type/__netbox/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service similarity index 88% rename from type/__netbox/files/netbox.service rename to type/__netbox_gunicorn/files/netbox.service index 18eb045..475ddc4 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,5 +1,5 @@ [Unit] -Description=NetBox WSGI Service +Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ After=network-online.target Wants=network-online.target @@ -19,4 +19,5 @@ RestartSec=30 PrivateTmp=true [Install] +Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote new file mode 100755 index 0000000..6958976 --- /dev/null +++ b/type/__netbox_gunicorn/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh -e + +curr_installed="$(cat "$__object/explorer/installed")" +should_installed="$(cat "$__object/explorer/should_installed")" + +# gunicorn version change +if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" + + do_restart=yes + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" +fi + +# configuration changes +if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configuration\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no message cause it should be obvious +fi + + +# restart gunicorn +if [ "$do_restart"]; then + cat << EOF +# Restart service +service gunicorn-netbox restart +EOF + echo restarted >> "$__messages_out" +fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst new file mode 100644 index 0000000..38ac4a5 --- /dev/null +++ b/type/__netbox_gunicorn/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_gunicorn - run netbox with gunicorn + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The hosts the gunicorn socket should be bind to. Formats are `IP`, + `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple + times. Defaults to ``127.0.0.1:8001``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +updated $old to $new + The version of the gunicorn software was updated from `$old` to `$new`. + +configuration + Configuration for gunicorn changed. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_gunicorn + + # with arguments + __netbox $args + require="__netbox" __netbox_gunicorn \ + --bind-to 0.0.0.0:8001 \ + --bind-to 1.2.3.4:5678 + + +SEE ALSO +-------- +:strong:`__netbox`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest new file mode 100755 index 0000000..dab8181 --- /dev/null +++ b/type/__netbox_gunicorn/manifest @@ -0,0 +1,20 @@ +#!/bin/sh -e + +HOST="" +while read -r host; do + HOST="$HOST '$host'," +done < "$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + +# gunicorn config file +__file /opt/netbox/gunicorn.py \ + --mode 644 --source "$__object/files/gunicorn.py" + +# install service file +__systemd_unit gunicorn-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_gunicorn/parameter/default/bind-to b/type/__netbox_gunicorn/parameter/default/bind-to new file mode 100644 index 0000000..f4c980e --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:8001 diff --git a/type/__netbox_gunicorn/parameter/optional_multiple b/type/__netbox_gunicorn/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to diff --git a/type/__netbox_gunicorn/singleton b/type/__netbox_gunicorn/singleton new file mode 100644 index 0000000..e69de29 From bbce0030aba387232f48a6c4b676c2b82cba28ed Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 21:07:31 +0200 Subject: [PATCH 51/98] __netbox: enable ldap usage via configuration.py The REMOTE_AUTH_BACKEND must be set to use LDAP. It now exports USE_LDAP to generally say if LDAP is being used in the configuration or not. --- type/__netbox/files/configuration.py.sh | 14 ++++++++++++++ type/__netbox/files/ldap_config.py.sh | 2 +- type/__netbox/manifest | 12 ++++++------ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 4b7be11..ea73133 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -240,9 +240,23 @@ PREFER_IPV4 = False RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = 22 RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 +EOF + +if [ "$USE_LDAP" ]; then + cat << EOF +# Remote authentication support with ldap +REMOTE_AUTH_ENABLED = True +REMOTE_AUTH_BACKEND = 'netbox.authentication.LDAPBackend' +EOF +else + cat << EOF # Remote authentication support REMOTE_AUTH_ENABLED = False REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' +EOF +fi + +cat << EOF REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index c31f46a..a49d800 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if [ -z "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then +if [ -z "$USE_LDAP" ]; then # skip cat << EOF ############################## diff --git a/type/__netbox/manifest b/type/__netbox/manifest index f0da2a3..079ea48 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -57,38 +57,38 @@ export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + USE_LDAP=yes export LDAP_SERVER fi - if [ -f "$__object/parameter/ldap-bind-dn" ]; then LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + USE_LDAP=yes export LDAP_BIND_DN fi - if [ -f "$__object/parameter/ldap-bind-password" ]; then LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + USE_LDAP=yes export LDAP_BIND_PASSWORD fi - if [ -f "$__object/parameter/ldap-user-base" ]; then LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + USE_LDAP=yes export LDAP_USER_BASE fi - if [ -f "$__object/parameter/ldap-group-base" ]; then LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") export LDAP_GROUP_BASE fi - if [ -f "$__object/parameter/ldap-require-group" ]; then LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") export LDAP_REQUIRE_GROUP fi - if [ -f "$__object/parameter/ldap-superuser-group" ]; then LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") export LDAP_SUPERUSER_GROUP fi +# export if base ldap parameters are used +export USE_LDAP # have default values REDIS_HOST="$(cat "$__object/parameter/redis-host")" From 1ef4420c537570206780757fa5b78aa6895f5829 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 27 Aug 2020 19:46:09 +0200 Subject: [PATCH 52/98] __netbox_uwsgi: new type to handle uwsgi for netbox --- type/__netbox/gencode-remote | 2 +- type/__netbox/man.rst | 2 +- type/__netbox_gunicorn/gencode-remote | 5 +- type/__netbox_gunicorn/man.rst | 2 +- type/__netbox_uwsgi/explorer/installed | 4 + type/__netbox_uwsgi/explorer/upgradeable | 4 + type/__netbox_uwsgi/files/netbox.service | 23 ++++++ type/__netbox_uwsgi/files/uwsgi.ini.sh | 22 ++++++ type/__netbox_uwsgi/gencode-remote | 35 +++++++++ type/__netbox_uwsgi/man.rst | 75 +++++++++++++++++++ type/__netbox_uwsgi/manifest | 18 +++++ type/__netbox_uwsgi/parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + 13 files changed, 188 insertions(+), 6 deletions(-) create mode 100755 type/__netbox_uwsgi/explorer/installed create mode 100755 type/__netbox_uwsgi/explorer/upgradeable create mode 100644 type/__netbox_uwsgi/files/netbox.service create mode 100755 type/__netbox_uwsgi/files/uwsgi.ini.sh create mode 100755 type/__netbox_uwsgi/gencode-remote create mode 100644 type/__netbox_uwsgi/man.rst create mode 100755 type/__netbox_uwsgi/manifest create mode 100644 type/__netbox_uwsgi/parameter/default/bind-to create mode 100644 type/__netbox_uwsgi/parameter/optional_multiple diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index af4d8c8..0153579 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -89,7 +89,7 @@ EOF fi # meta - printf "configuration\n" >> "$__messages_out" + printf "configured\n" >> "$__messages_out" changes=yes fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 2af2bef..6bb1e36 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -176,7 +176,7 @@ MESSAGES installed $VERSION Netbox was fresh installed or updated. The new version number is appended. -configuration +configured Some configuration files got updated and therefore the service was restarted. This message will not be echoed if configuration got updated due a standard installation. diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 6958976..e424c22 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -16,7 +16,7 @@ fi # configuration changes if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then do_restart=yes - printf "configuration\n" >> "$__messages_out" + printf "configured\n" >> "$__messages_out" fi # application @@ -27,10 +27,9 @@ fi # restart gunicorn -if [ "$do_restart"]; then +if [ "$do_restart" ]; then cat << EOF # Restart service service gunicorn-netbox restart EOF - echo restarted >> "$__messages_out" fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 38ac4a5..43f5864 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -34,7 +34,7 @@ MESSAGES updated $old to $new The version of the gunicorn software was updated from `$old` to `$new`. -configuration +configured Configuration for gunicorn changed. In both cases, and at messages from the `__netbox` type, it restarts the diff --git a/type/__netbox_uwsgi/explorer/installed b/type/__netbox_uwsgi/explorer/installed new file mode 100755 index 0000000..a2393d0 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show uwsgi | awk '/Version:/{print $2}' diff --git a/type/__netbox_uwsgi/explorer/upgradeable b/type/__netbox_uwsgi/explorer/upgradeable new file mode 100755 index 0000000..f4b0a22 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/upgradeable @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print latest version if availble +/opt/netbox/venv/bin/pip3 list --outdated | awk '$1 == "uwsgi" {print $3}' diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service new file mode 100644 index 0000000..cfce752 --- /dev/null +++ b/type/__netbox_uwsgi/files/netbox.service @@ -0,0 +1,23 @@ +[Unit] +Description=Netbox uWSGI WSGI Service +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +PIDFile=/var/tmp/netbox.pid +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/uwsgi --master --wsgi-file netbox/netbox/wsgi.py uwsgi.ini + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +Alias=netbox.service +WantedBy=multi-user.target diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh new file mode 100755 index 0000000..557a5fa --- /dev/null +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# Generates uwsgi config +# see https://uwsgi-docs.readthedocs.io/en/latest/Options.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF +[uwsgi] +; socket to bind +socket = $HOST + +; processes and threads +processes = $(( 2*cores + 1 )) +threads = 2 +EOF diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote new file mode 100755 index 0000000..5cc82d1 --- /dev/null +++ b/type/__netbox_uwsgi/gencode-remote @@ -0,0 +1,35 @@ +#!/bin/sh -e + +# not installed +if ! [ -s "$__object/explorer/installed" ]; then + echo "/opt/netbox/venv/bin/pip3 install uwsgi" + do_restart=yes + printf "installed\n" >> "$__messages_out" + +# updates available +elif [ -s "$__object/explorer/upgradeable" ]; then + echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + do_restart=yes + printf "upgraded\n" >> "$__messages_out" +fi + +# changed configuration +if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no messages cause this is obvious +fi + + +# restart uwsgi +if [ "$do_restart" ]; then + cat << EOF +# Restart service +service uwsgi-netbox restart +EOF +fi diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst new file mode 100644 index 0000000..5bf3160 --- /dev/null +++ b/type/__netbox_uwsgi/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_uwsgi - run netbox with uwsgi + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. + Defaults to ``127.0.0.1:3031``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +installed + The uwsgi service was installed. + +upgraded + The uwsgi service was upgraded. + +configured + The uwsgi configuration got updated. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_uwsgi + + # with special bind + require="__netbox" __netbox_uwsgi --bind-to 0.0.0.0:3032 \ + --bind-to 0.0.0.0:3033 + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest new file mode 100755 index 0000000..779f93b --- /dev/null +++ b/type/__netbox_uwsgi/manifest @@ -0,0 +1,18 @@ +#!/bin/sh -e + +HOST="$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" + +# uwsgi config file +# TODO maybe patching with __key_value cause of .ini ? +__file /opt/netbox/uwsgi.ini \ + --mode 644 --source "$__object/files/uwsgi.ini" + +# install service file +__systemd_unit uwsgi-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_uwsgi/parameter/default/bind-to b/type/__netbox_uwsgi/parameter/default/bind-to new file mode 100644 index 0000000..c696456 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:3031 diff --git a/type/__netbox_uwsgi/parameter/optional_multiple b/type/__netbox_uwsgi/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to From 9d8b3ebe743685daac13d54ac3bf2449517c3318 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 1 Sep 2020 10:02:08 +0200 Subject: [PATCH 53/98] __netbox: add wrapper service to manage the wsgi services The wrapper service will "control" the services added from the __netbox_* types to provide a general interface. This is more dynamic than the alias approach used previously. Through this, it is possible to handle multiple wsgi services for netbox - if this works .. See as a reference: http://alesnosek.com/blog/2016/12/04/controlling-a-multi-service-application-with-systemd/ --- type/__netbox/files/netbox.service | 12 ++++++++++++ type/__netbox/gencode-remote | 1 + type/__netbox/manifest | 10 ++++++---- type/__netbox_gunicorn/files/netbox.service | 3 ++- type/__netbox_gunicorn/gencode-remote | 6 ------ type/__netbox_gunicorn/man.rst | 3 +-- type/__netbox_uwsgi/files/netbox.service | 3 ++- type/__netbox_uwsgi/gencode-remote | 6 ------ type/__netbox_uwsgi/man.rst | 3 +-- 9 files changed, 25 insertions(+), 22 deletions(-) create mode 100644 type/__netbox/files/netbox.service diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service new file mode 100644 index 0000000..88baaeb --- /dev/null +++ b/type/__netbox/files/netbox.service @@ -0,0 +1,12 @@ +[Unit] +Description=NetBox Service Wrapper +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 0153579..d598aa8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -98,6 +98,7 @@ fi if [ "$changes" = "yes" ]; then cat << EOF # Restart service. +service netbox restart service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 079ea48..95688d3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,7 +204,9 @@ $SECRET_KEY SECRET -# Upload systemd unit for worker service -__systemd_unit netbox-rq.service \ - --source "$__type/files/netbox-rq.service" \ - --enablement-state enabled +# Upload systemd unit for worker and wsgi service +for unit in netbox netbox-rq; do + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled +done diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 475ddc4..2154d27 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,6 +1,8 @@ [Unit] Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ +PartOf=netbox.service +After=netbox.service After=network-online.target Wants=network-online.target @@ -19,5 +21,4 @@ RestartSec=30 PrivateTmp=true [Install] -Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index e424c22..163bb1a 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -19,12 +19,6 @@ if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then printf "configured\n" >> "$__messages_out" fi -# application -if grep -q "^__netbox:" "$__messages_in"; then - do_restart=yes - # no message cause it should be obvious -fi - # restart gunicorn if [ "$do_restart" ]; then diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 43f5864..ac86261 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -37,8 +37,7 @@ updated $old to $new configured Configuration for gunicorn changed. -In both cases, and at messages from the `__netbox` type, it restarts the -service to using the up-to-date version. +In both cases, it restarts the service to use the up-to-date version. EXAMPLES diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index cfce752..bc10571 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -1,6 +1,8 @@ [Unit] Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ +PartOf=netbox.service +After=netbox.service After=network-online.target Wants=network-online.target @@ -19,5 +21,4 @@ RestartSec=30 PrivateTmp=true [Install] -Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 5cc82d1..3127b8b 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -19,12 +19,6 @@ if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then printf "configured\n" >> "$__messages_out" fi -# application -if grep -q "^__netbox:" "$__messages_in"; then - do_restart=yes - # no messages cause this is obvious -fi - # restart uwsgi if [ "$do_restart" ]; then diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 5bf3160..f328496 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -39,8 +39,7 @@ upgraded configured The uwsgi configuration got updated. -In both cases, and at messages from the `__netbox` type, it restarts the -service to using the up-to-date version. +In all cases, it restarts the service to use the up-to-date version. EXAMPLES From facb5a64d3824df095c6374a39f3a78482987046 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 1 Sep 2020 12:40:05 +0200 Subject: [PATCH 54/98] __netbox{,_gunicorn}: disabled shellcheck warnings Shellcheck warned about creating content for a python array. As the string will be printed literally into the config, the warning does not match to the current case. --- type/__netbox/manifest | 3 ++- type/__netbox_gunicorn/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 95688d3..aef50c1 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -38,9 +38,10 @@ export DATABASE_PORT # list of hosts ALLOWED_HOSTS="" while read -r hostname; do + # shellcheck disable=SC2089 ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," done < "$__object/parameter/host" -export ALLOWED_HOSTS +export ALLOWED_HOSTS # shellcheck disable=SC2090 if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index dab8181..d61cbe6 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -2,9 +2,10 @@ HOST="" while read -r host; do + # shellcheck disable=SC2089 HOST="$HOST '$host'," done < "$__object/parameter/bind-to" -export HOST +export HOST # shellcheck disable=SC2090 # process template mkdir "$__object/files" From dbc91cb33914e0b3c899dd169d61613f7a766c2a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Sep 2020 07:50:12 +0200 Subject: [PATCH 55/98] __netbox*: updated man pages --- type/__netbox/man.rst | 20 +++++++++++++++++--- type/__netbox_gunicorn/man.rst | 13 ++++++++++--- type/__netbox_uwsgi/man.rst | 13 ++++++++++--- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 6bb1e36..a3f283c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -12,8 +12,12 @@ This (singleton) type installs and configures a NetBox instance, a web application to help manage and document computer networks. It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. -Netbox will be run via `gnuicorn` as WSGI service. It setup systemd unit files -for the services `netbox` and `netbox-rq`. +It setup systemd unit files for the services `netbox` and `netbox-rq`. To +access the application through WSGI, uWSGI or Gunicorn can be used. The setup +can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. + +The Gunicorn setup is recommended from the NetBox documentation. Consult each +manual page to decide. The types must be called after the `__netbox` type. REQUIRED PARAMETERS @@ -36,6 +40,7 @@ host Hostname (domain or IP address) on which the application is served. Multiple hostnames are possible; given as multiple arguments. + OPTIONAL PARAMETERS ------------------- secret-key @@ -147,6 +152,7 @@ scripts-root needed. By default, it will be stored into the installation directory (``/opt/netbox/netbox/netbox/scripts``). + BOOLEAN PARAMETERS ------------------ redis-ssl @@ -171,6 +177,7 @@ update-notify Enables the NetBox version check for new upstream updates. It checks every 24 hours for new releases and notify the admin users in the gui if any. + MESSAGES -------- installed $VERSION @@ -199,6 +206,8 @@ EXAMPLES --ldap-group-base "ou=groups,dc=domain,dc=tld" \ --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + # using recommended gunicorn setup + require="__netbox" __netbox_gunicorn NOTES @@ -216,9 +225,14 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. + SEE ALSO -------- -- `NetBox documentation `_ +`NetBox documentation `_ + +:strong:`cdist-type__netbox_gunicorn`\ (7) +:strong:`cdist-type__netbox_uwsgi`\ (7) + AUTHORS ------- diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index ac86261..c51be0c 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -3,12 +3,16 @@ cdist-type__netbox_uwsgi(7) NAME ---- -cdist-type__netbox_gunicorn - run netbox with gunicorn +cdist-type__netbox_gunicorn - Run NetBox with Gunicorn DESCRIPTION ----------- -This space intentionally left blank. +This (singleton) type installs Gunicorn into the NetBox `python-venv` to host +the NetBox WSGI application. It provides the application as HTTP over the given +sockets. Static content must be served independent of Gunicorn. The Gunicorn +daemon is available as the `gunicorn-netbox` systemd service, but also +available via the `netbox` wrapper service. REQUIRED PARAMETERS @@ -58,7 +62,10 @@ EXAMPLES SEE ALSO -------- -:strong:`__netbox`\ (7) +`Gunicorn Documentation `_ + +:strong:`cdist-type__netbox`\ (7) +:strong:`cdist-type__netbox_uwsgi`\ (7) AUTHORS diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index f328496..9d5ced7 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -3,12 +3,16 @@ cdist-type__netbox_uwsgi(7) NAME ---- -cdist-type__netbox_uwsgi - run netbox with uwsgi +cdist-type__netbox_uwsgi - Run NetBox with uWSGI DESCRIPTION ----------- -This space intentionally left blank. +This (singleton) type installs uWSGI into the NetBox `python-venv`. It hosts +the NetBox WSGI application over the WSGI protocol. A further server must be +installed to provide it as HTTP. This application is available via the +`uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper +service, too. REQUIRED PARAMETERS @@ -58,7 +62,10 @@ EXAMPLES SEE ALSO -------- -:strong:`TODO`\ (7) +`uWSGI Documentation `_ + +:strong:`cdist-type__netbox`\ (7) +:strong:`cdist-type__netbox_gunicorn`\ (7) AUTHORS From 2805b6beffc61c380ebffe26a487bbdf587c992b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Sep 2020 09:15:45 +0200 Subject: [PATCH 56/98] __netbox{,_gunicorn}: fix shellcheck directives --- type/__netbox/manifest | 3 ++- type/__netbox_gunicorn/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index aef50c1..273423e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -41,7 +41,8 @@ while read -r hostname; do # shellcheck disable=SC2089 ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," done < "$__object/parameter/host" -export ALLOWED_HOSTS # shellcheck disable=SC2090 +# shellcheck disable=SC2090 +export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index d61cbe6..25ea452 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -5,7 +5,8 @@ while read -r host; do # shellcheck disable=SC2089 HOST="$HOST '$host'," done < "$__object/parameter/bind-to" -export HOST # shellcheck disable=SC2090 +# shellcheck disable=SC2090 +export HOST # process template mkdir "$__object/files" From 243e34f0a5e6d7e012c3eee3865cc995432f6f07 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 11:20:20 +0200 Subject: [PATCH 57/98] __netbox_uwsgi: enable further protocols to bind to Enables multiple protocols like fastcgi or HTTP to bind to. This makes it more flexible to use. Also, a little fix for __netbox was done: correctly output a error msg. --- type/__netbox/manifest | 2 +- type/__netbox_uwsgi/files/uwsgi.ini.sh | 29 +++++++++++++++++-- type/__netbox_uwsgi/man.rst | 29 +++++++++++++++---- .../parameter/optional_multiple | 4 +++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 273423e..cb6062e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -127,7 +127,7 @@ fi export SMTP_USE_SSL if [ -f "$__object/parameter/smtp-use-tls" ]; then if [ "$SMTP_USE_SSL" = "True" ]; then - echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" + echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" >&2 exit 2 fi SMTP_USE_TLS="True" diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 557a5fa..324b150 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -2,6 +2,18 @@ # Generates uwsgi config # see https://uwsgi-docs.readthedocs.io/en/latest/Options.html +# or https://uwsgi-docs-additions.readthedocs.io/en/latest/Options.html + +# params: +# 1: parameter name +# 2: parameter value file +# +# output: the lines for the configuration option +multi_options() { + while read -r line; do + printf "%s = %s\n" "$1" "$line" + done < "$2" +} # fix missing $__explorer # see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 @@ -13,8 +25,21 @@ cores="$(cat "$__explorer/cpu_cores")" cat << EOF [uwsgi] -; socket to bind -socket = $HOST +; socket(s) to bind +EOF + +# special protocol to bind +for param in $(find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print); do + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" + socket_changes=yes +done +# else, default bind to +if [ -z "$socket_changes" ]; then + multi_options "socket" "$__object/parameter/bind-to" +fi + + +cat << EOF ; processes and threads processes = $(( 2*cores + 1 )) diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 9d5ced7..7501573 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -9,10 +9,11 @@ cdist-type__netbox_uwsgi - Run NetBox with uWSGI DESCRIPTION ----------- This (singleton) type installs uWSGI into the NetBox `python-venv`. It hosts -the NetBox WSGI application over the WSGI protocol. A further server must be -installed to provide it as HTTP. This application is available via the -`uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper -service, too. +the NetBox WSGI application via the WSGI protocol. A further server must be +installed to provide it as HTTP and serve static content. It supports multiple +protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This +application is available via the `uwsgi-netbox` systemd service. It is +controllable via the `netbox` wrapper service, too. REQUIRED PARAMETERS @@ -24,7 +25,15 @@ OPTIONAL PARAMETERS ------------------- bind-to The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. - Defaults to ``127.0.0.1:3031``. + Defaults to ``127.0.0.1:3031``. Can be set multiple times. + +uwsgi-bind +http-bind +fastcgi-bind +scgi-bind + Bind the application to a specific protocol instead of implicit uwsgi via + ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must + be a UNIX/TCP socket. Can be set multiple times. BOOLEAN PARAMETERS @@ -55,10 +64,18 @@ EXAMPLES __netbox $args require="__netbox" __netbox_uwsgi - # with special bind + # with multiple binds + __netbox $args require="__netbox" __netbox_uwsgi --bind-to 0.0.0.0:3032 \ --bind-to 0.0.0.0:3033 + # with multiple protocols + # parameter `--bind-to` will be ignored + __netbox $args + require="__netbox" __netbox_uwsgi --uwsgi-bind 0.0.0.0:3031 \ + --http-bind 0.0.0.0:8080 \ + --fastcgi-bind 1.2.3.4:5678 + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/parameter/optional_multiple b/type/__netbox_uwsgi/parameter/optional_multiple index 331c077..3f3e7d4 100644 --- a/type/__netbox_uwsgi/parameter/optional_multiple +++ b/type/__netbox_uwsgi/parameter/optional_multiple @@ -1 +1,5 @@ bind-to +uwsgi-bind +http-bind +fastcgi-bind +scgi-bind From af91b6e3a1ad15f6579983e25570a0bda8796a9b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 11:26:15 +0200 Subject: [PATCH 58/98] __netbox_uwsgi: mark as singleton This should have been done much earlier .. :/ --- type/__netbox_uwsgi/singleton | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 type/__netbox_uwsgi/singleton diff --git a/type/__netbox_uwsgi/singleton b/type/__netbox_uwsgi/singleton new file mode 100644 index 0000000..e69de29 From 314a0d4d8ef6235c70fbf3d7b1421f1b34cb18c2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 13:53:05 +0200 Subject: [PATCH 59/98] __netbox: kill all running venv processes To avoid aborts because of the python venv could not be updated by killing all processes that uses the venv. It will be done all times to prevent any error, because it could not be reliably detected if the type installs or updates NetBox. --- type/__netbox/gencode-remote | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d598aa8..f200408 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -30,6 +30,17 @@ tar xf '$archive' cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" +EOF + + # Stop everything in the pyenv to update + cat << EOF +# Try to kill everything in the venv +systemctl stop netbox gunicorn-netbox uwsgi-netbox || true +ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true + +EOF + + cat << EOF # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' From 755bd9098ede2dbad6c557f7c4983037fe3aab22 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 15:18:49 +0200 Subject: [PATCH 60/98] __netbox*: update systemd service files The `uwsgi-netbox` service now works, also the `netbox` wrapper service. The PID file was removed from the Gunicorn service as it is not required and a bit more efford to move it to `/run/` due to permissions. Generally, all depend on `network.target` instead of `network-online.target` now, and signals for reload, stop and kill were added (especially required the uwsgi service). --- type/__netbox/files/netbox.service | 5 +++-- type/__netbox_gunicorn/files/netbox.service | 14 ++++++++------ type/__netbox_uwsgi/files/netbox.service | 16 +++++++++------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service index 88baaeb..738e178 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox/files/netbox.service @@ -1,12 +1,13 @@ [Unit] Description=NetBox Service Wrapper Documentation=https://netbox.readthedocs.io/en/stable/ -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=oneshot RemainAfterExit=yes +ExecStart=/bin/true [Install] WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 2154d27..55f3549 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -3,22 +3,24 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service After=netbox.service -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=simple User=netbox Group=netbox -PIDFile=/var/tmp/netbox.pid WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi +ExecStart=/opt/netbox/venv/bin/gunicorn --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi +# signals: https://docs.gunicorn.org/en/stable/signals.html +ExecReload=kill -HUP $MAINPID +ExecStop=kill -TERM $MAINPID +KillSignal=SIGQUIT Restart=on-failure RestartSec=30 -PrivateTmp=true [Install] -WantedBy=multi-user.target +WantedBy=netbox.service diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index bc10571..10ea734 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -3,22 +3,24 @@ Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service After=netbox.service -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=simple User=netbox Group=netbox -PIDFile=/var/tmp/netbox.pid -WorkingDirectory=/opt/netbox +WorkingDirectory=/opt/netbox/netbox/ -ExecStart=/opt/netbox/venv/bin/uwsgi --master --wsgi-file netbox/netbox/wsgi.py uwsgi.ini +ExecStart=/opt/netbox/venv/bin/uwsgi --master --module netbox.wsgi uwsgi.ini +# signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi +ExecReload=kill -HUP $MAINPID +ExecStop=kill -INT $MAINPID +KillSignal=SIGQUIT Restart=on-failure RestartSec=30 -PrivateTmp=true [Install] -WantedBy=multi-user.target +WantedBy=netbox.service From 5a403de0571b57c0da07824c91d703a21231e303 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 19:09:34 +0200 Subject: [PATCH 61/98] __netbox_uwsgi: fix shellcheck Fixed shellcheck cause of the find command. This required a bigger change due to variables and subshell. Now, input is used through a here-document. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 10 +++++++--- type/__netbox_uwsgi/manifest | 3 --- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 324b150..0f85804 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -29,10 +29,14 @@ cat << EOF EOF # special protocol to bind -for param in $(find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print); do +while read -r param; do multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" - socket_changes=yes -done + socket_changes="yes" + +done << INPUT # here-doc cause of SC2031 +$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print ) +INPUT + # else, default bind to if [ -z "$socket_changes" ]; then multi_options "socket" "$__object/parameter/bind-to" diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index 779f93b..3d9c1b4 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,8 +1,5 @@ #!/bin/sh -e -HOST="$__object/parameter/bind-to" -export HOST - # process template mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" From ffba3ae776ac4e4bd67d4812b8073c2e81a69cdd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 21:48:04 +0200 Subject: [PATCH 62/98] __netbox_uwsgi: add --serve-static parameter New option to directly serve static content via uWSGI. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 11 +++++++++++ type/__netbox_uwsgi/man.rst | 11 ++++++++++- type/__netbox_uwsgi/manifest | 6 ++++++ type/__netbox_uwsgi/parameter/boolean | 1 + 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 type/__netbox_uwsgi/parameter/boolean diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 0f85804..4abc59a 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -43,9 +43,20 @@ if [ -z "$socket_changes" ]; then fi +# multi-process settings cat << EOF ; processes and threads processes = $(( 2*cores + 1 )) threads = 2 EOF + + +# optional mapping of static content +if [ "$STATIC_MAP" != "" ]; then + cat << EOF + +; map static content +static-map = /static=/opt/netbox/netbox/static +EOF +fi diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 7501573..f5eff0b 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -38,7 +38,12 @@ scgi-bind BOOLEAN PARAMETERS ------------------ -None. +serve-static + Setup uWSGI to serve the static content, too. This is generally not + recommended for real production setups, as it is the job of the reverse + proxy server, who will thread it as static cachable content. This option + is only recommended for small setups or direct usage of the uWSGI socket + like using it as standalone HTTP server for NetBox. MESSAGES @@ -76,6 +81,10 @@ EXAMPLES --http-bind 0.0.0.0:8080 \ --fastcgi-bind 1.2.3.4:5678 + # as standalone server + __netbox $args + require="__netbox" __netbox_uwsgi --serve-static --http-bind 0.0.0.0:80 + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index 3d9c1b4..bb4f23f 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,5 +1,11 @@ #!/bin/sh -e +# *bind* parameters are directly processed in the gen script +if [ -f "$__object/parameter/serve-static" ]; then + STATIC_MAP="yes" + export STATIC_MAP +fi + # process template mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" diff --git a/type/__netbox_uwsgi/parameter/boolean b/type/__netbox_uwsgi/parameter/boolean new file mode 100644 index 0000000..aa08196 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/boolean @@ -0,0 +1 @@ +serve-static From 090a8f015e86f8dd0385b0b37a234a2538c254da Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:08:37 +0200 Subject: [PATCH 63/98] __netbox_uwsgi: fix uwsgi netbox service file Revert working directory changes as the configuration file still needs to be accessable. An absolute path would work, too, but it is not the preferred way. To still work with the python wsgi application, `--chdir` is used. --- type/__netbox_uwsgi/files/netbox.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index 10ea734..8d41cd0 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -11,9 +11,9 @@ Type=simple User=netbox Group=netbox -WorkingDirectory=/opt/netbox/netbox/ +WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/uwsgi --master --module netbox.wsgi uwsgi.ini +ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi ExecReload=kill -HUP $MAINPID ExecStop=kill -INT $MAINPID From 9d330a91bc5bdbdbe69a4db98ddad0eeaed344cb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:27:42 +0200 Subject: [PATCH 64/98] __netbox_{gunicorn,uwsgi}: add netbox ownership to config file --- type/__netbox_gunicorn/manifest | 3 ++- type/__netbox_uwsgi/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index 25ea452..b72b8fd 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -14,7 +14,8 @@ mkdir "$__object/files" # gunicorn config file __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__object/files/gunicorn.py" + --mode 644 --owner netbox \ + --source "$__object/files/gunicorn.py" # install service file __systemd_unit gunicorn-netbox.service \ diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index bb4f23f..b6c0308 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -13,7 +13,8 @@ mkdir "$__object/files" # uwsgi config file # TODO maybe patching with __key_value cause of .ini ? __file /opt/netbox/uwsgi.ini \ - --mode 644 --source "$__object/files/uwsgi.ini" + --mode 644 --owner netbox \ + --source "$__object/files/uwsgi.ini" # install service file __systemd_unit uwsgi-netbox.service \ From b10f6b71d051377f023fd6e96abbc00ff06e5d5d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:28:48 +0200 Subject: [PATCH 65/98] __netbox: avoid useless generated code-remote Because `set -e` got printed all the time, the type __netbox always had some generated code for the remote side. This line was removed because this is already done by cdist when executing the code-remote script. Rather, the exit-on-error option was set to some scirpts (two ..). --- type/__netbox/gencode-remote | 4 +--- type/__netbox/manifest | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index f200408..9ae6b5b 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -1,6 +1,4 @@ -#!/bin/sh - -echo "set -e" +#!/bin/sh -e old_version="$(cat "$__object/explorer/version")" VERSION=$(cat "$__object/parameter/version") diff --git a/type/__netbox/manifest b/type/__netbox/manifest index cb6062e..5f1762a 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e os=$(cat "$__global/explorer/os") From 3b07a660b3a7dd442e3184600fad25ed27769a5f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:55:43 +0200 Subject: [PATCH 66/98] __netbox_uwsgi: add hint to the --serve-static parameter --- type/__netbox_uwsgi/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index f5eff0b..6826cca 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -45,6 +45,11 @@ serve-static is only recommended for small setups or direct usage of the uWSGI socket like using it as standalone HTTP server for NetBox. + **Hint**: This parameter does not work in junction with the `__netbox` + parameter ``--basepath``. It is because this type does not know the + parameter value and this case is very unlikly to happen; although an + implementation is not difficult. + MESSAGES -------- From c9e4e8d7dce88c3f82a4bbf053b174d88a70a4ce Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 13:44:03 +0200 Subject: [PATCH 67/98] __netbox_{gunicorn,uwsgi}: add state parameter Adds the --state parameter to both types. With it, the transition between both types can be done smothly. --- type/__netbox_gunicorn/gencode-remote | 59 +++++++++++------ type/__netbox_gunicorn/man.rst | 40 ++++++++++-- type/__netbox_gunicorn/manifest | 64 ++++++++++++++----- .../__netbox_gunicorn/parameter/default/state | 1 + type/__netbox_gunicorn/parameter/optional | 1 + type/__netbox_uwsgi/gencode-remote | 58 +++++++++++------ type/__netbox_uwsgi/man.rst | 32 +++++++++- type/__netbox_uwsgi/manifest | 62 +++++++++++++----- type/__netbox_uwsgi/parameter/default/state | 1 + type/__netbox_uwsgi/parameter/optional | 1 + 10 files changed, 244 insertions(+), 75 deletions(-) create mode 100644 type/__netbox_gunicorn/parameter/default/state create mode 100644 type/__netbox_gunicorn/parameter/optional create mode 100644 type/__netbox_uwsgi/parameter/default/state create mode 100644 type/__netbox_uwsgi/parameter/optional diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 163bb1a..0da92f0 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -1,29 +1,50 @@ #!/bin/sh -e -curr_installed="$(cat "$__object/explorer/installed")" -should_installed="$(cat "$__object/explorer/should_installed")" +# control state +state="$(cat "$__object/parameter/state")" -# gunicorn version change -if [ "$curr_installed" != "$should_installed" ]; then - # (re)installing gunicorn - echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" +case "$state" in + # install gunicorn + enabled|disabled) + curr_installed="$(cat "$__object/explorer/installed")" + should_installed="$(cat "$__object/explorer/should_installed")" - do_restart=yes - printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ - >> "$__messages_out" -fi + # gunicorn version change + if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" -# configuration changes -if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then - do_restart=yes - printf "configured\n" >> "$__messages_out" -fi + if [ "$curr_installed" != "" ]; then + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" + else + printf "installed\n" >> "$__messages_out" + fi + do_restart=yes + fi + + # configuration changes + if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" + fi -# restart gunicorn -if [ "$do_restart" ]; then - cat << EOF + # restart gunicorn + if [ "$do_restart" ] && [ "$state" != "disabled" ]; then + cat << EOF # Restart service service gunicorn-netbox restart EOF -fi + fi + ;; + + # uninstall + absent) + # check if installed + if [ -s "$__object/explorer/installed" ]; then + # service already disabled + echo "/opt/netbox/venv/bin/pip3 uninstall -y gunicorn" + printf "uninstalled\n" >> "$__messages_out" + fi +esac diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index c51be0c..1c09d78 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -1,5 +1,5 @@ -cdist-type__netbox_uwsgi(7) -=========================== +cdist-type__netbox_gunicorn(7) +============================== NAME ---- @@ -22,6 +22,20 @@ None. OPTIONAL PARAMETERS ------------------- +state + Represents the state of the Gunciron application. Defaults to ``enabled``. + + enabled + The Gunicorn service is enabled and running. + disabled + The Gunicorn service is installed, but disabled. + absent + The uWSGI service is not installed and all configuration removed. + + This type does not guarantee anything about the running state of the + service. To be sure about the service is stopped or not, use the type + :strong:`cdist-type__systemd_service`\ (7) after this execution. + bind-to The hosts the gunicorn socket should be bind to. Formats are `IP`, `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple @@ -35,13 +49,20 @@ None. MESSAGES -------- -updated $old to $new +installed + The software was installed. + +upgraded $old to $new The version of the gunicorn software was updated from `$old` to `$new`. configured Configuration for gunicorn changed. -In both cases, it restarts the service to use the up-to-date version. +uninstalled + The Gunicorn application was removed. + +In all cases where the application is still present, it restarts the service to +use the up-to-date version. EXAMPLES @@ -59,6 +80,17 @@ EXAMPLES --bind-to 0.0.0.0:8001 \ --bind-to 1.2.3.4:5678 + # replace uwsgi with gunicorn + __netbox $args + require="__netbox" __netbox_uwsgi --state absent + # it should depend on __netbox_uwsgi if they use the same socket + require="__netbox_uwsgi" __netbox_gunicorn --state enabled + + # be sure the service is disabled + __netbox $args + require="__netbox" __netbox_gunicorn --state disabled + require="__netbox_gunicorn" __systemd_service gunicorn-netbox --state stopped + SEE ALSO -------- diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index b72b8fd..2e5ccce 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -1,23 +1,55 @@ #!/bin/sh -e +# __netbox_gunicorn/manifest -HOST="" -while read -r host; do - # shellcheck disable=SC2089 - HOST="$HOST '$host'," -done < "$__object/parameter/bind-to" -# shellcheck disable=SC2090 -export HOST +# Check states +state="" +unit_state="" +param_state="$(cat "$__object/parameter/state")" -# process template -mkdir "$__object/files" -"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" +case "$param_state" in + enabled|disabled) + state="present" + unit_state="$param_state" + ;; + + absent) + state="absent" + unit_state="disabled" + ;; + + *) + # does not exist + printf "The state '%s' does not exist, can't continue!\n" "$param_state" >&2 + exit 2 + ;; +esac + + +if [ "$state" = "present" ]; then + HOST="" + while read -r host; do + # shellcheck disable=SC2089 + HOST="$HOST '$host'," + done < "$__object/parameter/bind-to" + # shellcheck disable=SC2090 + export HOST + + # process template + mkdir "$__object/files" + "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + + # gunicorn config file + __file /opt/netbox/gunicorn.py \ + --mode 644 --owner netbox \ + --source "$__object/files/gunicorn.py" + +else + # absent config file + __file /opt/netbox/gunicorn.py --state absent +fi -# gunicorn config file -__file /opt/netbox/gunicorn.py \ - --mode 644 --owner netbox \ - --source "$__object/files/gunicorn.py" # install service file __systemd_unit gunicorn-netbox.service \ - --source "$__type/files/netbox.service" \ - --enablement-state enabled --restart + --state "$state" --enablement-state "$unit_state" \ + --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_gunicorn/parameter/default/state b/type/__netbox_gunicorn/parameter/default/state new file mode 100644 index 0000000..86981e6 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/state @@ -0,0 +1 @@ +enabled diff --git a/type/__netbox_gunicorn/parameter/optional b/type/__netbox_gunicorn/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional @@ -0,0 +1 @@ +state diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 3127b8b..9b6a603 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -1,29 +1,47 @@ #!/bin/sh -e -# not installed -if ! [ -s "$__object/explorer/installed" ]; then - echo "/opt/netbox/venv/bin/pip3 install uwsgi" - do_restart=yes - printf "installed\n" >> "$__messages_out" +# control state +state="$(cat "$__object/parameter/state")" -# updates available -elif [ -s "$__object/explorer/upgradeable" ]; then - echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" - do_restart=yes - printf "upgraded\n" >> "$__messages_out" -fi +case "$state" in + # install uwsgi + enabled|disabled) + # not installed + if ! [ -s "$__object/explorer/installed" ]; then + echo "/opt/netbox/venv/bin/pip3 install uwsgi" + do_restart=yes + printf "installed\n" >> "$__messages_out" -# changed configuration -if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then - do_restart=yes - printf "configured\n" >> "$__messages_out" -fi + # updates available + elif [ -s "$__object/explorer/upgradeable" ]; then + echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + do_restart=yes + printf "upgraded\n" >> "$__messages_out" + fi + + # changed configuration + if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" + fi -# restart uwsgi -if [ "$do_restart" ]; then - cat << EOF + # restart uwsgi + if [ "$do_restart" ] && [ "$state" != "disabled" ]; then + cat << EOF # Restart service service uwsgi-netbox restart EOF -fi + fi + ;; + + # uninstall + absent) + # check if installed + if [ -s "$__object/explorer/installed" ]; then + # service already disabled + echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi" + printf "uninstalled\n" >> "$__messages_out" + fi + ;; +esac diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 6826cca..3fb8515 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -23,6 +23,21 @@ None. OPTIONAL PARAMETERS ------------------- +state + Represents the state of the uWSGI application. Defaults to ``enabled``. + + enabled + The uWSGI service is enabled and running. + disabled + The uWSGI service is installed, but disabled. + absent + The uWSGI service is not installed and all configuration removed. + + This type does not guarantee anything about the running state of the + service. To be sure about the service is stopped or not, use the type + :strong:`cdist-type__systemd_service`\ (7) after this execution. + + bind-to The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. Defaults to ``127.0.0.1:3031``. Can be set multiple times. @@ -62,7 +77,11 @@ upgraded configured The uwsgi configuration got updated. -In all cases, it restarts the service to use the up-to-date version. +uninstalled + The uWSGI application was removed. + +In all cases where the application is still present, it restarts the service to +use the up-to-date version. EXAMPLES @@ -90,6 +109,17 @@ EXAMPLES __netbox $args require="__netbox" __netbox_uwsgi --serve-static --http-bind 0.0.0.0:80 + # replace gunicorn with uwsgi + __netbox $args + require="__netbox" __netbox_gunicorn --state absent + # it should depend on __netbox_gunicorn if they use the same socket + require="__netbox_gunicorn" __netbox_uwsgi --state enabled + + # be sure the service is disabled + __netbox $args + require="__netbox" __netbox_uwsgi --state disabled + require="__netbox_uwsgi" __systemd_service uwsgi-netbox --state stopped + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index b6c0308..c5885c9 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,22 +1,54 @@ #!/bin/sh -e +# __netbox_uwsgi/manifest -# *bind* parameters are directly processed in the gen script -if [ -f "$__object/parameter/serve-static" ]; then - STATIC_MAP="yes" - export STATIC_MAP +# Check states +state="" +unit_state="" +param_state="$(cat "$__object/parameter/state")" + +case "$param_state" in + enabled|disabled) + state="present" + unit_state="$param_state" + ;; + + absent) + state="absent" + unit_state="disabled" + ;; + + *) + # does not exist + printf "The state '%s' does not exist, can't continue!\n" "$param_state" >&2 + exit 2 + ;; +esac + + +if [ "$state" = "present" ]; then + # *bind* parameters are directly processed in the gen script + if [ -f "$__object/parameter/serve-static" ]; then + STATIC_MAP="yes" + export STATIC_MAP + fi + + # process template + mkdir "$__object/files" + "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" + + # uwsgi config file + # TODO maybe patching with __key_value cause of .ini ? + __file /opt/netbox/uwsgi.ini \ + --mode 644 --owner netbox \ + --source "$__object/files/uwsgi.ini" + +else + # absent config file + __file /opt/netbox/uwsgi.ini --state absent fi -# process template -mkdir "$__object/files" -"$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" - -# uwsgi config file -# TODO maybe patching with __key_value cause of .ini ? -__file /opt/netbox/uwsgi.ini \ - --mode 644 --owner netbox \ - --source "$__object/files/uwsgi.ini" # install service file __systemd_unit uwsgi-netbox.service \ - --source "$__type/files/netbox.service" \ - --enablement-state enabled --restart + --state "$state" --enablement-state "$unit_state" \ + --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/parameter/default/state b/type/__netbox_uwsgi/parameter/default/state new file mode 100644 index 0000000..86981e6 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/state @@ -0,0 +1 @@ +enabled diff --git a/type/__netbox_uwsgi/parameter/optional b/type/__netbox_uwsgi/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__netbox_uwsgi/parameter/optional @@ -0,0 +1 @@ +state From 0afc7136f8e4f304f179965f229ed368ce73abb6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 18:39:07 +0200 Subject: [PATCH 68/98] __netbox_uwsgi: fix if only --bind-to given If the here-document is empty because no file could be found, an empty line will be read. The new line of code will skip in those cases. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 4abc59a..7835a3e 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -30,6 +30,8 @@ EOF # special protocol to bind while read -r param; do + if [ -z "$param" ]; then continue; fi # ignore empty lines from the here-doc + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" socket_changes="yes" From eed3515424668b8684058b0d02368746ee5f5c06 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Sep 2020 17:10:59 +0200 Subject: [PATCH 69/98] __netbox: add --ldap-staff-group parameter Required to get "admin access". --- type/__netbox/files/ldap_config.py.sh | 15 +++++++++++---- type/__netbox/man.rst | 4 ++++ type/__netbox/manifest | 4 ++++ type/__netbox/parameter/optional | 1 + 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index a49d800..4e6b099 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -53,6 +53,8 @@ AUTH_LDAP_GROUP_TYPE = PosixGroupType() # Mirror LDAP group assignments. AUTH_LDAP_MIRROR_GROUPS = True +# For more granular permissions, map LDAP groups to Django groups. +AUTH_LDAP_FIND_GROUP_PERMS = True EOF if [ "$LDAP_REQUIRE_GROUP" != "" ]; then @@ -63,13 +65,18 @@ AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" EOF fi - if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then - cat << EOF + cat << EOF # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { - "is_superuser": "$LDAP_SUPERUSER_GROUP", -} EOF + # superuser + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + echo " \"is_superuser\": \"$LDAP_SUPERUSER_GROUP\"," fi + # staff user + if [ "$LDAP_STAFF_GROUP" != "" ]; then + echo " \"is_staff\": \"$LDAP_STAFF_GROUP\"," + fi + echo "}" fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index a3f283c..bc4bbc9 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -78,6 +78,10 @@ ldap-group-base ldap-require-group Group required to login. +ldap-staff-group + Make members of this group to "staff". This gives the users "Admin Access", + which means access to the "NetBox Administration" site. + ldap-superuser-group Make members of this groups superusers. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 5f1762a..3681ea0 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -89,6 +89,10 @@ if [ -f "$__object/parameter/ldap-superuser-group" ]; then LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") export LDAP_SUPERUSER_GROUP fi +if [ -f "$__object/parameter/ldap-staff-group" ]; then + LDAP_STAFF_GROUP="$(cat "$__object/parameter/ldap-staff-group")" + export LDAP_STAFF_GROUP +fi # export if base ldap parameters are used export USE_LDAP diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 47722e6..9495f7a 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -7,6 +7,7 @@ ldap-bind-password ldap-user-base ldap-group-base ldap-require-group +ldap-staff-group ldap-superuser-group redis-host redis-port From 3389752decfb48975f539b1702d37fe07d4e7755 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Sep 2020 17:23:22 +0200 Subject: [PATCH 70/98] __netbox: fix detection of the secret-key to use Mixed up a parameter. Should be correct now. --- type/__netbox/manifest | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 3681ea0..06d0c77 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -47,13 +47,13 @@ export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") elif [ -s "$__object/explorer/secretkey" ]; then + # take the key that is already used + SECRET_KEY="$(cat "$__object/explorer/secretkey")" +else # Can be done over netbox/generate_secret_key.py too, but it's to # complicated with the variable setup (can't generated right now!). # Generates a 50-character long key (without ' cause of python quotes) SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" -else - # take the key that is already used - SECRET_KEY="$(cat "$__object/explorer/secretkey")" fi export SECRET_KEY From 398a3da10eeaf67caa042b2a99a1a3e652b31e0f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 18:10:15 +0200 Subject: [PATCH 71/98] __netbox: fix gerneration random generated secret key Because `/dev/random` was used, the `cdist config` could hang a long time to get real random values. The pseudo-generated values through `/dev/urandom` are fully enought for the secret key. --- type/__netbox/manifest | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 06d0c77..4ad7901 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -50,10 +50,11 @@ elif [ -s "$__object/explorer/secretkey" ]; then # take the key that is already used SECRET_KEY="$(cat "$__object/explorer/secretkey")" else - # Can be done over netbox/generate_secret_key.py too, but it's to - # complicated with the variable setup (can't generated right now!). + # Can be done over netbox/generate_secret_key.py too, but it can't be + # generated right now where it's required (only if it's preloaded for + # this type to execute it now). # Generates a 50-character long key (without ' cause of python quotes) - SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" + SECRET_KEY="$(tr -cd '[:graph:]' < /dev/urandom | tr -d \' | head -c50)" fi export SECRET_KEY From 549feb87f96518a38ab7d3368bd3f2a2ed552d0d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 19:20:22 +0200 Subject: [PATCH 72/98] __netbox: fix --update-notify Template script for the `configuration.py` interpreted the boolean flag incorrectly and did the thing the user do not wanted. --- type/__netbox/files/configuration.py.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index ea73133..06a1c86 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -272,12 +272,12 @@ EOF if [ "$UPDATE_CHECK" != "" ]; then cat << EOF -RELEASE_CHECK_URL = None +RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' EOF else cat << EOF -RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' +RELEASE_CHECK_URL = None EOF fi From 5d437839f68da0c29787172ea3daa3170926f8d2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 19:52:03 +0200 Subject: [PATCH 73/98] __netbox: replaced secret key character set The used character set should now only contain characters used by the `netbox/generate_secret_key.py` program. Finially got correct escaping. --- type/__netbox/manifest | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 4ad7901..d754ae6 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -53,8 +53,9 @@ else # Can be done over netbox/generate_secret_key.py too, but it can't be # generated right now where it's required (only if it's preloaded for # this type to execute it now). - # Generates a 50-character long key (without ' cause of python quotes) - SECRET_KEY="$(tr -cd '[:graph:]' < /dev/urandom | tr -d \' | head -c50)" + # Generates a 50-character long key with the same character set like + # the helper script. Must escape the '-' to be no character range. + SECRET_KEY="$(tr -cd '!@#$%^&*(\-_=+)[:alnum:]' < /dev/urandom | head -c50)" fi export SECRET_KEY From dd167f075d25ecec4f1aca461658d2b93ff1ae8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 9 Sep 2020 19:08:46 +0200 Subject: [PATCH 74/98] __netbox*: fix service restart order Cause of corrupt databases if the services are restarted incorrectly, the order and dependencies are adjusted. Now, the `netbox-rq` service will be included in restarts of `netbox` and required for the WSGI servers that it must running. For these changes, the restart command of `__netbox` was adjusted. The other ones where edited too, to use the same command. All services now require redis and postgresql to be started before them to prevent any start order issues. If someone asked for what the RQ worker is required, see here: https://netbox.readthedocs.io/en/stable/additional-features/webhooks/#webhook-processing --- type/__netbox/files/netbox-rq.service | 7 ++++-- type/__netbox/files/netbox.service | 2 +- type/__netbox/gencode-remote | 7 +++--- type/__netbox/man.rst | 24 ++++++++++++++++++++- type/__netbox_gunicorn/files/netbox.service | 4 +++- type/__netbox_gunicorn/gencode-remote | 2 +- type/__netbox_uwsgi/files/netbox.service | 4 +++- type/__netbox_uwsgi/gencode-remote | 2 +- 8 files changed, 41 insertions(+), 11 deletions(-) diff --git a/type/__netbox/files/netbox-rq.service b/type/__netbox/files/netbox-rq.service index 77d7091..330e675 100644 --- a/type/__netbox/files/netbox-rq.service +++ b/type/__netbox/files/netbox-rq.service @@ -1,8 +1,11 @@ [Unit] Description=NetBox Request Queue Worker Documentation=https://netbox.readthedocs.io/en/stable/ -After=network-online.target -Wants=network-online.target +PartOf=netbox.service +Wants=network.target +After=netbox.service +After=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service index 738e178..68010e9 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox/files/netbox.service @@ -1,8 +1,8 @@ [Unit] Description=NetBox Service Wrapper Documentation=https://netbox.readthedocs.io/en/stable/ -After=network.target Wants=network.target +After=network.target [Service] Type=oneshot diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 9ae6b5b..0370949 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -105,9 +105,10 @@ fi # check for changes if [ "$changes" = "yes" ]; then + # to avoid database corruption at config changes, both services must be + # stopped at the same time (noted in the manual, too). cat << EOF -# Restart service. -service netbox restart -service netbox-rq restart +# Restart service. All required services are included with netbox.service. +systemctl restart netbox EOF fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index bc4bbc9..3330b1b 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -12,7 +12,11 @@ This (singleton) type installs and configures a NetBox instance, a web application to help manage and document computer networks. It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. -It setup systemd unit files for the services `netbox` and `netbox-rq`. To +It setup systemd unit files for the services `netbox` and `netbox-rq`. The +`netbox` service only wrap all netbox related services, e.g. restarting and +so one will be delegated to all related services. + +The application is still not accessable because WSGI server is required. To access the application through WSGI, uWSGI or Gunicorn can be used. The setup can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. @@ -229,6 +233,24 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. +If you change a configuration, the database may go corrupt if two instances of +the application are running with different configurations at the same time. +This most commonly happens when the WSGI server and RQ-worker restarts after a +configuration change. This occours in the following case for example: + +.. code-block:: sh + + systemctl restart gunicorn-netbox # WSGI-server already online with new + # configuration after this command. + systemctl restart netbox-rq # RQ-Worker still worked with the old + # configuration till here. + +This type handles the restart of both services correctly to avoid such database +corruptions. To safely manual restart the whole netbox instance manual, simply +restart all services in one ``systemctl restart netbox`` command, as it ensures +that first all services are shut down before starting one of them. The service +``netbox`` wraps all required services that are available. + SEE ALSO -------- diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 55f3549..dcf1a66 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -2,9 +2,11 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service +Requires=netbox-rq.service +Wants=network.target After=netbox.service After=network.target -Wants=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 0da92f0..4fae788 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -34,7 +34,7 @@ case "$state" in if [ "$do_restart" ] && [ "$state" != "disabled" ]; then cat << EOF # Restart service -service gunicorn-netbox restart +systemctl restart gunicorn-netbox EOF fi ;; diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index 8d41cd0..cd06e82 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -2,9 +2,11 @@ Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service +Requires=netbox-rq.service +Wants=network.target After=netbox.service After=network.target -Wants=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 9b6a603..c219643 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -30,7 +30,7 @@ case "$state" in if [ "$do_restart" ] && [ "$state" != "disabled" ]; then cat << EOF # Restart service -service uwsgi-netbox restart +systemctl restart uwsgi-netbox EOF fi ;; From e800f42a6d95b9378516252d53944383f673653d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 9 Sep 2020 20:33:20 +0200 Subject: [PATCH 75/98] __netbox: consistency with __systemd_unit --restart This commit brings consistency into the --restart parameter for systemd units. All units except the netbox wrapper service will be restarted on unit change. --- type/__netbox/manifest | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index d754ae6..f8ed5fc 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -213,8 +213,10 @@ SECRET # Upload systemd unit for worker and wsgi service -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done +# does not restart netbox on change cause it only restart all other services +__systemd_unit netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled --restart From b55186544f4cfeb9f371c4c0ff429bdc2a63c6cc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 28 Sep 2020 19:49:29 +0200 Subject: [PATCH 76/98] __netbox: adding seperate /opt/netbox/cdist/ directory Adding a place to keep all configuration stuff for cdist seperated from the installation directory. For safety reasons. --- type/__netbox/explorer/secretkey | 2 +- type/__netbox/explorer/version | 2 +- type/__netbox/gencode-remote | 26 +++++++------------------- type/__netbox/manifest | 17 ++++++++++------- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/type/__netbox/explorer/secretkey b/type/__netbox/explorer/secretkey index 9f0331c..7cce279 100755 --- a/type/__netbox/explorer/secretkey +++ b/type/__netbox/explorer/secretkey @@ -2,7 +2,7 @@ # Explorer will output the key if he exists. -secretkey="/opt/netbox/.secretkey" +secretkey="/opt/netbox/cdist/secretkey" if [ -f "$secretkey" ]; then cat "$secretkey" fi diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version index 5231e16..ee3dde8 100755 --- a/type/__netbox/explorer/version +++ b/type/__netbox/explorer/version @@ -1,5 +1,5 @@ #!/bin/sh -e # output version if exist -version_path="/opt/netbox/netbox/cdist/version" +version_path="/opt/netbox/cdist/version" if [ -f "$version_path" ]; then cat "$version_path"; fi diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 0370949..aa3864e 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -24,16 +24,13 @@ cd "\$tmpdir" curl -sS -L '$url' > '$archive' tar xf '$archive' -# Save cdist-upload configuration file. -cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" -cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" - EOF # Stop everything in the pyenv to update cat << EOF # Try to kill everything in the venv -systemctl stop netbox gunicorn-netbox uwsgi-netbox || true +systemctl -q --wait stop netbox gunicorn-netbox uwsgi-netbox || true +# don't know if this is required since using --wait ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true EOF @@ -42,11 +39,10 @@ EOF # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' -mkdir '$install_dir/cdist' cp '$src/requirements.txt' /opt/netbox/ # backup dependency info -cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' -cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' +ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. # forcing python3 to be sure (till python4 gets released ..) @@ -79,24 +75,16 @@ cd / rm -r "\$tmpdir" # Save version after successful installation -printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' +printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version EOF # meta printf "installed %s\n" "$VERSION" >> "$__messages_out" changes=yes +fi # check if configuration changed -elif grep -q "^__file/opt/netbox/" "$__messages_in"; then - # check if coping is required - if grep -q "^__file/opt/netbox/netbox/cdist/" "$__messages_in"; then - cat << EOF -# Copy configuration -cp '$install_dir/cdist/configuration.py' '$install_dir/netbox/configuration.py' -cp '$install_dir/cdist/ldap_config.py' '$install_dir/netbox/ldap_config.py' -EOF - fi - +if grep -q "^__file/opt/netbox/" "$__messages_in"; then # meta printf "configured\n" >> "$__messages_out" changes=yes diff --git a/type/__netbox/manifest b/type/__netbox/manifest index f8ed5fc..89b89ac 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -194,20 +194,23 @@ mkdir -p "$__object/files" "$__type/files/configuration.py.sh" > "$__object/files/configuration.py" "$__type/files/ldap_config.py.sh" > "$__object/files/ldap_config.py" -require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents -require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ +require="__user/netbox" __directory /opt/netbox/cdist +require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/configuration.py --mode 640 --owner netbox \ --source "$__object/files/configuration.py" if [ -f "$__object/parameter/ldap-server" ]; then - require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ --source "$__object/files/ldap_config.py" +else + require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/ldap_config.py --state absent fi # save secret -require="__user/netbox" __file /opt/netbox/.secretkey --mode 400 \ - --owner netbox --source - << SECRET +require="__directory/opt/netbox/cdist" __file /opt/netbox/cdist/secretkey \ + --mode 400 --owner netbox --source - << SECRET $SECRET_KEY SECRET From 050812305b282e65be51c3902dba691c4718ca4d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 28 Sep 2020 20:04:22 +0200 Subject: [PATCH 77/98] __netbox: add mange.py calls recommended from upstream Calls where added because there are used upstream in the `upgrade.sh`-Script, too. Upgrade-Script: https://github.com/netbox-community/netbox/blob/develop/upgrade.sh --- type/__netbox/gencode-remote | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index aa3864e..d7bc219 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -64,11 +64,17 @@ EOF # Set final permissions. chown -R netbox /opt/netbox +# NetBox manage scripts # Run database migrations. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate - # Generate static assets. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collectstatic --no-input +# Delete any stale content types +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py remove_stale_contenttypes --no-input +# Delete any expired user sessions +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py clearsessions +# Clear all cached data +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py invalidate all # Remove temporary working directory. cd / From 14f7cf88396f8a2870b4edfdb12f9b8448ac48b8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 4 Oct 2020 17:40:01 +0200 Subject: [PATCH 78/98] __netbox: set --data-root default value The default value is set to preserve all data saved by netbox. As explained in the manpage, it sets the directory for all data directories to `~netbox/data/` (would resolve to `/opt/netbox/data/`), so upgrades will not remove this data. --- type/__netbox/man.rst | 17 ++++++++++------- type/__netbox/manifest | 11 +++++------ type/__netbox/parameter/default/data-root | 1 + 3 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 type/__netbox/parameter/default/data-root diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 3330b1b..6d50268 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -131,7 +131,7 @@ data-root This parameter set's the media, reports and scripts root to subdirectories of the given directory. Values can be overwritten by special parameters like `--media-root` for example. Use this option if you want to store persistant - data of netbox on an other partition. A trailing shlash is not needed. + data of netbox on an other partition. A trailing slash is not needed. The data directories have following predefined sub-directory names: @@ -142,23 +142,26 @@ data-root scripts root: ``$data_root/scripts`` + To preserve all data from installation upgrades - which just replace the + installation directory - the data will be kept in the netbox home directory + rather than the installation directory by default (``/opt/netbox/data/``). + This way, no data will be deleted after the installation directory + replacement because it remains outside of the installation directory. + media-root The file path to where media files (like image attachments) are stored. Change this path if you require to store data on an other partiotion. - A trailing slash is not needed. By default, it will be stored into the - installation directory (``/opt/netbox/netbox/netbox/media``). + A trailing slash is not needed. Defaults to ``$data_root/media``. reports-root The file path of where custom reports are kept. Change this path if you require to store data on an other partition. A trailing slash is not - needed. By default, it will be stored into the installation directory - (``/opt/netbox/netbox/netbox/reports``). + needed. Defaults to ``$data_root/reports``. scripts-root The file path of where custom scripts are kept. Change this path if you require to store data on an other partition. A trailing slash is not - needed. By default, it will be stored into the installation directory - (``/opt/netbox/netbox/netbox/scripts``). + needed. Defaults to ``$data_root/scripts``. BOOLEAN PARAMETERS diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 89b89ac..fbcf253 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -161,12 +161,11 @@ else fi export LOGIN_REQUIRED -if [ -f "$__object/parameter/data-root" ]; then - data_root="$(cat "$__object/parameter/data-root")" - MEDIA_ROOT="$data_root/media" - REPORTS_ROOT="$data_root/reports" - SCRIPTS_ROOT="$data_root/scripts" -fi +data_root="$(cat "$__object/parameter/data-root")" +MEDIA_ROOT="$data_root/media" +REPORTS_ROOT="$data_root/reports" +SCRIPTS_ROOT="$data_root/scripts" + if [ -f "$__object/parameter/media-root" ]; then MEDIA_ROOT="$(cat "$__object/parameter/media-root")" fi diff --git a/type/__netbox/parameter/default/data-root b/type/__netbox/parameter/default/data-root new file mode 100644 index 0000000..45faab0 --- /dev/null +++ b/type/__netbox/parameter/default/data-root @@ -0,0 +1 @@ +/opt/netbox/data From 0c85b2d3fd3e9dee6a79091064e4fb7078adcdfb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 5 Oct 2020 18:26:47 +0200 Subject: [PATCH 79/98] __netbox: uninstall not anymore required packages This code tries to remove packages not needed anymore. As it presumably not removing dependencies, it's not that perfect pice of script. --- type/__netbox/gencode-remote | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d7bc219..1cd4939 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -36,18 +36,35 @@ ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || tr EOF cat << EOF +# backup requirement files +if [ -f /opt/netbox/requirements.txt]; then + cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt +else + # preseve file-not-found errors and warnings + touch /opt/netbox/old-requirements.txt +fi +cp '$src/requirements.txt' /opt/netbox/ + # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' -cp '$src/requirements.txt' /opt/netbox/ # backup dependency info ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' + # Setup & enter python virtualenv. # forcing python3 to be sure (till python4 gets released ..) virtualenv -p python3 /opt/netbox/venv +# Uninstall packages not required anymore +# if versions not shortend, they will be ignored by pip, but not by comm +# all of this could be done with grep, too, but it's still must be shortend with awk +awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" +awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" +comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ + | xargs /opt/netbox/venv/bin/pip3 uninstall -y + # Install python dependencies. # avoid gunicorn, because it will be done in an other type grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ @@ -57,7 +74,7 @@ EOF if [ -f "$__object/parameter/ldap-server" ]; then echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" else - echo "/opt/netbox/venv/bin/pip3 uninstall django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 uninstall -y django-auth-ldap" fi cat << EOF From 8f1b56026ca75b6a9d66dd40d40c239c34f6927b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 5 Oct 2020 19:02:05 +0200 Subject: [PATCH 80/98] __netbox: force link creation for config files This adds the force flag to `ln` to avoid aborts cause the link already exists and so forth. It also adds robustness to the execution. --- type/__netbox/gencode-remote | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 1cd4939..3b4e05c 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -48,9 +48,9 @@ cp '$src/requirements.txt' /opt/netbox/ # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' - -ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' -ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' +# force links to the cdist directory +ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. From 0b3bc1453073de6e45101e9386b5de1e8f93227a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 6 Oct 2020 17:29:31 +0200 Subject: [PATCH 81/98] __netbox: adjusted command flags Mostly added -q to the pip calls. Also forced the temp directory removal. --- type/__netbox/gencode-remote | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 3b4e05c..eb3bcd4 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -63,18 +63,18 @@ virtualenv -p python3 /opt/netbox/venv awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ - | xargs /opt/netbox/venv/bin/pip3 uninstall -y + | xargs /opt/netbox/venv/bin/pip3 uninstall -qy # Install python dependencies. # avoid gunicorn, because it will be done in an other type grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ - | xargs /opt/netbox/venv/bin/pip3 install + | xargs /opt/netbox/venv/bin/pip3 install -q EOF if [ -f "$__object/parameter/ldap-server" ]; then - echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 install -q django-auth-ldap" else - echo "/opt/netbox/venv/bin/pip3 uninstall -y django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 uninstall -qy django-auth-ldap" fi cat << EOF @@ -95,7 +95,7 @@ sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py invalid # Remove temporary working directory. cd / -rm -r "\$tmpdir" +rm -rf "\$tmpdir" # Save version after successful installation printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version From 3b780c4794b1c9df9e3f6582e7ad17ad1cfd0402 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 8 Oct 2020 19:54:48 +0200 Subject: [PATCH 82/98] __netbox: fixup small bugs and venv process killing Changed flag (force to ignore a non-existant directory), typo and swapped arguments are done. Also, the process to stop all processes from the virtal environment has changed: Now, it stops all potential services and ignore errors (because a service doesn't exist). After that, it sends a kill signal to all processes and then gracefully wait since there is no option to do that with systemd. --- type/__netbox/gencode-remote | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index eb3bcd4..1ef78cd 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -28,16 +28,24 @@ EOF # Stop everything in the pyenv to update cat << EOF -# Try to kill everything in the venv -systemctl -q --wait stop netbox gunicorn-netbox uwsgi-netbox || true -# don't know if this is required since using --wait -ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true +# Try to stop everything in the venv (ignore non-existant services) +systemctl -q stop netbox gunicorn-netbox uwsgi-netbox || true + +# kill and poll till all venv processes end +# at least if they are called with full path in the cmd (like in the services) +pids="\$( ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' )" +if [ "\$pids" ]; then + kill \$pids + while ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | grep -q . ; do + sleep 0.5 + done +fi EOF cat << EOF # backup requirement files -if [ -f /opt/netbox/requirements.txt]; then +if [ -f /opt/netbox/requirements.txt ]; then cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt else # preseve file-not-found errors and warnings @@ -46,7 +54,7 @@ fi cp '$src/requirements.txt' /opt/netbox/ # Deploy sources and restore configuration. -rm -r '$install_dir' +rm -rf '$install_dir' cp -r '$src/netbox' '$install_dir' # force links to the cdist directory ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' @@ -62,7 +70,7 @@ virtualenv -p python3 /opt/netbox/venv # all of this could be done with grep, too, but it's still must be shortend with awk awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" -comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ +comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" \ | xargs /opt/netbox/venv/bin/pip3 uninstall -qy # Install python dependencies. From 13e97d171bb700f577cdcfb03dde81a193b0c10c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:39:19 +0200 Subject: [PATCH 83/98] __netbox*: added systemd socket support The Gunicorn type now supports systemd sockets only. With uWSGI, you can choose between it and the native sockets based on the parameters chosen. This is done because it could not be implemented to have multiple protocols with the systemd sockets (so you may choose). The systemd socket unit file is generally available, so both types use the same script to generate the socket unit file. --- type/__netbox/files/netbox.socket.sh | 33 ++++++++++ type/__netbox_gunicorn/files/gunicorn.py.sh | 2 +- type/__netbox_gunicorn/files/netbox.service | 3 +- type/__netbox_gunicorn/files/netbox.socket.sh | 1 + type/__netbox_gunicorn/man.rst | 8 ++- type/__netbox_gunicorn/manifest | 22 +++---- type/__netbox_uwsgi/explorer/bind-capability | 12 ++++ .../{netbox.service => netbox.service.sh} | 18 +++++- type/__netbox_uwsgi/files/netbox.socket.sh | 1 + type/__netbox_uwsgi/files/uwsgi.ini.sh | 24 ++++---- type/__netbox_uwsgi/gencode-remote | 60 ++++++++++++++++++- type/__netbox_uwsgi/man.rst | 33 +++++++++- type/__netbox_uwsgi/manifest | 38 +++++++++++- .../__netbox_uwsgi/parameter/default/protocol | 1 + type/__netbox_uwsgi/parameter/optional | 1 + 15 files changed, 218 insertions(+), 39 deletions(-) create mode 100755 type/__netbox/files/netbox.socket.sh create mode 120000 type/__netbox_gunicorn/files/netbox.socket.sh create mode 100755 type/__netbox_uwsgi/explorer/bind-capability rename type/__netbox_uwsgi/files/{netbox.service => netbox.service.sh} (70%) mode change 100644 => 100755 create mode 120000 type/__netbox_uwsgi/files/netbox.socket.sh create mode 100644 type/__netbox_uwsgi/parameter/default/protocol diff --git a/type/__netbox/files/netbox.socket.sh b/type/__netbox/files/netbox.socket.sh new file mode 100755 index 0000000..03e3d44 --- /dev/null +++ b/type/__netbox/files/netbox.socket.sh @@ -0,0 +1,33 @@ +#!/bin/sh -e +# __netbox/files/netbox.socket.sh + +# This is shared between all WSGI-server types. + +# Arguments: +# 1: File which list all sockets to listen on (sepearated by \n) + +if [ $# -ne 1 ]; then + printf "netbox.socket.sh: argument \$1 missing or too much given!\n" >&2 + exit 1 +fi + + +cat << UNIT +[Unit] +Description=Socket for NetBox via $TYPE + +[Socket] +UNIT + +# read all sockets to listen to +while read line; do + printf "ListenStream=%s\n" "$line" +done < "$1" + +cat << UNIT +SocketUser=netbox +SocketGroup=www-data + +[Install] +WantedBy=sockets.target +UNIT diff --git a/type/__netbox_gunicorn/files/gunicorn.py.sh b/type/__netbox_gunicorn/files/gunicorn.py.sh index 862ba06..c1e6ee5 100755 --- a/type/__netbox_gunicorn/files/gunicorn.py.sh +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -13,7 +13,7 @@ cores="$(cat "$__explorer/cpu_cores")" cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = [$HOST ] +#bind = done via systemd socket 'gunicorn-netbox.socket' # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index dcf1a66..28b6b45 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -3,13 +3,14 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service Requires=netbox-rq.service +Requires=gunicorn-netbox.socket Wants=network.target After=netbox.service After=network.target After=redis-server.service postgresql.service [Service] -Type=simple +Type=notify User=netbox Group=netbox diff --git a/type/__netbox_gunicorn/files/netbox.socket.sh b/type/__netbox_gunicorn/files/netbox.socket.sh new file mode 120000 index 0000000..28ce920 --- /dev/null +++ b/type/__netbox_gunicorn/files/netbox.socket.sh @@ -0,0 +1 @@ +../../__netbox/files/netbox.socket.sh \ No newline at end of file diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 1c09d78..8860294 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -14,6 +14,9 @@ sockets. Static content must be served independent of Gunicorn. The Gunicorn daemon is available as the `gunicorn-netbox` systemd service, but also available via the `netbox` wrapper service. +It will use systemd socket activation to listen to the given sockets. This +should allow to bind to privileaged ports (all below 1024) and hot reloads. + REQUIRED PARAMETERS ------------------- @@ -38,8 +41,9 @@ state bind-to The hosts the gunicorn socket should be bind to. Formats are `IP`, - `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple - times. Defaults to ``127.0.0.1:8001``. + `IP:PORT`, `PATH` or anything other that systemd socket units will + understand as stream. Parameter can be set multiple times. Defaults + to ``127.0.0.1:8001``. BOOLEAN PARAMETERS diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index 2e5ccce..5748e9d 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -25,17 +25,10 @@ case "$param_state" in esac -if [ "$state" = "present" ]; then - HOST="" - while read -r host; do - # shellcheck disable=SC2089 - HOST="$HOST '$host'," - done < "$__object/parameter/bind-to" - # shellcheck disable=SC2090 - export HOST +mkdir "$__object/files" +if [ "$state" = "present" ]; then # process template - mkdir "$__object/files" "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" # gunicorn config file @@ -49,7 +42,16 @@ else fi -# install service file +TYPE="Gunicorn" +export TYPE + +"$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \ + > "$__object/files/netbox.socket" + +# install systemd files +__systemd_unit gunicorn-netbox.socket \ + --state "$state" --enablement-state "$unit_state" \ + --source "$__object/files/netbox.socket" --restart __systemd_unit gunicorn-netbox.service \ --state "$state" --enablement-state "$unit_state" \ --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/explorer/bind-capability b/type/__netbox_uwsgi/explorer/bind-capability new file mode 100755 index 0000000..c5c0365 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/bind-capability @@ -0,0 +1,12 @@ +#!/bin/sh -e +# explorer/bind-capablility + +# Checks if the uWSGI binary have the capability to bind to privileaged ports +# as a non-root user. It's required if no systemd sockets are used (cause of +# the use of multiple protocols etc.) + +binary="/opt/netbox/venv/bin/uwsgi" +# -v verifies if capability is set +if setcap -q -v CAP_NET_BIND_SERVICE+ep "$binary"; then + echo set +fi diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service.sh old mode 100644 new mode 100755 similarity index 70% rename from type/__netbox_uwsgi/files/netbox.service rename to type/__netbox_uwsgi/files/netbox.service.sh index cd06e82..3705769 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service.sh @@ -1,15 +1,26 @@ +#!/bin/sh -e + +cat << EOF [Unit] Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service Requires=netbox-rq.service +EOF + +# Add dependency to own socket +if [ "$(cat "$__object/files/systemd_socket")" = "yes" ]; then + echo "Requires=uwsgi-netbox.socket" +fi + +cat << EOF Wants=network.target After=netbox.service After=network.target After=redis-server.service postgresql.service [Service] -Type=simple +Type=notify User=netbox Group=netbox @@ -17,8 +28,8 @@ WorkingDirectory=/opt/netbox ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi -ExecReload=kill -HUP $MAINPID -ExecStop=kill -INT $MAINPID +ExecReload=kill -HUP \$MAINPID +ExecStop=kill -INT \$MAINPID KillSignal=SIGQUIT Restart=on-failure @@ -26,3 +37,4 @@ RestartSec=30 [Install] WantedBy=netbox.service +EOF diff --git a/type/__netbox_uwsgi/files/netbox.socket.sh b/type/__netbox_uwsgi/files/netbox.socket.sh new file mode 120000 index 0000000..28ce920 --- /dev/null +++ b/type/__netbox_uwsgi/files/netbox.socket.sh @@ -0,0 +1 @@ +../../__netbox/files/netbox.socket.sh \ No newline at end of file diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 7835a3e..4bae613 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -28,20 +28,16 @@ cat << EOF ; socket(s) to bind EOF -# special protocol to bind -while read -r param; do - if [ -z "$param" ]; then continue; fi # ignore empty lines from the here-doc - - multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" - socket_changes="yes" - -done << INPUT # here-doc cause of SC2031 -$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print ) -INPUT - -# else, default bind to -if [ -z "$socket_changes" ]; then - multi_options "socket" "$__object/parameter/bind-to" +if [ "$SYSTEMD_SOCKET" != "yes" ]; then + # special protocol to bind + find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print \ + | while read -r param; do + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" + done +else + # else, systemd will offer socket + echo "; sockets managed via 'uwsgi-netbox.socket'" + printf "protocol = %s\n" "$PROTOCOL" fi diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index c219643..7c3b826 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -3,18 +3,67 @@ # control state state="$(cat "$__object/parameter/state")" +# Set capabilities to aquire privileaged ports as netbox user. Two modes are +# available to efficiently set capabilites. Assumes libcap-bin is installed as +# default on debian systems. +# +# Arguments: +# 1: mode to detect if capabilites are required to set ('set' or 'correct') +set_bind_cap() { + cap_mode="" # reset variable from the execution before + + # check if capabilites are required after given mode + case "$1" in + # assumes capabilites are not set (cause of new binaries) + set) + if [ "$SYSTEMD_SOCKET" != "yes" ]; then + cap_mode="+ep" + fi + ;; + + # check if capabilities have changed + correct) + if [ -s "$__object/explorer/bind-capability" ]; then + # capabilites are set + if [ "$SYSTEMD_SOCKET" = "yes" ]; then + cap_mode="-ep" # unset + fi + else + # capabilities are unset + if [ "$SYSTEMD_SOCKET" != "yes" ]; then + cap_mode="+ep" # set + fi + fi + ;; + + # faulty mode + *) + echo "called set_bind_cap incorrect (\$1 missing)" >&2 + ;; + esac + + # set capabilities if any + if [ "$cap_mode" ]; then + printf "setcap -q CAP_NET_BIND_SERVICE%s /opt/netbox/venv/bin/uwsgi\n" "$cap_mode" + fi +} +SYSTEMD_SOCKET="$(cat "$__object/files/systemd_socket")" + + case "$state" in # install uwsgi enabled|disabled) # not installed if ! [ -s "$__object/explorer/installed" ]; then - echo "/opt/netbox/venv/bin/pip3 install uwsgi" + echo "/opt/netbox/venv/bin/pip3 install -q uwsgi" + set_bind_cap set do_restart=yes printf "installed\n" >> "$__messages_out" # updates available elif [ -s "$__object/explorer/upgradeable" ]; then - echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + echo "/opt/netbox/venv/bin/pip3 install -q --upgrade uwsgi" + set_bind_cap set do_restart=yes printf "upgraded\n" >> "$__messages_out" fi @@ -25,6 +74,11 @@ case "$state" in printf "configured\n" >> "$__messages_out" fi + # if no capabilities were set yet, check if any are required + if [ -z "$cap_mode" ]; then + set_bind_cap correct + fi + # restart uwsgi if [ "$do_restart" ] && [ "$state" != "disabled" ]; then @@ -40,7 +94,7 @@ EOF # check if installed if [ -s "$__object/explorer/installed" ]; then # service already disabled - echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi" + echo "/opt/netbox/venv/bin/pip3 uninstall -qy uwsgi" printf "uninstalled\n" >> "$__messages_out" fi ;; diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 3fb8515..d54401e 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -15,6 +15,19 @@ protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This application is available via the `uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper service, too. +**As uWSGI will be started as netbox user, it does not have privileges to +bind to a privileaged port (all ports below 1024).** Because uWSGI will +drop privileages anyway before binding to a port, solutions are to use +the systemd sockets to activate the ports as root or set linux kernel +capabilites to bind to such a privileaged port. + +As systemd sockets (or uwsgi itself) do not allow to distinguish multiple +sockets if different protocols are used for different sockets, this type does +not use systemd sockets if it is requested from the user. Using the +``--bind-to`` and ``--protocol`` parameters, it uses the systemd socket +activation. Else, it set the different sockets and protocols natively to uwsgi +and add kernel capabilities to be able to listen to privileaged ports. + REQUIRED PARAMETERS ------------------- @@ -39,8 +52,18 @@ state bind-to - The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. - Defaults to ``127.0.0.1:3031``. Can be set multiple times. + The socket uwsgi should bind to. Must be UNIX/TCP (or anything that + systemd sockets accept as stream). Defaults to ``127.0.0.1:3031``. Can be + set multiple times. The used protocol is defined by ``--protocol``. + + **By setting up the socket via this parameter, it uses systemd sockets to + handle these.** This parameter will be ignored if a more detailed paramter + is given (``--$proto-bind``). + +protocol + The protocol which should be used for the socket given by the ``--bind-to`` + parameter. Possible values are ``uwsgi``, ``http``, ``fastcgi`` and + ``scgi``. If nothing given, it defaults to ``uwsgi``. uwsgi-bind http-bind @@ -50,6 +73,12 @@ scgi-bind ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must be a UNIX/TCP socket. Can be set multiple times. + **By using such parameters instead of ``--bind-to``, no systemd sockets + will be used because it can not handle sockets for multiple protocols.** + Instead, the native socket binding will be used. It will add kernel + capabilites to bind to privileaged ports, too. This allow binds to ports + like 80 as netbox user. + BOOLEAN PARAMETERS ------------------ diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index c5885c9..7c593e8 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -25,15 +25,30 @@ case "$param_state" in esac +mkdir "$__object/files" + +# check if systemd sockets will be used +if [ -f "$__object/parameter/bind-to" ]; then + SYSTEMD_SOCKET="yes" +fi +if find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print -quit | grep -q .; then + SYSTEMD_SOCKET="no" +fi +echo "$SYSTEMD_SOCKET" > "$__object/files/systemd_socket" + if [ "$state" = "present" ]; then - # *bind* parameters are directly processed in the gen script + # already checked outside this if-clause + export SYSTEMD_SOCKET + + PROTOCOL="$(cat "$__object/parameter/protocol")" + export PROTOCOL + if [ -f "$__object/parameter/serve-static" ]; then STATIC_MAP="yes" export STATIC_MAP fi # process template - mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" # uwsgi config file @@ -48,7 +63,24 @@ else fi +# handle the systemd socket +if [ "$SYSTEMD_SOCKET" = "yes" ]; then + TYPE="uWSGI" + export TYPE + + # generate and set the socket unit + "$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \ + > "$__object/files/netbox.socket" + __systemd_unit uwsgi-netbox.socket \ + --state "$state" --enablement-state "$unit_state" \ + --source "$__object/files/netbox.socket" --restart +else + # remove the systemd socket unit + __systemd_unit uwsgi-netbox.socket --state absent +fi + # install service file +"$__type/files/netbox.service.sh" > "$__object/files/netbox.service" __systemd_unit uwsgi-netbox.service \ --state "$state" --enablement-state "$unit_state" \ - --source "$__type/files/netbox.service" --restart + --source "$__object/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/parameter/default/protocol b/type/__netbox_uwsgi/parameter/default/protocol new file mode 100644 index 0000000..caf986e --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/protocol @@ -0,0 +1 @@ +uwsgi diff --git a/type/__netbox_uwsgi/parameter/optional b/type/__netbox_uwsgi/parameter/optional index ff72b5c..3284ccc 100644 --- a/type/__netbox_uwsgi/parameter/optional +++ b/type/__netbox_uwsgi/parameter/optional @@ -1 +1,2 @@ state +protocol From 32c95ccb220cb3302d1c43cca5ace01fa66a6755 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:50:22 +0200 Subject: [PATCH 84/98] __netbox: don't call `pip3 uninstall` if nothing available To avoid an error from pip, it will be checked if there are packages to uninstall. This is better than just doing `|| true`. --- type/__netbox/gencode-remote | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 1ef78cd..07b0fcf 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -66,12 +66,16 @@ ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Uninstall packages not required anymore -# if versions not shortend, they will be ignored by pip, but not by comm +# if versions not be shortend, they will be ignored by pip, but not by comm # all of this could be done with grep, too, but it's still must be shortend with awk awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" -comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" \ - | xargs /opt/netbox/venv/bin/pip3 uninstall -qy +comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" > "\$tmpdir/pip-uninstall.txt" + +# only uninstall if something is available (to avoid errors cause of this) +if [ -s "\$tmpdir/pip-uninstall.txt" ]; then + /opt/netbox/venv/bin/pip3 uninstall -qy -r "\$tmpdir/pip-uninstall.txt" +fi # Install python dependencies. # avoid gunicorn, because it will be done in an other type @@ -86,6 +90,7 @@ EOF fi cat << EOF + # Set final permissions. chown -R netbox /opt/netbox From 72f2996c509e64a9f35f53d7b347ae65d1e6df59 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:56:54 +0200 Subject: [PATCH 85/98] __netbox*: fix shellcheck SC2162 --- type/__netbox/files/netbox.socket.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/netbox.socket.sh b/type/__netbox/files/netbox.socket.sh index 03e3d44..2ef9e81 100755 --- a/type/__netbox/files/netbox.socket.sh +++ b/type/__netbox/files/netbox.socket.sh @@ -20,7 +20,7 @@ Description=Socket for NetBox via $TYPE UNIT # read all sockets to listen to -while read line; do +while read -r line; do printf "ListenStream=%s\n" "$line" done < "$1" From 144cae3b796831d4c96e221611b25945f4462f68 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 10:16:25 +0200 Subject: [PATCH 86/98] __netbox: use __pyenv and remove obsolete netbox stop Now, the python venv is now created via `pyvenv` or `python3 -m venv` instead of the legacy `virtualenv`. For this, not all python processes from the venv need to be stopped. Migration from previous versions of this type may be difficult, but solvable if the venv will be recreated. --- type/__netbox/gencode-remote | 32 +++++++------------------------- type/__netbox/manifest | 2 ++ 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 07b0fcf..591717f 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -24,26 +24,7 @@ cd "\$tmpdir" curl -sS -L '$url' > '$archive' tar xf '$archive' -EOF - # Stop everything in the pyenv to update - cat << EOF -# Try to stop everything in the venv (ignore non-existant services) -systemctl -q stop netbox gunicorn-netbox uwsgi-netbox || true - -# kill and poll till all venv processes end -# at least if they are called with full path in the cmd (like in the services) -pids="\$( ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' )" -if [ "\$pids" ]; then - kill \$pids - while ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | grep -q . ; do - sleep 0.5 - done -fi - -EOF - - cat << EOF # backup requirement files if [ -f /opt/netbox/requirements.txt ]; then cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt @@ -61,9 +42,7 @@ ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' -# Setup & enter python virtualenv. -# forcing python3 to be sure (till python4 gets released ..) -virtualenv -p python3 /opt/netbox/venv +# virtualenv is given already by __pyvenv, just using it # Uninstall packages not required anymore # if versions not be shortend, they will be ignored by pip, but not by comm @@ -112,6 +91,7 @@ rm -rf "\$tmpdir" # Save version after successful installation printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version + EOF # meta @@ -127,10 +107,12 @@ if grep -q "^__file/opt/netbox/" "$__messages_in"; then fi -# check for changes +# Check for changes if [ "$changes" = "yes" ]; then - # to avoid database corruption at config changes, both services must be - # stopped at the same time (noted in the manual, too). + # After the upstream upgrade.sh script, it's ok to migrate while the + # application is running ;) + + # restarting after changes cat << EOF # Restart service. All required services are included with netbox.service. systemctl restart netbox diff --git a/type/__netbox/manifest b/type/__netbox/manifest index fbcf253..475cadb 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -187,6 +187,8 @@ fi # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home +# Generate python environment (user will be set by gencode-remote) +require="__user/netbox" __pyvenv /opt/netbox/venv/ # Generate and upload netbox configuration. mkdir -p "$__object/files" From 9def5700c8a1de7e59ed0ac3c40cdd1b7c484e05 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 10:21:52 +0200 Subject: [PATCH 87/98] __netbox: replace redis DEFAULT_TIMEOUT with RQ_DEFAULT_TIMEOUT The first one was deprecated by the second one. --- type/__netbox/files/configuration.py.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 06a1c86..31ebd05 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -36,7 +36,6 @@ REDIS = { # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '$REDIS_PASSWORD', 'DATABASE': $((REDIS_DBID_OFFSET + 0)), - 'DEFAULT_TIMEOUT': 300, 'SSL': $REDIS_SSL, }, 'caching': { @@ -47,10 +46,10 @@ REDIS = { # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '$REDIS_PASSWORD', 'DATABASE': $((REDIS_DBID_OFFSET + 1)), - 'DEFAULT_TIMEOUT': 300, 'SSL': $REDIS_SSL, } } +RQ_DEFAULT_TIMEOUT = 300 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and From 558021d2eeb10697d2dec0e7dc563fbf26e63ba0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 11:08:11 +0200 Subject: [PATCH 88/98] __netbox: reorder generated code for shorter transition Now, the venv and pip things are done before the program files are replaced, which should minimize the outage where the application is not available. --- type/__netbox/gencode-remote | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 591717f..5d4b7be 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -25,30 +25,22 @@ curl -sS -L '$url' > '$archive' tar xf '$archive' +# virtualenv is given already by __pyvenv, just using it + # backup requirement files if [ -f /opt/netbox/requirements.txt ]; then - cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt + mv /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt else # preseve file-not-found errors and warnings touch /opt/netbox/old-requirements.txt fi cp '$src/requirements.txt' /opt/netbox/ -# Deploy sources and restore configuration. -rm -rf '$install_dir' -cp -r '$src/netbox' '$install_dir' -# force links to the cdist directory -ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' -ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' - - -# virtualenv is given already by __pyvenv, just using it - # Uninstall packages not required anymore # if versions not be shortend, they will be ignored by pip, but not by comm # all of this could be done with grep, too, but it's still must be shortend with awk -awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" -awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" +awk -F== '{print \$1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" +awk -F== '{print \$1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" > "\$tmpdir/pip-uninstall.txt" # only uninstall if something is available (to avoid errors cause of this) @@ -70,9 +62,17 @@ EOF cat << EOF +# Deploy sources and restore configuration. +rm -rf '$install_dir' +cp -r '$src/netbox' '$install_dir' +# force links to the cdist directory +ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' + # Set final permissions. chown -R netbox /opt/netbox + # NetBox manage scripts # Run database migrations. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate From 46bff41f6d8bb9a0fd50ac7b659690f5e94377dc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 11:22:47 +0200 Subject: [PATCH 89/98] __netbox{,_uwsgi}: updated manpages Mostly added notes of what does not work that well. --- type/__netbox/man.rst | 28 +++++++++++++--------------- type/__netbox_uwsgi/man.rst | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 6d50268..f34ead7 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -16,7 +16,7 @@ It setup systemd unit files for the services `netbox` and `netbox-rq`. The `netbox` service only wrap all netbox related services, e.g. restarting and so one will be delegated to all related services. -The application is still not accessable because WSGI server is required. To +The application is still not accessable because a WSGI server is required. To access the application through WSGI, uWSGI or Gunicorn can be used. The setup can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. @@ -236,23 +236,21 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. -If you change a configuration, the database may go corrupt if two instances of -the application are running with different configurations at the same time. -This most commonly happens when the WSGI server and RQ-worker restarts after a -configuration change. This occours in the following case for example: +You may also be interested of writing a own type which handles the creation of +the super user. To do this non-interactivly, see the ansible role as `reference +`. -.. code-block:: sh +If you change the secret key while the netbox instance is running, there is a +time frame where the access to the application corrupts the whole database. +Then, you need to restore a backup or wipe the database. - systemctl restart gunicorn-netbox # WSGI-server already online with new - # configuration after this command. - systemctl restart netbox-rq # RQ-Worker still worked with the old - # configuration till here. +Currently, the cause is not clear, but it should work if you do not touch +netbox while the configuration is done (do not shut it down, too). It only +applies for changes of the secret key, which not happen normally. -This type handles the restart of both services correctly to avoid such database -corruptions. To safely manual restart the whole netbox instance manual, simply -restart all services in one ``systemctl restart netbox`` command, as it ensures -that first all services are shut down before starting one of them. The service -``netbox`` wraps all required services that are available. +Maybe the `--restart` flag for the `__systemd_unit` types is not the best idea, +but avoids that the changes will not be applied. It could be solved if the type +would send messages from his actions. SEE ALSO diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index d54401e..c5e51ca 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -129,6 +129,7 @@ EXAMPLES # with multiple protocols # parameter `--bind-to` will be ignored + # avoids systemd sockets, but can handle multiple protocols __netbox $args require="__netbox" __netbox_uwsgi --uwsgi-bind 0.0.0.0:3031 \ --http-bind 0.0.0.0:8080 \ @@ -150,6 +151,20 @@ EXAMPLES require="__netbox_uwsgi" __systemd_service uwsgi-netbox --state stopped +NOTES +----- +If systemd sockets are used, uwsgi can not be reloaded because it does not +handle the socket correctly. It works by completly restarting uwsgi (because +it is near the same cause of the systemd socket) or tweaking the service unit +with the line ``StandardInput=socket``, which limits you to only one address +to bind to (else, the service will not start). + +Maybe someone is interested in enabling log files, because the "log to stdout" +is not the fanciest approach (because it is shown in the journal). See the +`uwsgi documentation ` for +reference. + + SEE ALSO -------- `uWSGI Documentation `_ From c1d5d5b088724c22c8b8e79f00a1b82ed88436f2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 08:52:43 +0200 Subject: [PATCH 90/98] __netbox: remove dependency to package virtualenv This is not required any more because it was completly replaced by `python3-venv`. The venv handling is already given to `__pyvenv`. --- type/__netbox/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 475cadb..4be49bd 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -6,7 +6,7 @@ case "$os" in debian|ubuntu) # Install netbox dependencies. for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl sudo; do __package $pkg done From 87e115da98425c69d8bb37397ccdf47c0f2a4dc2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 25 Oct 2020 19:13:10 +0100 Subject: [PATCH 91/98] __netbox: some manpage fixes Mostly invalid links .. --- type/__netbox/man.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index f34ead7..135304c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -100,8 +100,7 @@ redis-password redis-dbid-offset Offset to set the redis database id's. The `tasks` database id is - `offset + 0` and `caching` is `offset + 1`. The offset defaults - to ``0``. + `offset + 0` and `caching` is `offset + 1`. The offset defaults to ``0``. smtp-host Host of the SMTP email server. Defaults to ``localhost``. @@ -172,12 +171,12 @@ redis-ssl smtp-use-tls Uses TLS to connect to the SMTP email server. `See documentation - `_ for more information. smtp-use-ssl Uses implicit TLS with the SMTP email server. `See documentation - `_ for more information. login-required @@ -229,16 +228,16 @@ is a more good way to inject python code for dynamic configuration variables, you are welcome to contribute! - `Possible optional settings - ` + `_ If you not setup ldap authentification, you may be interested into how to `setting up a super user -` +`_ directly on the machine to be able to access and use NetBox. You may also be interested of writing a own type which handles the creation of the super user. To do this non-interactivly, see the ansible role as `reference -`. +`_. If you change the secret key while the netbox instance is running, there is a time frame where the access to the application corrupts the whole database. From 9f4a85901ef8d58e95924dd9f584ca6437934030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 11:04:07 +0100 Subject: [PATCH 92/98] Import html/man doc generation logic from upstream cdist --- .gitlab-ci.yml | 10 +- Makefile | 70 ++++++++++ docs/src/Makefile | 235 ++++++++++++++++++++++++++++++++++ docs/src/conf.py | 101 +++++++++++++++ docs/src/index.rst.sh | 45 +++++++ docs/src/manpage.py | 87 +++++++++++++ type/__matrix_element/man.rst | 2 +- type/__matterbridge/man.rst | 4 +- type/__netbox/man.rst | 5 +- type/__netbox_uwsgi/man.rst | 5 +- 10 files changed, 552 insertions(+), 12 deletions(-) create mode 100644 Makefile create mode 100644 docs/src/Makefile create mode 100644 docs/src/conf.py create mode 100755 docs/src/index.rst.sh create mode 100644 docs/src/manpage.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dba7864..79c532e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,20 @@ stages: - test + - doc image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest shellcheck: stage: test script: - - ./scripts/run-shellcheck.sh + - make lint manpages: stage: test script: - - ./scripts/run-manpage-checks.sh + - make check-manpages + +docs: + stage: doc + script: + - make docs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1a0cfb3 --- /dev/null +++ b/Makefile @@ -0,0 +1,70 @@ +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo "man build only man user documentation" + @echo "html build only html user documentation" + @echo "docs build both man and html user documentation" + @echo "check-manpages check for manpage in types" + @echo "lint run shellcheck on types" + @echo "check run both type manpage checks and linting" + @echo "clean clean" + +DOCS_SRC_DIR=./docs/src +TYPEDIR=./type + +SPHINXM=make -C $(DOCS_SRC_DIR) man +SPHINXH=make -C $(DOCS_SRC_DIR) html +SPHINXC=make -C $(DOCS_SRC_DIR) clean + +################################################################################ +# Manpages +# +MAN7DSTDIR=$(DOCS_SRC_DIR)/man7 + +# Use shell / ls to get complete list - $(TYPEDIR)/*/man.rst does not work +# Using ls does not work if no file with given pattern exist, so use wildcard +MANTYPESRC=$(wildcard $(TYPEDIR)/*/man.rst) +MANTYPEPREFIX=$(subst $(TYPEDIR)/,$(MAN7DSTDIR)/cdist-type,$(MANTYPESRC)) +MANTYPES=$(subst /man.rst,.rst,$(MANTYPEPREFIX)) + +# Link manpage: do not create man.html but correct named file +$(MAN7DSTDIR)/cdist-type%.rst: $(TYPEDIR)/%/man.rst + mkdir -p $(MAN7DSTDIR) + ln -sf "../../../$^" $@ + +DOCSINDEX=$(MAN7DSTDIR)/index.rst +DOCSINDEXH=$(DOCS_SRC_DIR)/index.rst.sh + +$(DOCSINDEX): $(DOCSINDEXH) + $(DOCSINDEXH) + +# Manpages: .cdist Types +DOT_CDIST_PATH=${HOME}/.cdist +DOTMAN7DSTDIR=$(MAN7DSTDIR) +DOTTYPEDIR=$(DOT_CDIST_PATH)/type + +# Link manpage: do not create man.html but correct named file +$(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst + ln -sf "$^" $@ + +man: $(MANTYPES) $(DOCSINDEX) + $(SPHINXM) + +html: $(MANTYPES) $(DOCSINDEX) + $(SPHINXH) + +docs: man html + +check-manpages: + ./scripts/run-manpage-checks.sh + +lint: + ./scripts/run-shellcheck.sh + +check: check-manpages lint + +clean: + $(SPHINXC) + rm -f docs/src/index.rst + rm -rf docs/src/man7/ + rm -rf docs/src/__pycache__/ diff --git a/docs/src/Makefile b/docs/src/Makefile new file mode 100644 index 0000000..2e9d6ce --- /dev/null +++ b/docs/src/Makefile @@ -0,0 +1,235 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +PAPER ?= +BUILDDIR ?= ../dist +# for cache, etc. +_BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) + $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(_BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + rm -rf $(_BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/cdist-docs.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/cdist-docs.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/cdist-docs" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/cdist-docs" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b cman $(ALLSPHINXOPTS) $(BUILDDIR)/man + mkdir -p $(BUILDDIR)/man/man7 + mv -f $(BUILDDIR)/man/*.7 $(BUILDDIR)/man/man7/ + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/src/conf.py b/docs/src/conf.py new file mode 100644 index 0000000..19b2dfd --- /dev/null +++ b/docs/src/conf.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import sys +import os +import sphinx_rtd_theme + +from datetime import date + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", ".."))) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'docs.src.manpage', + 'sphinx.ext.extlinks', +] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = ['.rst'] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'cdist-contrib' +copyright = 'cdist-contrib contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + +version = str(date.today()) +release = os.popen('git rev-parse HEAD').read() + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Output file base name for HTML help builder. +htmlhelp_basename = 'cdistcontribdoc' + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +root_mandir = os.path.dirname(os.path.realpath(__file__)) +mandirs = [] +for mansubdir in ('man7',): + mandirs.append((os.path.join(root_mandir, mansubdir), mansubdir[-1])) +man_pages = [] +for mandir, section in mandirs: + for root, dirs, files in os.walk(mandir): + for fname in files: + froot, fext = os.path.splitext(fname) + if fext == '.rst': + man_page = (os.path.join('man' + str(section), froot), + froot, '', [], section) + man_pages.append(man_page) + +# man_pages = [ +# ('cdist-type', 'cdist-type', 'cdist-type documentation', +# [author], 1), +# ('man7/cdist-type__file', 'cdist-type__file', +# '', [], 1), +# ('cdist-type__directory', 'cdist-type__directory', +# 'cdist-type__directory documentation', [author], 1), +# ] + +# If true, show URL addresses after external links. +# man_show_urls = False diff --git a/docs/src/index.rst.sh b/docs/src/index.rst.sh new file mode 100755 index 0000000..28c5078 --- /dev/null +++ b/docs/src/index.rst.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +__cdist_pwd="$(pwd -P)" +__cdist_mydir="${0%/*}"; +__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)" +__cdist_myname=${0##*/}; +__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" + +filename="${__cdist_myname%.sh}" +dest="$__cdist_abs_mydir/$filename" + +cd "$__cdist_abs_mydir" + +exec > "$dest" +cat << EOF +cdist-contrib - Community maintained cdist types +================================================ + +This project extends the \`cdist \`_ configuration management +tool with community-maitained types which are either too specific to fit/be +maintained in cdist itself or were not accepted in code cdist but could still +be useful. + +Please note this project is a **rolling release**! The documentation you're +reading has been generated from the |version| state (commit |release|). +Sources are available on \`code.ungleich.ch +\`_. + + +.. toctree:: + :hidden: + +EOF + +# If there is no such file then ls prints error to stderr, +# so redirect stderr to /dev/null. +for type in $(ls man7/cdist-type__*.rst 2>/dev/null | LC_ALL=C sort); do + no_dir="${type#man7/}"; + no_type="${no_dir#cdist-type}"; + name="${no_type%.rst}"; + manref="${no_dir%.rst}" + man="${manref}(7)" + + echo " $name" "" +done diff --git a/docs/src/manpage.py b/docs/src/manpage.py new file mode 100644 index 0000000..1f8ac4f --- /dev/null +++ b/docs/src/manpage.py @@ -0,0 +1,87 @@ +import sphinx.builders.manpage +import sphinx.writers.manpage +from docutils.frontend import OptionParser +from sphinx.util.console import bold, darkgreen +from six import string_types +from docutils.io import FileOutput +from os import path +from sphinx.util.nodes import inline_all_toctrees +from sphinx import addnodes +from sphinx.util import logging + +""" + Extension based on sphinx builtin manpage. + It does not write its own .SH NAME based on config, + but leaves everything to actual reStructuredText file content. +""" + + +logger = logging.getLogger(__name__) + + +class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator): + + def header(self): + tmpl = (".TH \"%(title_upper)s\" \"%(manual_section)s\"" + " \"%(date)s\" \"%(version)s\" \"%(manual_group)s\"\n") + return tmpl % self._docinfo + + +class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): + + def __init__(self, builder): + super().__init__(builder) + self.translator_class = ( + self.builder.get_translator_class() or ManualPageTranslator) + + +class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): + + name = 'cman' + default_translator_class = ManualPageTranslator + + def write(self, *ignored): + docwriter = ManualPageWriter(self) + docsettings = OptionParser( + defaults=self.env.settings, + components=(docwriter,), + read_config_files=True).get_default_values() + + logger.info(bold('writing... '), nonl=True) + + for info in self.config.man_pages: + docname, name, description, authors, section = info + if isinstance(authors, string_types): + if authors: + authors = [authors] + else: + authors = [] + + targetname = '%s.%s' % (name, section) + logger.info(darkgreen(targetname) + ' { ', nonl=True) + destination = FileOutput( + destination_path=path.join(self.outdir, targetname), + encoding='utf-8') + + tree = self.env.get_doctree(docname) + docnames = set() + largetree = inline_all_toctrees(self, docnames, docname, tree, + darkgreen, [docname]) + logger.info('} ', nonl=True) + self.env.resolve_references(largetree, docname, self) + # remove pending_xref nodes + for pendingnode in largetree.traverse(addnodes.pending_xref): + pendingnode.replace_self(pendingnode.children) + + largetree.settings = docsettings + largetree.settings.title = name + largetree.settings.subtitle = description + largetree.settings.authors = authors + largetree.settings.section = section + + docwriter.write(largetree, destination) + logger.info("") + + +def setup(app): + app.add_builder(ManualPageBuilder) diff --git a/type/__matrix_element/man.rst b/type/__matrix_element/man.rst index c3e0d5a..05f0685 100644 --- a/type/__matrix_element/man.rst +++ b/type/__matrix_element/man.rst @@ -1,5 +1,5 @@ cdist-type__matrix_element(7) -====================== +============================= NAME ---- diff --git a/type/__matterbridge/man.rst b/type/__matterbridge/man.rst index 02eef2c..babf34b 100644 --- a/type/__matterbridge/man.rst +++ b/type/__matterbridge/man.rst @@ -33,9 +33,9 @@ EXAMPLES .. code-block:: sh - __matterbridge --version 1.16.3 --config - << EOF + __matterbridge --version 1.16.3 --config - <<- EOF [...] -EOF + EOF SEE ALSO diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 135304c..5f78f1d 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -122,7 +122,6 @@ basepath webroot ``/``. For example, if installed at https://example.com/netbox/, set the value ``netbox/``. -http-proxy https-proxy Proxy which will be used with any HTTP request like webhooks. @@ -171,12 +170,12 @@ redis-ssl smtp-use-tls Uses TLS to connect to the SMTP email server. `See documentation - `_ + `__ for more information. smtp-use-ssl Uses implicit TLS with the SMTP email server. `See documentation - `_ + `__ for more information. login-required diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index c5e51ca..13dc6dc 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -65,10 +65,7 @@ protocol parameter. Possible values are ``uwsgi``, ``http``, ``fastcgi`` and ``scgi``. If nothing given, it defaults to ``uwsgi``. -uwsgi-bind -http-bind -fastcgi-bind -scgi-bind +scgi-bind, uwsgi-bind, http-bind, fastcgi-bind Bind the application to a specific protocol instead of implicit uwsgi via ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must be a UNIX/TCP socket. Can be set multiple times. From cd833363224493e8a63d7b0b02a83958ffc97663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 11:26:45 +0100 Subject: [PATCH 93/98] Use our own image for CI --- .gitlab-ci.yml | 2 +- scripts/ci-container/Dockerfile | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 scripts/ci-container/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79c532e..31505a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ stages: - test - doc -image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest +image: code.ungleich.ch:5050/ungleich-public/cdist-contrib/ci-container:latest shellcheck: stage: test diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile new file mode 100644 index 0000000..33646f4 --- /dev/null +++ b/scripts/ci-container/Dockerfile @@ -0,0 +1,7 @@ +# This image is used in the cdist-contrib CI for linting and generating the +# documentation. +FROM fedora:latest +MAINTAINER Timothée Floure + +RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ + ShellCheck From 2bedbe96870d492d7f7d6cd868c210a7ba0e4f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 10:55:25 +0100 Subject: [PATCH 94/98] Automatically deploy docs from master (at contrib.cdi.st) --- .gitlab-ci.yml | 11 +++++++++++ scripts/ci-container/Dockerfile | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31505a5..4b61568 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,5 +16,16 @@ manpages: docs: stage: doc + only: + - master + before_script: + - eval $(ssh-agent -s) + - echo "$CD_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - mkdir -p ~/.ssh + - echo "$CD_SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts script: - make docs + - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<- EOF + put -r docs/dist/html/* + EOF diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile index 33646f4..daa0f26 100644 --- a/scripts/ci-container/Dockerfile +++ b/scripts/ci-container/Dockerfile @@ -4,4 +4,4 @@ FROM fedora:latest MAINTAINER Timothée Floure RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ - ShellCheck + ShellCheck openssh-clients From ec41ef34901bf164b5decb5fa8fa9f150c131a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:05:09 +0100 Subject: [PATCH 95/98] Patch sftp call for doc deployment in CI --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4b61568..9392cd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,4 @@ docs: - chmod 644 ~/.ssh/known_hosts script: - make docs - - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<- EOF - put -r docs/dist/html/* - EOF + - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<< "put -r docs/dist/html/*" From ec0dc30c87a1b770c647641bb3f3abaa1f735032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:34:54 +0100 Subject: [PATCH 96/98] Generate documentation homepage from README --- .gitlab-ci.yml | 2 +- README.md | 3 ++- docs/src/index.rst.sh | 21 ++++++++------------- scripts/ci-container/Dockerfile | 2 +- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9392cd9..81db798 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,5 +25,5 @@ docs: - echo "$CD_SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts script: - - make docs + - make html - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<< "put -r docs/dist/html/*" diff --git a/README.md b/README.md index 4231de2..4d20c22 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ maintained in cdist itself or were not accepted in code cdist but could still be useful. This project does not have releases and is continously updated: see git history -for change log. +for change log. You will find HTML documentation at +[contrib.cdi.st](https://contrib.cdi.st). ## Using cdist-contrib diff --git a/docs/src/index.rst.sh b/docs/src/index.rst.sh index 28c5078..babc1d9 100755 --- a/docs/src/index.rst.sh +++ b/docs/src/index.rst.sh @@ -9,23 +9,18 @@ __cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" filename="${__cdist_myname%.sh}" dest="$__cdist_abs_mydir/$filename" +if ! command -v pandoc > /dev/null; then + echo "Pandoc is required to generate HTML index from README." >&2 + exit 1 +fi + cd "$__cdist_abs_mydir" exec > "$dest" + +pandoc -f markdown -t rst ../../README.md + cat << EOF -cdist-contrib - Community maintained cdist types -================================================ - -This project extends the \`cdist \`_ configuration management -tool with community-maitained types which are either too specific to fit/be -maintained in cdist itself or were not accepted in code cdist but could still -be useful. - -Please note this project is a **rolling release**! The documentation you're -reading has been generated from the |version| state (commit |release|). -Sources are available on \`code.ungleich.ch -\`_. - .. toctree:: :hidden: diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile index daa0f26..9900322 100644 --- a/scripts/ci-container/Dockerfile +++ b/scripts/ci-container/Dockerfile @@ -4,4 +4,4 @@ FROM fedora:latest MAINTAINER Timothée Floure RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ - ShellCheck openssh-clients + ShellCheck openssh-clients pandoc From 73e31e6d1ee69181f762847e6844c77e6a0b9db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:56:37 +0100 Subject: [PATCH 97/98] Mention known external cdist resources in README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 4d20c22..d9c5e22 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,11 @@ Every type in cdist-contrib must: * Have a `man.rst` documentation page. * Pass [shellcheck](http://shellcheck.net/) without errors. + +## Other resources + +Some people/organizations are known to keep some cdist types that might be of +interest to others: + +* [cdist-evilham](https://git.sr.ht/~evilham/cdist-evilham): Evilham's cdist-types +* [cdist-recycledcloud](https://code.recycled.cloud/e-Durable/cdist-recycledcloud): e-Durable SA / Recycled Cloud public types From f01f73f33d428ecc2e09bc8754d63e512862a61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:58:21 +0100 Subject: [PATCH 98/98] Remove mention to legacy mattermost room from README --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index d9c5e22..28f54db 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,11 @@ And you would run [cdist][cdist] from the same directory as follows: ## Participating in the [cdist][cdist] community -Join us on [#cdist:ungleich.ch][cdistmatrix] on matrix or on -[#cdist over mattermost][cdistmattermost]. - +Join us on [#cdist:ungleich.ch][cdistmatrix] on matrix! [cdist]: https://www.cdi.st/ [cdistconfig]: https://www.cdi.st/manual/latest/cdist-configuration.html [cdistmatrix]: https://matrix.to/#/#cdist:ungleich.ch -[cdistmattermost]: https://chat.ungleich.ch/ungleich/channels/cdist ## Contributing