__netbox*: added systemd socket support

The Gunicorn type now supports systemd sockets only. With uWSGI, you can
choose between it and the native sockets based on the parameters chosen.
This is done because it could not be implemented to have multiple
protocols with the systemd sockets (so you may choose).

The systemd socket unit file is generally available, so both types use
the same script to generate the socket unit file.

type/__netbox/files/netbox.socket.sh

@@ -0,0 +1,33 @@
+#!/bin/sh -e
+# __netbox/files/netbox.socket.sh
+
+# This is shared between all WSGI-server types.
+
+# Arguments:
+#  1: File which list all sockets to listen on (sepearated by \n)
+
+if [ $# -ne 1 ]; then
+    printf "netbox.socket.sh: argument \$1 missing or too much given!\n" >&2
+    exit 1
+fi
+
+
+cat << UNIT
+[Unit]
+Description=Socket for NetBox via $TYPE
+
+[Socket]
+UNIT
+
+# read all sockets to listen to
+while read line; do
+    printf "ListenStream=%s\n" "$line"
+done < "$1"
+
+cat << UNIT
+SocketUser=netbox
+SocketGroup=www-data
+
+[Install]
+WantedBy=sockets.target
+UNIT

type/__netbox_gunicorn/files/gunicorn.py.sh

@@ -13,7 +13,7 @@ cores="$(cat "$__explorer/cpu_cores")"
 
 cat << EOF
 # The IP address (typically localhost) and port that the Netbox WSGI process should listen on
-bind = [$HOST ]
+#bind = done via systemd socket 'gunicorn-netbox.socket'
 
 # Number of gunicorn workers to spawn. This should typically be 2n+1, where
 # n is the number of CPU cores present.

type/__netbox_gunicorn/files/netbox.service

@@ -3,13 +3,14 @@ Description=NetBox Gunicorn WSGI Service
 Documentation=https://netbox.readthedocs.io/en/stable/
 PartOf=netbox.service
 Requires=netbox-rq.service
+Requires=gunicorn-netbox.socket
 Wants=network.target
 After=netbox.service
 After=network.target
 After=redis-server.service postgresql.service
 
 [Service]
-Type=simple
+Type=notify
 
 User=netbox
 Group=netbox

type/__netbox_gunicorn/files/netbox.socket.sh

@@ -0,0 +1 @@
+../../__netbox/files/netbox.socket.sh

type/__netbox_gunicorn/man.rst

@@ -14,6 +14,9 @@ sockets. Static content must be served independent of Gunicorn. The Gunicorn
 daemon is available as the `gunicorn-netbox` systemd service, but also
 available via the `netbox` wrapper service.
 
+It will use systemd socket activation to listen to the given sockets. This
+should allow to bind to privileaged ports (all below 1024) and hot reloads.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -38,8 +41,9 @@ state
 
 bind-to
     The hosts the gunicorn socket should be bind to. Formats are `IP`,
-    `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple
-    times. Defaults to ``127.0.0.1:8001``.
+    `IP:PORT`, `PATH` or anything other that systemd socket units will
+    understand as stream. Parameter can be set multiple times. Defaults
+    to ``127.0.0.1:8001``.
 
 
 BOOLEAN PARAMETERS

type/__netbox_gunicorn/manifest

@@ -25,17 +25,10 @@ case "$param_state" in
 esac
 
 
-if [ "$state" = "present" ]; then
-    HOST=""
-    while read -r host; do
-        # shellcheck disable=SC2089
-        HOST="$HOST '$host',"
-    done < "$__object/parameter/bind-to"
-    # shellcheck disable=SC2090
-    export HOST
+mkdir "$__object/files"
 
+if [ "$state" = "present" ]; then
     # process template
-    mkdir "$__object/files"
     "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py"
 
     # gunicorn config file
@@ -49,7 +42,16 @@ else
 fi
 
 
-# install service file
+TYPE="Gunicorn"
+export TYPE
+
+"$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \
+    > "$__object/files/netbox.socket"
+
+# install systemd files
+__systemd_unit gunicorn-netbox.socket \
+    --state "$state" --enablement-state "$unit_state" \
+    --source "$__object/files/netbox.socket" --restart
 __systemd_unit gunicorn-netbox.service \
     --state "$state" --enablement-state "$unit_state" \
     --source "$__type/files/netbox.service" --restart

type/__netbox_uwsgi/explorer/bind-capability

@@ -0,0 +1,12 @@
+#!/bin/sh -e
+# explorer/bind-capablility
+
+# Checks if the uWSGI binary have the capability to bind to privileaged ports
+# as a non-root user. It's required if no systemd sockets are used (cause of
+# the use of multiple protocols etc.)
+
+binary="/opt/netbox/venv/bin/uwsgi"
+# -v verifies if capability is set
+if setcap -q -v CAP_NET_BIND_SERVICE+ep "$binary"; then
+    echo set
+fi

type/__netbox_uwsgi/files/netbox.service.sh

@@ -1,15 +1,26 @@
+#!/bin/sh -e
+
+cat << EOF
 [Unit]
 Description=Netbox uWSGI WSGI Service
 Documentation=https://netbox.readthedocs.io/en/stable/
 PartOf=netbox.service
 Requires=netbox-rq.service
+EOF
+
+# Add dependency to own socket
+if [ "$(cat "$__object/files/systemd_socket")" = "yes" ]; then
+    echo "Requires=uwsgi-netbox.socket"
+fi
+
+cat << EOF
 Wants=network.target
 After=netbox.service
 After=network.target
 After=redis-server.service postgresql.service
 
 [Service]
-Type=simple
+Type=notify
 
 User=netbox
 Group=netbox
@@ -17,8 +28,8 @@ WorkingDirectory=/opt/netbox
 
 ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini
 # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi
-ExecReload=kill -HUP $MAINPID
-ExecStop=kill -INT $MAINPID
+ExecReload=kill -HUP \$MAINPID
+ExecStop=kill -INT \$MAINPID
 KillSignal=SIGQUIT
 
 Restart=on-failure
@@ -26,3 +37,4 @@ RestartSec=30
 
 [Install]
 WantedBy=netbox.service
+EOF

type/__netbox_uwsgi/files/netbox.socket.sh

@@ -0,0 +1 @@
+../../__netbox/files/netbox.socket.sh

type/__netbox_uwsgi/files/uwsgi.ini.sh

@@ -28,20 +28,16 @@ cat << EOF
 ; socket(s) to bind
 EOF
 
-# special protocol to bind
-while read -r param; do
-    if [ -z "$param" ]; then continue; fi  # ignore empty lines from the here-doc
-
-    multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param"
-    socket_changes="yes"
-
-done << INPUT  # here-doc cause of SC2031
-$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print )
-INPUT
-
-# else, default bind to
-if [ -z "$socket_changes" ]; then
-    multi_options "socket" "$__object/parameter/bind-to"
+if [ "$SYSTEMD_SOCKET" != "yes" ]; then
+    # special protocol to bind
+    find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print \
+     | while read -r param; do
+        multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param"
+    done
+else
+    # else, systemd will offer socket
+    echo "; sockets managed via 'uwsgi-netbox.socket'"
+    printf "protocol = %s\n" "$PROTOCOL"
 fi
 
 

type/__netbox_uwsgi/gencode-remote

@@ -3,18 +3,67 @@
 # control state
 state="$(cat "$__object/parameter/state")"
 
+# Set capabilities to aquire privileaged ports as netbox user. Two modes are
+# available to efficiently set capabilites. Assumes libcap-bin is installed as
+# default on debian systems.
+#
+# Arguments:
+#  1: mode to detect if capabilites are required to set ('set' or 'correct')
+set_bind_cap() {
+    cap_mode=""  # reset variable from the execution before
+
+    # check if capabilites are required after given mode
+    case "$1" in
+        # assumes capabilites are not set (cause of new binaries)
+        set)
+            if [ "$SYSTEMD_SOCKET" != "yes" ]; then
+                cap_mode="+ep"
+            fi
+            ;;
+
+        # check if capabilities have changed
+        correct)
+            if [ -s "$__object/explorer/bind-capability" ]; then
+                # capabilites are set
+                if [ "$SYSTEMD_SOCKET" = "yes" ]; then
+                    cap_mode="-ep"  # unset
+                fi
+            else
+                # capabilities are unset
+                if [ "$SYSTEMD_SOCKET" != "yes" ]; then
+                    cap_mode="+ep"  # set
+                fi
+            fi
+            ;;
+
+        # faulty mode
+        *)
+            echo "called set_bind_cap incorrect (\$1 missing)" >&2
+            ;;
+    esac
+
+    # set capabilities if any
+    if [ "$cap_mode" ]; then
+        printf "setcap -q CAP_NET_BIND_SERVICE%s /opt/netbox/venv/bin/uwsgi\n" "$cap_mode"
+    fi
+}
+SYSTEMD_SOCKET="$(cat "$__object/files/systemd_socket")"
+
+
 case "$state" in
     # install uwsgi
     enabled|disabled)
         # not installed
         if ! [ -s "$__object/explorer/installed" ]; then
-            echo "/opt/netbox/venv/bin/pip3 install uwsgi"
+            echo "/opt/netbox/venv/bin/pip3 install -q uwsgi"
+            set_bind_cap set
             do_restart=yes
             printf "installed\n" >> "$__messages_out"
 
         # updates available
         elif [ -s "$__object/explorer/upgradeable" ]; then
-            echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi"
+            echo "/opt/netbox/venv/bin/pip3 install -q --upgrade uwsgi"
+            set_bind_cap set
             do_restart=yes
             printf "upgraded\n" >> "$__messages_out"
         fi
@@ -25,6 +74,11 @@ case "$state" in
             printf "configured\n" >> "$__messages_out"
         fi
 
+        # if no capabilities were set yet, check if any are required
+        if [ -z "$cap_mode" ]; then
+            set_bind_cap correct
+        fi
+
 
         # restart uwsgi
         if [ "$do_restart" ] && [ "$state" != "disabled" ]; then
@@ -40,7 +94,7 @@ EOF
         # check if installed
         if [ -s "$__object/explorer/installed" ]; then
             # service already disabled
-            echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi"
+            echo "/opt/netbox/venv/bin/pip3 uninstall -qy uwsgi"
             printf "uninstalled\n" >> "$__messages_out"
         fi
         ;;

type/__netbox_uwsgi/man.rst

@@ -15,6 +15,19 @@ protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This
 application is available via the `uwsgi-netbox` systemd service. It is
 controllable via the `netbox` wrapper service, too.
 
+**As uWSGI will be started as netbox user, it does not have privileges to
+bind to a privileaged port (all ports below 1024).** Because uWSGI will
+drop privileages anyway before binding to a port, solutions are to use
+the systemd sockets to activate the ports as root or set linux kernel
+capabilites to bind to such a privileaged port.
+
+As systemd sockets (or uwsgi itself) do not allow to distinguish multiple
+sockets if different protocols are used for different sockets, this type does
+not use systemd sockets if it is requested from the user. Using the
+``--bind-to`` and ``--protocol`` parameters, it uses the systemd socket
+activation. Else, it set the different sockets and protocols natively to uwsgi
+and add kernel capabilities to be able to listen to privileaged ports.
+
 
 REQUIRED PARAMETERS
 -------------------
@@ -39,8 +52,18 @@ state
 
 
 bind-to
-    The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol.
-    Defaults to ``127.0.0.1:3031``. Can be set multiple times.
+    The socket uwsgi should bind to. Must be UNIX/TCP (or anything that
+    systemd sockets accept as stream). Defaults to ``127.0.0.1:3031``. Can be
+    set multiple times. The used protocol is defined by ``--protocol``.
+
+    **By setting up the socket via this parameter, it uses systemd sockets to
+    handle these.** This parameter will be ignored if a more detailed paramter
+    is given (``--$proto-bind``).
+
+protocol
+    The protocol which should be used for the socket given by the ``--bind-to``
+    parameter. Possible values are ``uwsgi``, ``http``, ``fastcgi`` and
+    ``scgi``. If nothing given, it defaults to ``uwsgi``.
 
 uwsgi-bind
 http-bind
@@ -50,6 +73,12 @@ scgi-bind
     ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must
     be a UNIX/TCP socket. Can be set multiple times.
 
+    **By using such parameters instead of ``--bind-to``, no systemd sockets
+    will be used because it can not handle sockets for multiple protocols.**
+    Instead, the native socket binding will be used. It will add kernel
+    capabilites to bind to privileaged ports, too. This allow binds to ports
+    like 80 as netbox user.
+
 
 BOOLEAN PARAMETERS
 ------------------

type/__netbox_uwsgi/manifest

@@ -25,15 +25,30 @@ case "$param_state" in
 esac
 
 
+mkdir "$__object/files"
+
+# check if systemd sockets will be used
+if [ -f "$__object/parameter/bind-to" ]; then
+    SYSTEMD_SOCKET="yes"
+fi
+if find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print -quit | grep -q .; then
+    SYSTEMD_SOCKET="no"
+fi
+echo "$SYSTEMD_SOCKET" > "$__object/files/systemd_socket"
+
 if [ "$state" = "present" ]; then
-    # *bind* parameters are directly processed in the gen script
+    # already checked outside this if-clause
+    export SYSTEMD_SOCKET
+
+    PROTOCOL="$(cat "$__object/parameter/protocol")"
+    export PROTOCOL
+
     if [ -f "$__object/parameter/serve-static" ]; then
         STATIC_MAP="yes"
         export STATIC_MAP
     fi
 
     # process template
-    mkdir "$__object/files"
     "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini"
 
     # uwsgi config file
@@ -48,7 +63,24 @@ else
 fi
 
 
+# handle the systemd socket
+if [ "$SYSTEMD_SOCKET" = "yes" ]; then
+    TYPE="uWSGI"
+    export TYPE
+
+    # generate and set the socket unit
+    "$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \
+        > "$__object/files/netbox.socket"
+    __systemd_unit uwsgi-netbox.socket \
+        --state "$state" --enablement-state "$unit_state" \
+        --source "$__object/files/netbox.socket" --restart
+else
+    # remove the systemd socket unit
+    __systemd_unit uwsgi-netbox.socket --state absent
+fi
+
 # install service file
+"$__type/files/netbox.service.sh" > "$__object/files/netbox.service"
 __systemd_unit uwsgi-netbox.service \
     --state "$state" --enablement-state "$unit_state" \
-    --source "$__type/files/netbox.service" --restart
+    --source "$__object/files/netbox.service" --restart

type/__netbox_uwsgi/parameter/default/protocol

@@ -0,0 +1 @@
+uwsgi

type/__netbox_uwsgi/parameter/optional

@@ -1 +1,2 @@
 state
+protocol