From e0a1b4f66343415b002c6d1c86114da3c9d402f8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 20:01:23 +0200 Subject: [PATCH] __netbox_gunicorn: extract gunicorn setup from __netbox Because someone *want* to use something other than just gunicorn, it was extracted to a own type. Because gunicorn is a bit deep in the netbox installation process, it's a bit harder to isolate it. `__netbox_uwsgi` will come, too. --- type/__netbox/gencode-remote | 8 +- type/__netbox/manifest | 14 +--- type/__netbox_gunicorn/explorer/installed | 4 + .../explorer/should_installed | 3 + .../files/gunicorn.py.sh} | 19 ++++- .../files/netbox.service | 3 +- type/__netbox_gunicorn/gencode-remote | 36 +++++++++ type/__netbox_gunicorn/man.rst | 75 +++++++++++++++++++ type/__netbox_gunicorn/manifest | 20 +++++ .../parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + type/__netbox_gunicorn/singleton | 0 12 files changed, 168 insertions(+), 16 deletions(-) create mode 100755 type/__netbox_gunicorn/explorer/installed create mode 100755 type/__netbox_gunicorn/explorer/should_installed rename type/{__netbox/files/gunicorn.py => __netbox_gunicorn/files/gunicorn.py.sh} (56%) mode change 100644 => 100755 rename type/{__netbox => __netbox_gunicorn}/files/netbox.service (88%) create mode 100755 type/__netbox_gunicorn/gencode-remote create mode 100644 type/__netbox_gunicorn/man.rst create mode 100755 type/__netbox_gunicorn/manifest create mode 100644 type/__netbox_gunicorn/parameter/default/bind-to create mode 100644 type/__netbox_gunicorn/parameter/optional_multiple create mode 100644 type/__netbox_gunicorn/singleton diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index abe7ffa..af4d8c8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -35,6 +35,7 @@ rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' mkdir '$install_dir/cdist' +cp '$src/requirements.txt' /opt/netbox/ # backup dependency info cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' @@ -43,7 +44,9 @@ cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Install python dependencies. -/opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" +# avoid gunicorn, because it will be done in an other type +grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ + | xargs /opt/netbox/venv/bin/pip3 install EOF if [ -f "$__object/parameter/ldap-server" ]; then @@ -94,8 +97,7 @@ fi # check for changes if [ "$changes" = "yes" ]; then cat << EOF -# Restart services. -service netbox restart +# Restart service. service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2f5cf4e..f0da2a3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,13 +204,7 @@ $SECRET_KEY SECRET -# Upload systemd units and gunicorn configuration. -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done - -# Python worker configuration. -require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" +# Upload systemd unit for worker service +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled diff --git a/type/__netbox_gunicorn/explorer/installed b/type/__netbox_gunicorn/explorer/installed new file mode 100755 index 0000000..c6f5d87 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show gunicorn | awk '/Version:/{print $2}' diff --git a/type/__netbox_gunicorn/explorer/should_installed b/type/__netbox_gunicorn/explorer/should_installed new file mode 100755 index 0000000..073be92 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/should_installed @@ -0,0 +1,3 @@ +#!/bin/sh -e + +awk -v FS="==" '$1 ~ /gunicorn/{print $2}' /opt/netbox/requirements.txt diff --git a/type/__netbox/files/gunicorn.py b/type/__netbox_gunicorn/files/gunicorn.py.sh old mode 100644 new mode 100755 similarity index 56% rename from type/__netbox/files/gunicorn.py rename to type/__netbox_gunicorn/files/gunicorn.py.sh index c582677..862ba06 --- a/type/__netbox/files/gunicorn.py +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -1,9 +1,23 @@ +#!/bin/sh -e + +# Generates gunicorn config +# see https://docs.gunicorn.org/en/stable/settings.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = '127.0.0.1:8001' +bind = [$HOST ] # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. -workers = 3 +workers = $(( 2*cores + 1 )) # Number of threads per worker process threads = 3 @@ -14,3 +28,4 @@ timeout = 120 # The maximum number of requests a worker can handle before being respawned max_requests = 5000 max_requests_jitter = 500 +EOF diff --git a/type/__netbox/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service similarity index 88% rename from type/__netbox/files/netbox.service rename to type/__netbox_gunicorn/files/netbox.service index 18eb045..475ddc4 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,5 +1,5 @@ [Unit] -Description=NetBox WSGI Service +Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ After=network-online.target Wants=network-online.target @@ -19,4 +19,5 @@ RestartSec=30 PrivateTmp=true [Install] +Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote new file mode 100755 index 0000000..6958976 --- /dev/null +++ b/type/__netbox_gunicorn/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh -e + +curr_installed="$(cat "$__object/explorer/installed")" +should_installed="$(cat "$__object/explorer/should_installed")" + +# gunicorn version change +if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" + + do_restart=yes + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" +fi + +# configuration changes +if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configuration\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no message cause it should be obvious +fi + + +# restart gunicorn +if [ "$do_restart"]; then + cat << EOF +# Restart service +service gunicorn-netbox restart +EOF + echo restarted >> "$__messages_out" +fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst new file mode 100644 index 0000000..38ac4a5 --- /dev/null +++ b/type/__netbox_gunicorn/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_gunicorn - run netbox with gunicorn + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The hosts the gunicorn socket should be bind to. Formats are `IP`, + `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple + times. Defaults to ``127.0.0.1:8001``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +updated $old to $new + The version of the gunicorn software was updated from `$old` to `$new`. + +configuration + Configuration for gunicorn changed. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_gunicorn + + # with arguments + __netbox $args + require="__netbox" __netbox_gunicorn \ + --bind-to 0.0.0.0:8001 \ + --bind-to 1.2.3.4:5678 + + +SEE ALSO +-------- +:strong:`__netbox`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest new file mode 100755 index 0000000..dab8181 --- /dev/null +++ b/type/__netbox_gunicorn/manifest @@ -0,0 +1,20 @@ +#!/bin/sh -e + +HOST="" +while read -r host; do + HOST="$HOST '$host'," +done < "$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + +# gunicorn config file +__file /opt/netbox/gunicorn.py \ + --mode 644 --source "$__object/files/gunicorn.py" + +# install service file +__systemd_unit gunicorn-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_gunicorn/parameter/default/bind-to b/type/__netbox_gunicorn/parameter/default/bind-to new file mode 100644 index 0000000..f4c980e --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:8001 diff --git a/type/__netbox_gunicorn/parameter/optional_multiple b/type/__netbox_gunicorn/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to diff --git a/type/__netbox_gunicorn/singleton b/type/__netbox_gunicorn/singleton new file mode 100644 index 0000000..e69de29