From efab6ac23ec52e8d5dad8cda9df9b050d66a23f1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 16 Aug 2020 15:18:47 +0200 Subject: [PATCH 01/66] __netbox: add version memorisation This saves the installed version and checks if a new version must be installed. If no update required, it updates the configuration files if required. --- type/__netbox/explorer/version | 5 ++++ type/__netbox/gencode-remote | 42 ++++++++++++++++++++++++++++------ type/__netbox/man.rst | 10 ++++++++ type/__netbox/manifest | 0 4 files changed, 50 insertions(+), 7 deletions(-) create mode 100755 type/__netbox/explorer/version mode change 100644 => 100755 type/__netbox/manifest diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version new file mode 100755 index 0000000..a9ed590 --- /dev/null +++ b/type/__netbox/explorer/version @@ -0,0 +1,5 @@ +#!/bin/sh + +# output version if exist +version_path="/opt/netbox/netbox/cdist/version" +if [ -f "$version_path" ]; then cat "$version_path"; fi diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 122df96..48463f7 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -1,5 +1,8 @@ #!/bin/sh +echo "set -e" + +old_version="$(cat "$__object/explorer/version")" VERSION=$(cat "$__object/parameter/version") src="netbox-$VERSION" @@ -7,9 +10,8 @@ archive="v$VERSION.tar.gz" url="https://github.com/netbox-community/netbox/archive/$archive" install_dir=/opt/netbox/netbox -cat << EOF -set -e - +if [ "$VERSION" != "$old_version" ]; then + cat << EOF # Ensure that coreutils is installed. if [ ! -x \$(which mktemp) ]; then echo "mktemp is not available on the remote host." >&2 @@ -41,11 +43,11 @@ virtualenv /opt/netbox/venv /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" EOF -if [ -f "$__object/parameter/ldap-server" ]; then - echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" -fi + if [ -f "$__object/parameter/ldap-server" ]; then + echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + fi -cat << EOF + cat << EOF # Set final permissions. chown -R netbox /opt/netbox @@ -59,7 +61,33 @@ sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collect cd / rm -r "\$tmpdir" +# Save version after successful installation +printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' +EOF + + # meta + printf "%s %s\n" "installed" "$VERSION" >> "$__messages_out" + changes=yes + +# check if configuration changed +elif grep -q "^__file/opt/netbox/" "$__messages_in"; then + cat <> "$__messages_out" + changes=yes +fi + + +# check for changes +if [ "$changes" = "yes" ]; then + cat < Date: Sun, 16 Aug 2020 15:31:38 +0200 Subject: [PATCH 02/66] __netbox: add manpage explanation of the type --- type/__netbox/man.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 84f1681..e41b099 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -9,7 +9,11 @@ cdist-type__netbox - Install and configure NetBox DESCRIPTION ----------- This (singleton) type installs and configures a NetBox instance, a web -application to help manage and document computer networks +application to help manage and document computer networks. + +It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. +Netbox will be run via `gnuicorn` as WSGI service. It setup systemd unit files +for the services `netbox` and `netbox-rq`. REQUIRED PARAMETERS From 0281b2c8046832d067dfa3be0da8702ef590faf4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 20 Aug 2020 17:53:25 +0200 Subject: [PATCH 03/66] __netbox: no ldap config if no ldap parameters The ldap config file will be empty (only comments) if no parameter name set matches `^ldap-*`. It still generate a ldap config if one or more ldap parameters are set, but not all basic parameters. --- type/__netbox/files/ldap_config.py.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 901d6f5..3e7aaa6 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,5 +1,19 @@ #!/bin/sh +# no configuration if there are no ldap parameters +if ls -1qA "$__object/parameter"/ldap-* | grep -q .; then + # skip + cat << EOF +############################## +# LDAP-backed authentication # +############################## + +# no options set +EOF + exit 0 +fi + + cat << EOF ############################## # LDAP-backed authentication # From a05cb214e17303068649aab3749099e4a06ceef1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 20 Aug 2020 19:37:09 +0200 Subject: [PATCH 04/66] __netbox: fix shellcheck by changing parameter checks The working `ls | grep .` like solution was dropped based on the shellcheck warning. Rather a solution with `find(1)` is used. As alternative, it could be simply checked if the glob succeeds or not by test all given arguments if there are existing files. It was dropped in question if it's really better than the `find(1)` solution. --- type/__netbox/files/ldap_config.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 3e7aaa6..bc494fb 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if ls -1qA "$__object/parameter"/ldap-* | grep -q .; then +if [ "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then # skip cat << EOF ############################## From e6e6dbcd0029639539ce7fe2d8a6fb7d753ad727 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 21 Aug 2020 23:50:24 +0200 Subject: [PATCH 05/66] __netbox: added some netbox configuration options Those who might be useful are added .. --- type/__netbox/files/configuration.py.sh | 64 ++++++++++++------ type/__netbox/man.rst | 65 ++++++++++++++++++- type/__netbox/manifest | 65 +++++++++++++++++++ type/__netbox/parameter/boolean | 4 ++ type/__netbox/parameter/default/basepath | 1 + .../parameter/default/redis-dbid-offset | 1 + type/__netbox/parameter/default/redis-host | 1 + .../__netbox/parameter/default/redis-password | 1 + type/__netbox/parameter/default/redis-port | 1 + .../parameter/default/smtp-from-email | 1 + type/__netbox/parameter/default/smtp-host | 1 + type/__netbox/parameter/default/smtp-password | 1 + type/__netbox/parameter/default/smtp-port | 1 + type/__netbox/parameter/default/smtp-user | 1 + type/__netbox/parameter/optional | 12 ++++ 15 files changed, 200 insertions(+), 20 deletions(-) create mode 100644 type/__netbox/parameter/boolean create mode 100644 type/__netbox/parameter/default/basepath create mode 100644 type/__netbox/parameter/default/redis-dbid-offset create mode 100644 type/__netbox/parameter/default/redis-host create mode 100644 type/__netbox/parameter/default/redis-password create mode 100644 type/__netbox/parameter/default/redis-port create mode 100644 type/__netbox/parameter/default/smtp-from-email create mode 100644 type/__netbox/parameter/default/smtp-host create mode 100644 type/__netbox/parameter/default/smtp-password create mode 100644 type/__netbox/parameter/default/smtp-port create mode 100644 type/__netbox/parameter/default/smtp-user diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index b7e1597..e621845 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -29,26 +29,26 @@ DATABASE = { # to use two separate database IDs. REDIS = { 'tasks': { - 'HOST': 'localhost', - 'PORT': 6379, + 'HOST': '$REDIS_HOST', + 'PORT': $REDIS_PORT, # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], # 'SENTINEL_SERVICE': 'netbox', - 'PASSWORD': '', - 'DATABASE': 0, + 'PASSWORD': '$REDIS_PASSWORD', + 'DATABASE': $((REDIS_DBID_OFFSET + 0)), 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'SSL': $REDIS_SSL, }, 'caching': { - 'HOST': 'localhost', - 'PORT': 6379, + 'HOST': '$REDIS_HOST', + 'PORT': $REDIS_PORT, # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], # 'SENTINEL_SERVICE': 'netbox', - 'PASSWORD': '', - 'DATABASE': 1, + 'PASSWORD': '$REDIS_PASSWORD', + 'DATABASE': $((REDIS_DBID_OFFSET + 1)), 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'SSL': $REDIS_SSL, } } @@ -86,7 +86,7 @@ BANNER_LOGIN = '' # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: # BASE_PATH = 'netbox/' -BASE_PATH = '' +BASE_PATH = '$BASEPATH' # Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) CACHE_TIMEOUT = 900 @@ -112,14 +112,14 @@ DEBUG = False # Email settings EMAIL = { - 'SERVER': 'localhost', - 'PORT': 25, - 'USERNAME': '', - 'PASSWORD': '', - 'USE_SSL': False, - 'USE_TLS': False, + 'SERVER': '$SMTP_HOST', + 'PORT': $SMTP_PORT, + 'USERNAME': '$SMTP_USER', + 'PASSWORD': '$SMTP_PASSWORD', + 'USE_SSL': $SMTP_USE_SSL, + 'USE_TLS': $SMTP_USE_TLS, 'TIMEOUT': 10, # seconds - 'FROM_EMAIL': '', + 'FROM_EMAIL': '$SMTP_FROM_EMAIL', } # Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table @@ -134,12 +134,38 @@ EXEMPT_VIEW_PERMISSIONS = [ # 'ipam.prefix', ] +EOF + +if [ "$HTTP_PROXY" != "" ] || [ "$HTTPS_PROXY" != "" ]; then + cat << EOF +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +HTTP_PROXIES = { +EOF + if [ "$HTTP_PROXY" != "" ]; then + cat << EOF + 'http': '$HTTP_PROXY', +EOF + fi + if [ "$HTTPS_PROXY" != "" ]; then + cat << EOF + 'https': '$HTTPS_PROXY', +EOF + fi + cat << EOF +} +EOF + +else + cat << EOF # HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). # HTTP_PROXIES = { # 'http': 'http://10.10.1.10:3128', # 'https': 'http://10.10.1.10:1080', # } +EOF +fi +cat << EOF # IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing # NetBox from an internal IP. INTERNAL_IPS = ('127.0.0.1', '::1') @@ -150,7 +176,7 @@ LOGGING = {} # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users # are permitted to access most data in NetBox (excluding secrets) but not make any changes. -LOGIN_REQUIRED = False +LOGIN_REQUIRED = $LOGIN_REQUIRED # The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to # re-authenticate. (Default: 1209600 [14 days]) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index e41b099..3e607b5 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -58,9 +58,62 @@ ldap-require-group ldap-superuser-group Make members of this groups superusers. +redis-host + Redis database hostname. Defaults to ``localhost``. + +redis-port + Redis database port. Defaults to ``6379``. + +redis-password + Redis password. Defaults to empty password. + +redis-dbid-offset + Offset to set the redis database id's. The `tasks` database id is `offset + 0` + and `caching` is `offset + 1`. The offset defaults to ``0``. + +smtp-host + Host of the SMTP email server. Defaults to ``localhost``. + +smtp-port + Port of the SMTP email server. Defaults to ``25``. + +smtp-user + Username to access the SMTP email server. Defaults to empty. + +smtp-password + Password to access the SMTP email server. Defaults to empty. + +smtp-from-email + Email from which NetBox will be sent of. Defaults to empty. + +basepath + Base URL path if accessing netbox within a directory instead of directly the + webroot ``/``. For example, if installed at https://example.com/netbox/, set + the value ``netbox/``. + +http-proxy +https-proxy + Proxy which will be used with any HTTP request like webhooks. + BOOLEAN PARAMETERS ------------------ -None. +redis-ssl + Enables a secure TLS/SSL connection to the redis database. By default, ssl + is disabled. + +smtp-use-tls + Uses TLS to connect to the SMTP email server. `See documentation + ` + SEE ALSO -------- - `NetBox documentation `_ diff --git a/type/__netbox/manifest b/type/__netbox/manifest index fc9192c..1f7f8ed 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -23,6 +23,7 @@ case "$os" in ;; esac + DATABASE_NAME=$(cat "$__object/parameter/database") export DATABASE_NAME DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") @@ -67,6 +68,70 @@ if [ -f "$__object/parameter/ldap-superuser-group" ]; then export LDAP_SUPERUSER_GROUP fi +# have default values +REDIS_HOST="$(cat "$__object/parameter/redis-host")" +export REDIS_HOST +REDIS_PORT="$(cat "$__object/parameter/redis-port")" +export REDIS_PORT +REDIS_PASSWORD="$(cat "$__object/parameter/redis-password")" +export REDIS_PASSWORD +REDIS_DBID_OFFSET="$(cat "$__object/parameter/redis-dbid-offset")" +export REDIS_DBID_OFFSET +if [ -f "$__object/parameter/redis-ssl" ]; then + REDIS_SSL="True" +else + REDIS_SSL="False" +fi +export REDIS_SSL + +SMTP_HOST="$(cat "$__object/parameter/smtp-host")" +export SMTP_HOST +SMTP_PORT="$(cat "$__object/parameter/smtp-port")" +export SMTP_PORT +SMTP_USER="$(cat "$__object/parameter/smtp-user")" +export SMTP_USER +SMTP_PASSWORD="$(cat "$__object/parameter/smtp-password")" +export SMTP_PASSWORD +SMTP_FROM_EMAIL="$(cat "$__object/parameter/smtp-from-email")" +export SMTP_FROM_EMAIL + +if [ -f "$__object/parameter/smtp-use-ssl" ]; then + SMTP_USE_SSL="True" +else + SMTP_USE_SSL="False" +fi +export SMTP_USE_SSL +if [ -f "$__object/parameter/smtp-use-tls" ]; then + if [ "$SMTP_USE_SSL" = "True" ]; then + echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" + exit 2 + fi + SMTP_USE_TLS="True" +else + SMTP_USE_TLS="False" +fi +export SMTP_USE_TLS + +BASEPATH="$(cat "$__object/parameter/basepath")" +export BASEPATH + +if [ -f "$__object/parameter/http-proxy" ]; then + HTTP_PROXY=$(cat "$__object/parameter/http-proxy") + export HTTP_PROXY +fi +if [ -f "$__object/parameter/https-proxy" ]; then + HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") + export HTTPS_PROXY +fi + +if [ -f "$__object/parameter/login-required" ]; then + LOGIN_REQUIRED="True" +else + LOGIN_REQUIRED="False" +fi +export LOGIN_REQUIRED + + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/boolean b/type/__netbox/parameter/boolean new file mode 100644 index 0000000..a94808c --- /dev/null +++ b/type/__netbox/parameter/boolean @@ -0,0 +1,4 @@ +redis-ssl +smtp-use-ssl +smtp-use-tls +login-required diff --git a/type/__netbox/parameter/default/basepath b/type/__netbox/parameter/default/basepath new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/basepath @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/redis-dbid-offset b/type/__netbox/parameter/default/redis-dbid-offset new file mode 100644 index 0000000..573541a --- /dev/null +++ b/type/__netbox/parameter/default/redis-dbid-offset @@ -0,0 +1 @@ +0 diff --git a/type/__netbox/parameter/default/redis-host b/type/__netbox/parameter/default/redis-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/redis-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/redis-password b/type/__netbox/parameter/default/redis-password new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/redis-password @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/redis-port b/type/__netbox/parameter/default/redis-port new file mode 100644 index 0000000..a8c4b8e --- /dev/null +++ b/type/__netbox/parameter/default/redis-port @@ -0,0 +1 @@ +6379 diff --git a/type/__netbox/parameter/default/smtp-from-email b/type/__netbox/parameter/default/smtp-from-email new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-from-email @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/smtp-host b/type/__netbox/parameter/default/smtp-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/smtp-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/smtp-password b/type/__netbox/parameter/default/smtp-password new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-password @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/smtp-port b/type/__netbox/parameter/default/smtp-port new file mode 100644 index 0000000..7273c0f --- /dev/null +++ b/type/__netbox/parameter/default/smtp-port @@ -0,0 +1 @@ +25 diff --git a/type/__netbox/parameter/default/smtp-user b/type/__netbox/parameter/default/smtp-user new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-user @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 290c98b..159c813 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -5,3 +5,15 @@ ldap-user-base ldap-group-base ldap-require-group ldap-superuser-group +redis-host +redis-port +redis-password +redis-dbid-offset +smtp-host +smtp-port +smtp-user +smtp-password +smtp-from-email +basepath +http-proxy +https-proxy From 855843ea34ff896ccc23e08f5e84b6f0bdfb49b5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 09:19:12 +0200 Subject: [PATCH 06/66] __netbox: small code and config handling improvements fixed codestyle and detection if config files need to be copied. --- type/__netbox/gencode-remote | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 48463f7..8533f4a 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -66,26 +66,29 @@ printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' EOF # meta - printf "%s %s\n" "installed" "$VERSION" >> "$__messages_out" + printf "installed %s\n" "$VERSION" >> "$__messages_out" changes=yes # check if configuration changed elif grep -q "^__file/opt/netbox/" "$__messages_in"; then - cat <> "$__messages_out" + printf "configuration\n" >> "$__messages_out" changes=yes fi # check for changes if [ "$changes" = "yes" ]; then - cat < Date: Sat, 22 Aug 2020 14:40:07 +0200 Subject: [PATCH 07/66] __netbox: note + link to find NetBox versions --- type/__netbox/man.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 3e607b5..d91c6e8 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -19,7 +19,9 @@ for the services `netbox` and `netbox-rq`. REQUIRED PARAMETERS ------------------- version - NetBox version to be installed. + NetBox version to be installed. You can find the correct and newest version + on GitHub at the NetBox project page under + "`Releases `_". database PostgreSQL database name. From 987393942b9acc10e743c60b264f08dc203877bb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 15:53:24 +0200 Subject: [PATCH 08/66] __netbox: add more database parameters --- type/__netbox/files/configuration.py.sh | 4 ++-- type/__netbox/man.rst | 9 +++++++++ type/__netbox/manifest | 7 +++++++ type/__netbox/parameter/default/database-host | 1 + type/__netbox/parameter/default/database-port | 1 + type/__netbox/parameter/optional | 2 ++ type/__netbox/parameter/required | 1 + 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 type/__netbox/parameter/default/database-host create mode 100644 type/__netbox/parameter/default/database-port diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index e621845..65b00f5 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -19,8 +19,8 @@ DATABASE = { 'NAME': '$DATABASE_NAME', # Database name 'USER': '$DATABASE_USER', # PostgreSQL username 'PASSWORD': '$DATABASE_PASSWORD', # PostgreSQL password - 'HOST': 'localhost', # Database server - 'PORT': '', # Database port (leave blank for default) + 'HOST': '$DATABASE_HOST', # Database server + 'PORT': '$DATABASE_PORT', # Database port (leave blank for default) 'CONN_MAX_AGE': 300, # Max database connection age } diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index d91c6e8..a09b13a 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -26,6 +26,9 @@ version database PostgreSQL database name. +database-user + PostgreSQL database user. + database-password PostgreSQL database password. @@ -39,6 +42,12 @@ host OPTIONAL PARAMETERS ------------------- +database-host + PostgreSQL database hostname. Defaults to ``localhost``. + +database-port + PostgreSQL database port. Defaults to empty (uses the default port). + ldap-server LDAP server URI. Enables LDAP-backed authentication if specified. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 1f7f8ed..45c8afc 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -26,8 +26,15 @@ esac DATABASE_NAME=$(cat "$__object/parameter/database") export DATABASE_NAME +DATABASE_USER="$(cat "$__object/parameter/database-user")" +export DATABASE_USER DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") export DATABASE_PASSWORD +DATABASE_HOST="$(cat "$__object/parameter/database-host")" +export DATABASE_HOST +DATABASE_PORT="$(cat "$__object/parameter/database-port")" +export DATABASE_PORT + ALLOWED_HOST=$(cat "$__object/parameter/host") export ALLOWED_HOST SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox/parameter/default/database-host b/type/__netbox/parameter/default/database-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/database-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/database-port b/type/__netbox/parameter/default/database-port new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/database-port @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 159c813..d97f961 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,3 +1,5 @@ +database-host +database-port ldap-server ldap-bind-dn ldap-bind-password diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 388048a..84e8e4b 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -1,5 +1,6 @@ version database +database-user database-password secret-key host From 9fb3c7f52ef2a27e7d60d850d1918083c90d84c5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 16:00:15 +0200 Subject: [PATCH 09/66] __netbox: fix manpage indents --- type/__netbox/man.rst | 67 ++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index a09b13a..2e43bbd 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -43,88 +43,89 @@ host OPTIONAL PARAMETERS ------------------- database-host - PostgreSQL database hostname. Defaults to ``localhost``. + PostgreSQL database hostname. Defaults to ``localhost``. database-port - PostgreSQL database port. Defaults to empty (uses the default port). + PostgreSQL database port. Defaults to empty (uses the default port). ldap-server - LDAP server URI. Enables LDAP-backed authentication if specified. + LDAP server URI. Enables LDAP-backed authentication if specified. ldap-bind-dn - DN for the NetBox service account. Required for LDAP authentication. + DN for the NetBox service account. Required for LDAP authentication. ldap-bind-password - Password for the NetBox service account. Required for LDAP authentication. + Password for the NetBox service account. Required for LDAP authentication. ldap-user-base - Base used for searching user entries. Required for LDAP authentication. + Base used for searching user entries. Required for LDAP authentication. ldap-group-base - Base used for searching group entries. + Base used for searching group entries. ldap-require-group - Group required to login. + Group required to login. ldap-superuser-group - Make members of this groups superusers. + Make members of this groups superusers. redis-host - Redis database hostname. Defaults to ``localhost``. + Redis database hostname. Defaults to ``localhost``. redis-port - Redis database port. Defaults to ``6379``. + Redis database port. Defaults to ``6379``. redis-password - Redis password. Defaults to empty password. + Redis password. Defaults to empty password. redis-dbid-offset - Offset to set the redis database id's. The `tasks` database id is `offset + 0` - and `caching` is `offset + 1`. The offset defaults to ``0``. + Offset to set the redis database id's. The `tasks` database id is + `offset + 0` and `caching` is `offset + 1`. The offset defaults + to ``0``. smtp-host - Host of the SMTP email server. Defaults to ``localhost``. + Host of the SMTP email server. Defaults to ``localhost``. smtp-port - Port of the SMTP email server. Defaults to ``25``. + Port of the SMTP email server. Defaults to ``25``. smtp-user - Username to access the SMTP email server. Defaults to empty. + Username to access the SMTP email server. Defaults to empty. smtp-password - Password to access the SMTP email server. Defaults to empty. + Password to access the SMTP email server. Defaults to empty. smtp-from-email - Email from which NetBox will be sent of. Defaults to empty. + Email from which NetBox will be sent of. Defaults to empty. basepath - Base URL path if accessing netbox within a directory instead of directly the - webroot ``/``. For example, if installed at https://example.com/netbox/, set - the value ``netbox/``. + Base URL path if accessing netbox within a directory instead of directly the + webroot ``/``. For example, if installed at https://example.com/netbox/, set + the value ``netbox/``. http-proxy https-proxy - Proxy which will be used with any HTTP request like webhooks. + Proxy which will be used with any HTTP request like webhooks. BOOLEAN PARAMETERS ------------------ redis-ssl - Enables a secure TLS/SSL connection to the redis database. By default, ssl - is disabled. + Enables a secure TLS/SSL connection to the redis database. By default, ssl + is disabled. smtp-use-tls - Uses TLS to connect to the SMTP email server. `See documentation - Date: Sat, 22 Aug 2020 20:55:35 +0200 Subject: [PATCH 10/66] __netbox: auto-install sudo as dependency There are may be systems does not have `sudo` preinstalled like debian-minimal. Else, `su` could be used as alternative. --- type/__netbox/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 45c8afc..3033c4d 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -6,7 +6,7 @@ case "$os" in debian|ubuntu) # Install netbox dependencies. for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv; do + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do __package $pkg done From ae7ffac6bbf17e64b6ed79768f6db944395672dd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:29:30 +0200 Subject: [PATCH 11/66] __netbox: force virtualenv to be python3 Set a flag to be sure it is python3. Else, you may get an ugly error if python2 is not installed (and netbox requires python3.6 at minimum). --- type/__netbox/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 8533f4a..9359252 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -37,7 +37,8 @@ cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. -virtualenv /opt/netbox/venv +# forcing python3 to be sure (till python4 gets released ..) +virtualenv -p python3 /opt/netbox/venv # Install python dependencies. /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" From ce47cec2e7883723d8b11efda1f97090ac95cd8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:50:54 +0200 Subject: [PATCH 12/66] __netbox: silence curl progress bar .. but still showing errors if any. --- type/__netbox/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 9359252..e0eb09d 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -23,7 +23,7 @@ tmpdir=\$(mktemp -d) cd "\$tmpdir" # Download and extract sources. -curl -L '$url' > '$archive' +curl -sS -L '$url' > '$archive' tar xf '$archive' # Save cdist-upload configuration file. From 4a04a2c4728ef0625bc49a039ece3240058159f0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:58:44 +0200 Subject: [PATCH 13/66] __netbox: repalace tabs with whitespaces Doing `s/\t/ /g` to have all times the same indents. --- type/__netbox/files/ldap_config.py.sh | 14 ++--- type/__netbox/man.rst | 20 +++--- type/__netbox/manifest | 88 +++++++++++++-------------- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index bc494fb..cf4ed92 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -43,7 +43,7 @@ AUTH_LDAP_USER_ATTR_MAP = { EOF if [ "$LDAP_GROUP_BASE" != "" ]; then - cat << EOF + cat << EOF # This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group # hierarchy. @@ -55,21 +55,21 @@ AUTH_LDAP_GROUP_TYPE = PosixGroupType() AUTH_LDAP_MIRROR_GROUPS = True EOF - if [ "$LDAP_REQUIRE_GROUP" != "" ]; then - cat << EOF + if [ "$LDAP_REQUIRE_GROUP" != "" ]; then + cat << EOF # Define a group required to login. AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" EOF - fi + fi - if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then - cat << EOF + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + cat << EOF # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { "is_superuser": "$LDAP_SUPERUSER_GROUP", } EOF - fi + fi fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 2e43bbd..4e63ccc 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -144,16 +144,16 @@ EXAMPLES .. code-block:: sh __netbox --version 2.8.7 --database netbox \ - --database-password "secretsecretsecret" \ - --secret-key "secretsecretsecret" \ - --host "${__target_host:?}" \ - --ldap-server "ldaps://ldap.domain.tld" \ - --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ - --ldap-bind-password "secretsecretsecret" \ - --ldap-user-base "ou=users,dc=domain,dc=tld" \ - --ldap-group-base "ou=groups,dc=domain,dc=tld" \ - --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ - --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + --database-password "secretsecretsecret" \ + --secret-key "secretsecretsecret" \ + --host "${__target_host:?}" \ + --ldap-server "ldaps://ldap.domain.tld" \ + --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ + --ldap-bind-password "secretsecretsecret" \ + --ldap-user-base "ou=users,dc=domain,dc=tld" \ + --ldap-group-base "ou=groups,dc=domain,dc=tld" \ + --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ + --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" NOTES diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 3033c4d..21a9bc6 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -3,24 +3,24 @@ os=$(cat "$__global/explorer/os") case "$os" in - debian|ubuntu) - # Install netbox dependencies. - for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do - __package $pkg - done + debian|ubuntu) + # Install netbox dependencies. + for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do + __package $pkg + done - if [ -f "$__object/parameter/ldap-server" ]; then - for pkg in libldap2-dev libsasl2-dev libssl-dev; do - __package $pkg - done - fi - ;; - *) - printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; + if [ -f "$__object/parameter/ldap-server" ]; then + for pkg in libldap2-dev libsasl2-dev libssl-dev; do + __package $pkg + done + fi + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; esac @@ -41,38 +41,38 @@ SECRET_KEY=$(cat "$__object/parameter/secret-key") export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then - LDAP_SERVER=$(cat "$__object/parameter/ldap-server") - export LDAP_SERVER + LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + export LDAP_SERVER fi if [ -f "$__object/parameter/ldap-bind-dn" ]; then - LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") - export LDAP_BIND_DN + LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + export LDAP_BIND_DN fi if [ -f "$__object/parameter/ldap-bind-password" ]; then - LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") - export LDAP_BIND_PASSWORD + LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + export LDAP_BIND_PASSWORD fi if [ -f "$__object/parameter/ldap-user-base" ]; then - LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") - export LDAP_USER_BASE + LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + export LDAP_USER_BASE fi if [ -f "$__object/parameter/ldap-group-base" ]; then - LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") - export LDAP_GROUP_BASE + LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + export LDAP_GROUP_BASE fi if [ -f "$__object/parameter/ldap-require-group" ]; then - LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") - export LDAP_REQUIRE_GROUP + LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + export LDAP_REQUIRE_GROUP fi if [ -f "$__object/parameter/ldap-superuser-group" ]; then - LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") - export LDAP_SUPERUSER_GROUP + LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + export LDAP_SUPERUSER_GROUP fi # have default values @@ -123,12 +123,12 @@ BASEPATH="$(cat "$__object/parameter/basepath")" export BASEPATH if [ -f "$__object/parameter/http-proxy" ]; then - HTTP_PROXY=$(cat "$__object/parameter/http-proxy") - export HTTP_PROXY + HTTP_PROXY=$(cat "$__object/parameter/http-proxy") + export HTTP_PROXY fi if [ -f "$__object/parameter/https-proxy" ]; then - HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") - export HTTPS_PROXY + HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") + export HTTPS_PROXY fi if [ -f "$__object/parameter/login-required" ]; then @@ -149,23 +149,23 @@ mkdir -p "$__object/files" require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ - --source "$__object/files/configuration.py" + /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ + --source "$__object/files/configuration.py" if [ -f "$__object/parameter/ldap-server" ]; then - require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ - --source "$__object/files/ldap_config.py" + require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + --source "$__object/files/ldap_config.py" fi # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled done # Python worker configuration. require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" + --mode 644 --source "$__type/files/gunicorn.py" From 0b261070d362fed393c9e021654917bb525f8b7a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:04:33 +0200 Subject: [PATCH 14/66] __netbox: man note regards creating a superuser --- type/__netbox/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 4e63ccc..91313be 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -166,6 +166,11 @@ you are welcome to contribute! - `Possible optional settings ` +If you not setup ldap authentification, you may be interested into how to +`setting up a super user +` +directly on the machine to be able to access and use NetBox. + SEE ALSO -------- - `NetBox documentation `_ From 584e0bbda85669de99e907784c17a4d176aae996 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:23:36 +0200 Subject: [PATCH 15/66] __netbox: fix netbox version save The cdist directory needs to be created to write there. This is done directly after the new tarball is installed. --- type/__netbox/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index e0eb09d..d2ec5f9 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -33,6 +33,8 @@ cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' +mkdir '$install_dir/cdist' + cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' From 88ddebd0404a5b0d4adf597ec6b42c61c7240ea5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:47:15 +0200 Subject: [PATCH 16/66] __netbox: fix deprecated configuration option Fixes error message `/opt/netbox/netbox/netbox/settings.py:141: UserWarning: REMOTE_AUTH_DEFAULT_PERMISSIONS should be a dictionary. Backward compatibility will be removed in v2.10.` --- type/__netbox/files/configuration.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 65b00f5..8679437 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -247,7 +247,7 @@ REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] -REMOTE_AUTH_DEFAULT_PERMISSIONS = [] +REMOTE_AUTH_DEFAULT_PERMISSIONS = {} # This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. RELEASE_CHECK_TIMEOUT = 24 * 3600 From f5c8c26db6436777430a31c410c1c442fce184c7 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 11:36:35 +0200 Subject: [PATCH 17/66] __netbox: new config parameters for data paths .. to move them to an other partition. --- type/__netbox/files/configuration.py.sh | 36 ++++++++++++++++--------- type/__netbox/man.rst | 33 +++++++++++++++++++++++ type/__netbox/manifest | 19 +++++++++++++ type/__netbox/parameter/optional | 4 +++ 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 8679437..af19153 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -154,15 +154,6 @@ EOF cat << EOF } EOF - -else - cat << EOF -# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). -# HTTP_PROXIES = { -# 'http': 'http://10.10.1.10:3128', -# 'https': 'http://10.10.1.10:1080', -# } -EOF fi cat << EOF @@ -190,10 +181,18 @@ MAINTENANCE_MODE = False # all objects by specifying "?limit=0". MAX_PAGE_SIZE = 1000 +EOF + +if [ "$MEDIA_ROOT" != "" ]; then + cat << EOF # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that # the default value of this setting is derived from the installed location. -# MEDIA_ROOT = '/opt/netbox/netbox/media' +MEDIA_ROOT = '$MEDIA_ROOT' +EOF +fi + +cat << EOF # By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the # class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: # STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' @@ -257,14 +256,27 @@ RELEASE_CHECK_TIMEOUT = 24 * 3600 RELEASE_CHECK_URL = None # RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' +EOF + +if [ "$REPORTS_ROOT" != "" ]; then + cat << EOF # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -# REPORTS_ROOT = '/opt/netbox/netbox/reports' +REPORTS_ROOT = '$REPORTS_ROOT' +EOF +fi + +if [ "$SCRIPTS_ROOT" != "" ]; then + cat << EOF # The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -# SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' +SCRIPTS_ROOT = '$SCRIPTS_ROOT' +EOF +fi + +cat << EOF # By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use # local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 91313be..c910d5c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -107,6 +107,39 @@ http-proxy https-proxy Proxy which will be used with any HTTP request like webhooks. +data-root + This parameter set's the media, reports and scripts root to subdirectories + of the given directory. Values can be overwritten by special parameters like + `--media-root` for example. Use this option if you want to store persistant + data of netbox on an other partition. A trailing shlash is not needed. + + The data directories have following predefined sub-directory names: + + media root: + ``$data_root/media`` + reports root: + ``$data_root/reports`` + scripts root: + ``$data_root/scripts`` + +media-root + The file path to where media files (like image attachments) are stored. + Change this path if you require to store data on an other partiotion. + A trailing slash is not needed. By default, it will be stored into the + installation directory (``/opt/netbox/netbox/netbox/media``). + +reports-root + The file path of where custom reports are kept. Change this path if you + require to store data on an other partition. A trailing slash is not + needed. By default, it will be stored into the installation directory + (``/opt/netbox/netbox/netbox/reports``). + +scripts-root + The file path of where custom scripts are kept. Change this path if you + require to store data on an other partition. A trailing slash is not + needed. By default, it will be stored into the installation directory + (``/opt/netbox/netbox/netbox/scripts``). + BOOLEAN PARAMETERS ------------------ redis-ssl diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 21a9bc6..133b527 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -138,6 +138,25 @@ else fi export LOGIN_REQUIRED +if [ -f "$__object/parameter/data-root" ]; then + data_root="$(cat "$__object/parameter/data-root")" + MEDIA_ROOT="$data_root/media" + REPORTS_ROOT="$data_root/reports" + SCRIPTS_ROOT="$data_root/scripts" +fi +if [ -f "$__object/parameter/media-root" ]; then + MEDIA_ROOT="$(cat "$__object/parameter/media-root")" +fi +export MEDIA_ROOT +if [ -f "$__object/parameter/reports-root" ]; then + REPORTS_ROOT="$(cat "$__object/parameter/reports-root")" +fi +export REPORTS_ROOT +if [ -f "$__object/parameter/scripts-root" ]; then + SCRIPTS_ROOT="$(cat "$__object/parameter/scripts-root")" +fi +export SCRIPTS_ROOT + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index d97f961..21fea5f 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -19,3 +19,7 @@ smtp-from-email basepath http-proxy https-proxy +data-root +media-root +reports-root +scripts-root From 49bb527dea8752fe14155516581bd7ee9049143a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 16:52:02 +0200 Subject: [PATCH 18/66] __netbox: autogen secretkey and store at remote The secret key is generated if it is not set via parameter and the explorer does not return any. It will be saved in the netbox home directory to easily read the key for the config generation. --- type/__netbox/explorer/secretkey | 8 ++++++++ type/__netbox/explorer/version | 2 +- type/__netbox/man.rst | 16 +++++++++++----- type/__netbox/manifest | 19 ++++++++++++++++++- type/__netbox/parameter/optional | 1 + type/__netbox/parameter/required | 1 - 6 files changed, 39 insertions(+), 8 deletions(-) create mode 100755 type/__netbox/explorer/secretkey diff --git a/type/__netbox/explorer/secretkey b/type/__netbox/explorer/secretkey new file mode 100755 index 0000000..9f0331c --- /dev/null +++ b/type/__netbox/explorer/secretkey @@ -0,0 +1,8 @@ +#!/bin/sh -e + +# Explorer will output the key if he exists. + +secretkey="/opt/netbox/.secretkey" +if [ -f "$secretkey" ]; then + cat "$secretkey" +fi diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version index a9ed590..5231e16 100755 --- a/type/__netbox/explorer/version +++ b/type/__netbox/explorer/version @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # output version if exist version_path="/opt/netbox/netbox/cdist/version" diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index c910d5c..77b47d0 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -32,16 +32,22 @@ database-user database-password PostgreSQL database password. -secret-key - Random secret key of at least 50 alphanumeric characters. This key must be - unique to this installation and must not be shared outside the local - system. - host Hostname (domain or IP address) on which the application is served. OPTIONAL PARAMETERS ------------------- +secret-key + Random secret key of at least 50 alphanumeric characters and symbols. This + key must be unique to this installation and must not be shared outside the + local system. If no secret key is given, the type generates an own 50 chars + long key and saves it on the remote host to remember it for the next run. + + The secret, random string is used to assist in the creation new + cryptographic hashes for passwords and HTTP cookies. It is not directly + used for hasing user passwords or for encrpted storage. It can be changed + at any time, but will invalidate all existing sessions. + database-host PostgreSQL database hostname. Defaults to ``localhost``. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 133b527..cd47d38 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -37,7 +37,18 @@ export DATABASE_PORT ALLOWED_HOST=$(cat "$__object/parameter/host") export ALLOWED_HOST -SECRET_KEY=$(cat "$__object/parameter/secret-key") + +if [ -f "$__object/parameter/secret-key" ]; then + SECRET_KEY=$(cat "$__object/parameter/secret-key") +elif [ -s "$__object/explorer/secretkey" ]; then + # Can be done over netbox/generate_secret_key.py too, but it's to + # complicated with the variable setup (can't generated right now!). + # Generates a 50-character long key (without ' cause of python quotes) + SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" +else + # take the key that is already used + SECRET_KEY="$(cat "$__object/explorer/secretkey")" +fi export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then @@ -177,6 +188,12 @@ if [ -f "$__object/parameter/ldap-server" ]; then --source "$__object/files/ldap_config.py" fi +# save secret +require="__user/netbox" __file /opt/netbox/.secretkey --mode 400 \ + --owner netbox --source - << SECRET +$SECRET_KEY +SECRET + # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 21fea5f..47722e6 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,3 +1,4 @@ +secret-key database-host database-port ldap-server diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 84e8e4b..3d03e5a 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -2,5 +2,4 @@ version database database-user database-password -secret-key host From 4e90c93eef881a228272167f2e328de9a16bf892 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 17:15:32 +0200 Subject: [PATCH 19/66] __netbox: updated copyright Added me to the copyright after there are some more changes :-) --- type/__netbox/man.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 77b47d0..10485a1 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -217,11 +217,13 @@ SEE ALSO AUTHORS ------- Timothée Floure +Matthias Stecher COPYING ------- -Copyright \(C) 2020 Timothée Floure. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2020 Timothée Floure. +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. From 863c95059e7d5b10607b97b88aa49b402c0561d3 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 19:48:56 +0200 Subject: [PATCH 20/66] __netbox: fix REMOTE_AUTH_BACKEND module not found Updated with the backend from the current upstream configuration. --- type/__netbox/files/configuration.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index af19153..0b8304d 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -242,7 +242,7 @@ RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 # Remote authentication support REMOTE_AUTH_ENABLED = False -REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' +REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] From 95fd7d872ed1c649742008604d4becc88be010a4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 24 Aug 2020 18:57:27 +0200 Subject: [PATCH 21/66] __netbox: fix ldap config short-circuit Forgot to negotate if statement .. works now :/ --- type/__netbox/files/ldap_config.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index cf4ed92..c31f46a 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if [ "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then +if [ -z "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then # skip cat << EOF ############################## From 7d694342fffe1dd2bd850ac38c5fa7580da8fedb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 24 Aug 2020 19:31:49 +0200 Subject: [PATCH 22/66] __netbox: new parameter for automatic release checks --- type/__netbox/files/configuration.py.sh | 14 ++++++++++++-- type/__netbox/man.rst | 8 ++++++-- type/__netbox/manifest | 5 +++++ type/__netbox/parameter/boolean | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 0b8304d..21104b4 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -253,11 +253,21 @@ RELEASE_CHECK_TIMEOUT = 24 * 3600 # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the # version check or use the URL below to check for release in the official NetBox repository. -RELEASE_CHECK_URL = None -# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' EOF +if [ "$UPDATE_CHECK" != "" ]; then + cat << EOF +RELEASE_CHECK_URL = None + +EOF +else + cat << EOF +RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +EOF +fi + if [ "$REPORTS_ROOT" != "" ]; then cat << EOF # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 10485a1..761494a 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -163,8 +163,12 @@ smtp-use-ssl for more information. login-required - Sets if a login is required to access all sites. By default, anounymous users - can see most data (excluding secrets) but not make any changes. + Sets if a login is required to access all sites. By default, anonymous + users can see most data (excluding secrets) but not make any changes. + +update-notify + Enables the NetBox version check for new upstream updates. It checks every + 24 hours for new releases and notify the admin users in the gui if any. MESSAGES -------- diff --git a/type/__netbox/manifest b/type/__netbox/manifest index cd47d38..62da0a2 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -168,6 +168,11 @@ if [ -f "$__object/parameter/scripts-root" ]; then fi export SCRIPTS_ROOT +if [ -f "$__object/parameter/update-notify" ]; then + UPDATE_CHECK="yes" + export UPDATE_CHECK +fi + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/boolean b/type/__netbox/parameter/boolean index a94808c..d568037 100644 --- a/type/__netbox/parameter/boolean +++ b/type/__netbox/parameter/boolean @@ -2,3 +2,4 @@ redis-ssl smtp-use-ssl smtp-use-tls login-required +update-notify From 8ff7a4616dcedac333b5e93d994cbff2887ffbe9 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 25 Aug 2020 19:30:32 +0200 Subject: [PATCH 23/66] __netbox: allow multiple hostnames --- type/__netbox/files/configuration.py.sh | 2 +- type/__netbox/man.rst | 2 ++ type/__netbox/manifest | 8 ++++++-- type/__netbox/parameter/required | 1 - type/__netbox/parameter/required_multiple | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 type/__netbox/parameter/required_multiple diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 21104b4..4b7be11 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -11,7 +11,7 @@ cat << EOF # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. # # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] -ALLOWED_HOSTS = [ '$ALLOWED_HOST' ] +ALLOWED_HOSTS = [$ALLOWED_HOSTS ] # PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: # https://docs.djangoproject.com/en/stable/ref/settings/#databases diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 761494a..2af2bef 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -34,6 +34,7 @@ database-password host Hostname (domain or IP address) on which the application is served. + Multiple hostnames are possible; given as multiple arguments. OPTIONAL PARAMETERS ------------------- @@ -190,6 +191,7 @@ EXAMPLES --database-password "secretsecretsecret" \ --secret-key "secretsecretsecret" \ --host "${__target_host:?}" \ + --host "cool-netbox.xyz" \ --ldap-server "ldaps://ldap.domain.tld" \ --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ --ldap-bind-password "secretsecretsecret" \ diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 62da0a2..2f5cf4e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -35,8 +35,12 @@ export DATABASE_HOST DATABASE_PORT="$(cat "$__object/parameter/database-port")" export DATABASE_PORT -ALLOWED_HOST=$(cat "$__object/parameter/host") -export ALLOWED_HOST +# list of hosts +ALLOWED_HOSTS="" +while read -r hostname; do + ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," +done < "$__object/parameter/host" +export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 3d03e5a..02fca9f 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -2,4 +2,3 @@ version database database-user database-password -host diff --git a/type/__netbox/parameter/required_multiple b/type/__netbox/parameter/required_multiple new file mode 100644 index 0000000..c70dc2d --- /dev/null +++ b/type/__netbox/parameter/required_multiple @@ -0,0 +1 @@ +host From eaf639790b00bdb5d7d9197939d0e6fa7963f3a2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 17:39:19 +0200 Subject: [PATCH 24/66] __netbox: remove ldap pip3 package if not required --- type/__netbox/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d2ec5f9..abe7ffa 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -48,6 +48,8 @@ EOF if [ -f "$__object/parameter/ldap-server" ]; then echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + else + echo "/opt/netbox/venv/bin/pip3 uninstall django-auth-ldap" fi cat << EOF From e0a1b4f66343415b002c6d1c86114da3c9d402f8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 20:01:23 +0200 Subject: [PATCH 25/66] __netbox_gunicorn: extract gunicorn setup from __netbox Because someone *want* to use something other than just gunicorn, it was extracted to a own type. Because gunicorn is a bit deep in the netbox installation process, it's a bit harder to isolate it. `__netbox_uwsgi` will come, too. --- type/__netbox/gencode-remote | 8 +- type/__netbox/manifest | 14 +--- type/__netbox_gunicorn/explorer/installed | 4 + .../explorer/should_installed | 3 + .../files/gunicorn.py.sh} | 19 ++++- .../files/netbox.service | 3 +- type/__netbox_gunicorn/gencode-remote | 36 +++++++++ type/__netbox_gunicorn/man.rst | 75 +++++++++++++++++++ type/__netbox_gunicorn/manifest | 20 +++++ .../parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + type/__netbox_gunicorn/singleton | 0 12 files changed, 168 insertions(+), 16 deletions(-) create mode 100755 type/__netbox_gunicorn/explorer/installed create mode 100755 type/__netbox_gunicorn/explorer/should_installed rename type/{__netbox/files/gunicorn.py => __netbox_gunicorn/files/gunicorn.py.sh} (56%) mode change 100644 => 100755 rename type/{__netbox => __netbox_gunicorn}/files/netbox.service (88%) create mode 100755 type/__netbox_gunicorn/gencode-remote create mode 100644 type/__netbox_gunicorn/man.rst create mode 100755 type/__netbox_gunicorn/manifest create mode 100644 type/__netbox_gunicorn/parameter/default/bind-to create mode 100644 type/__netbox_gunicorn/parameter/optional_multiple create mode 100644 type/__netbox_gunicorn/singleton diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index abe7ffa..af4d8c8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -35,6 +35,7 @@ rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' mkdir '$install_dir/cdist' +cp '$src/requirements.txt' /opt/netbox/ # backup dependency info cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' @@ -43,7 +44,9 @@ cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Install python dependencies. -/opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" +# avoid gunicorn, because it will be done in an other type +grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ + | xargs /opt/netbox/venv/bin/pip3 install EOF if [ -f "$__object/parameter/ldap-server" ]; then @@ -94,8 +97,7 @@ fi # check for changes if [ "$changes" = "yes" ]; then cat << EOF -# Restart services. -service netbox restart +# Restart service. service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2f5cf4e..f0da2a3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,13 +204,7 @@ $SECRET_KEY SECRET -# Upload systemd units and gunicorn configuration. -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done - -# Python worker configuration. -require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" +# Upload systemd unit for worker service +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled diff --git a/type/__netbox_gunicorn/explorer/installed b/type/__netbox_gunicorn/explorer/installed new file mode 100755 index 0000000..c6f5d87 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show gunicorn | awk '/Version:/{print $2}' diff --git a/type/__netbox_gunicorn/explorer/should_installed b/type/__netbox_gunicorn/explorer/should_installed new file mode 100755 index 0000000..073be92 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/should_installed @@ -0,0 +1,3 @@ +#!/bin/sh -e + +awk -v FS="==" '$1 ~ /gunicorn/{print $2}' /opt/netbox/requirements.txt diff --git a/type/__netbox/files/gunicorn.py b/type/__netbox_gunicorn/files/gunicorn.py.sh old mode 100644 new mode 100755 similarity index 56% rename from type/__netbox/files/gunicorn.py rename to type/__netbox_gunicorn/files/gunicorn.py.sh index c582677..862ba06 --- a/type/__netbox/files/gunicorn.py +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -1,9 +1,23 @@ +#!/bin/sh -e + +# Generates gunicorn config +# see https://docs.gunicorn.org/en/stable/settings.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = '127.0.0.1:8001' +bind = [$HOST ] # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. -workers = 3 +workers = $(( 2*cores + 1 )) # Number of threads per worker process threads = 3 @@ -14,3 +28,4 @@ timeout = 120 # The maximum number of requests a worker can handle before being respawned max_requests = 5000 max_requests_jitter = 500 +EOF diff --git a/type/__netbox/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service similarity index 88% rename from type/__netbox/files/netbox.service rename to type/__netbox_gunicorn/files/netbox.service index 18eb045..475ddc4 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,5 +1,5 @@ [Unit] -Description=NetBox WSGI Service +Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ After=network-online.target Wants=network-online.target @@ -19,4 +19,5 @@ RestartSec=30 PrivateTmp=true [Install] +Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote new file mode 100755 index 0000000..6958976 --- /dev/null +++ b/type/__netbox_gunicorn/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh -e + +curr_installed="$(cat "$__object/explorer/installed")" +should_installed="$(cat "$__object/explorer/should_installed")" + +# gunicorn version change +if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" + + do_restart=yes + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" +fi + +# configuration changes +if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configuration\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no message cause it should be obvious +fi + + +# restart gunicorn +if [ "$do_restart"]; then + cat << EOF +# Restart service +service gunicorn-netbox restart +EOF + echo restarted >> "$__messages_out" +fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst new file mode 100644 index 0000000..38ac4a5 --- /dev/null +++ b/type/__netbox_gunicorn/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_gunicorn - run netbox with gunicorn + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The hosts the gunicorn socket should be bind to. Formats are `IP`, + `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple + times. Defaults to ``127.0.0.1:8001``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +updated $old to $new + The version of the gunicorn software was updated from `$old` to `$new`. + +configuration + Configuration for gunicorn changed. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_gunicorn + + # with arguments + __netbox $args + require="__netbox" __netbox_gunicorn \ + --bind-to 0.0.0.0:8001 \ + --bind-to 1.2.3.4:5678 + + +SEE ALSO +-------- +:strong:`__netbox`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest new file mode 100755 index 0000000..dab8181 --- /dev/null +++ b/type/__netbox_gunicorn/manifest @@ -0,0 +1,20 @@ +#!/bin/sh -e + +HOST="" +while read -r host; do + HOST="$HOST '$host'," +done < "$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + +# gunicorn config file +__file /opt/netbox/gunicorn.py \ + --mode 644 --source "$__object/files/gunicorn.py" + +# install service file +__systemd_unit gunicorn-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_gunicorn/parameter/default/bind-to b/type/__netbox_gunicorn/parameter/default/bind-to new file mode 100644 index 0000000..f4c980e --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:8001 diff --git a/type/__netbox_gunicorn/parameter/optional_multiple b/type/__netbox_gunicorn/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to diff --git a/type/__netbox_gunicorn/singleton b/type/__netbox_gunicorn/singleton new file mode 100644 index 0000000..e69de29 From bbce0030aba387232f48a6c4b676c2b82cba28ed Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 21:07:31 +0200 Subject: [PATCH 26/66] __netbox: enable ldap usage via configuration.py The REMOTE_AUTH_BACKEND must be set to use LDAP. It now exports USE_LDAP to generally say if LDAP is being used in the configuration or not. --- type/__netbox/files/configuration.py.sh | 14 ++++++++++++++ type/__netbox/files/ldap_config.py.sh | 2 +- type/__netbox/manifest | 12 ++++++------ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 4b7be11..ea73133 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -240,9 +240,23 @@ PREFER_IPV4 = False RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = 22 RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 +EOF + +if [ "$USE_LDAP" ]; then + cat << EOF +# Remote authentication support with ldap +REMOTE_AUTH_ENABLED = True +REMOTE_AUTH_BACKEND = 'netbox.authentication.LDAPBackend' +EOF +else + cat << EOF # Remote authentication support REMOTE_AUTH_ENABLED = False REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' +EOF +fi + +cat << EOF REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index c31f46a..a49d800 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if [ -z "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then +if [ -z "$USE_LDAP" ]; then # skip cat << EOF ############################## diff --git a/type/__netbox/manifest b/type/__netbox/manifest index f0da2a3..079ea48 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -57,38 +57,38 @@ export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + USE_LDAP=yes export LDAP_SERVER fi - if [ -f "$__object/parameter/ldap-bind-dn" ]; then LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + USE_LDAP=yes export LDAP_BIND_DN fi - if [ -f "$__object/parameter/ldap-bind-password" ]; then LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + USE_LDAP=yes export LDAP_BIND_PASSWORD fi - if [ -f "$__object/parameter/ldap-user-base" ]; then LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + USE_LDAP=yes export LDAP_USER_BASE fi - if [ -f "$__object/parameter/ldap-group-base" ]; then LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") export LDAP_GROUP_BASE fi - if [ -f "$__object/parameter/ldap-require-group" ]; then LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") export LDAP_REQUIRE_GROUP fi - if [ -f "$__object/parameter/ldap-superuser-group" ]; then LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") export LDAP_SUPERUSER_GROUP fi +# export if base ldap parameters are used +export USE_LDAP # have default values REDIS_HOST="$(cat "$__object/parameter/redis-host")" From 1ef4420c537570206780757fa5b78aa6895f5829 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 27 Aug 2020 19:46:09 +0200 Subject: [PATCH 27/66] __netbox_uwsgi: new type to handle uwsgi for netbox --- type/__netbox/gencode-remote | 2 +- type/__netbox/man.rst | 2 +- type/__netbox_gunicorn/gencode-remote | 5 +- type/__netbox_gunicorn/man.rst | 2 +- type/__netbox_uwsgi/explorer/installed | 4 + type/__netbox_uwsgi/explorer/upgradeable | 4 + type/__netbox_uwsgi/files/netbox.service | 23 ++++++ type/__netbox_uwsgi/files/uwsgi.ini.sh | 22 ++++++ type/__netbox_uwsgi/gencode-remote | 35 +++++++++ type/__netbox_uwsgi/man.rst | 75 +++++++++++++++++++ type/__netbox_uwsgi/manifest | 18 +++++ type/__netbox_uwsgi/parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + 13 files changed, 188 insertions(+), 6 deletions(-) create mode 100755 type/__netbox_uwsgi/explorer/installed create mode 100755 type/__netbox_uwsgi/explorer/upgradeable create mode 100644 type/__netbox_uwsgi/files/netbox.service create mode 100755 type/__netbox_uwsgi/files/uwsgi.ini.sh create mode 100755 type/__netbox_uwsgi/gencode-remote create mode 100644 type/__netbox_uwsgi/man.rst create mode 100755 type/__netbox_uwsgi/manifest create mode 100644 type/__netbox_uwsgi/parameter/default/bind-to create mode 100644 type/__netbox_uwsgi/parameter/optional_multiple diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index af4d8c8..0153579 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -89,7 +89,7 @@ EOF fi # meta - printf "configuration\n" >> "$__messages_out" + printf "configured\n" >> "$__messages_out" changes=yes fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 2af2bef..6bb1e36 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -176,7 +176,7 @@ MESSAGES installed $VERSION Netbox was fresh installed or updated. The new version number is appended. -configuration +configured Some configuration files got updated and therefore the service was restarted. This message will not be echoed if configuration got updated due a standard installation. diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 6958976..e424c22 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -16,7 +16,7 @@ fi # configuration changes if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then do_restart=yes - printf "configuration\n" >> "$__messages_out" + printf "configured\n" >> "$__messages_out" fi # application @@ -27,10 +27,9 @@ fi # restart gunicorn -if [ "$do_restart"]; then +if [ "$do_restart" ]; then cat << EOF # Restart service service gunicorn-netbox restart EOF - echo restarted >> "$__messages_out" fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 38ac4a5..43f5864 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -34,7 +34,7 @@ MESSAGES updated $old to $new The version of the gunicorn software was updated from `$old` to `$new`. -configuration +configured Configuration for gunicorn changed. In both cases, and at messages from the `__netbox` type, it restarts the diff --git a/type/__netbox_uwsgi/explorer/installed b/type/__netbox_uwsgi/explorer/installed new file mode 100755 index 0000000..a2393d0 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show uwsgi | awk '/Version:/{print $2}' diff --git a/type/__netbox_uwsgi/explorer/upgradeable b/type/__netbox_uwsgi/explorer/upgradeable new file mode 100755 index 0000000..f4b0a22 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/upgradeable @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print latest version if availble +/opt/netbox/venv/bin/pip3 list --outdated | awk '$1 == "uwsgi" {print $3}' diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service new file mode 100644 index 0000000..cfce752 --- /dev/null +++ b/type/__netbox_uwsgi/files/netbox.service @@ -0,0 +1,23 @@ +[Unit] +Description=Netbox uWSGI WSGI Service +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +PIDFile=/var/tmp/netbox.pid +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/uwsgi --master --wsgi-file netbox/netbox/wsgi.py uwsgi.ini + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +Alias=netbox.service +WantedBy=multi-user.target diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh new file mode 100755 index 0000000..557a5fa --- /dev/null +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# Generates uwsgi config +# see https://uwsgi-docs.readthedocs.io/en/latest/Options.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF +[uwsgi] +; socket to bind +socket = $HOST + +; processes and threads +processes = $(( 2*cores + 1 )) +threads = 2 +EOF diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote new file mode 100755 index 0000000..5cc82d1 --- /dev/null +++ b/type/__netbox_uwsgi/gencode-remote @@ -0,0 +1,35 @@ +#!/bin/sh -e + +# not installed +if ! [ -s "$__object/explorer/installed" ]; then + echo "/opt/netbox/venv/bin/pip3 install uwsgi" + do_restart=yes + printf "installed\n" >> "$__messages_out" + +# updates available +elif [ -s "$__object/explorer/upgradeable" ]; then + echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + do_restart=yes + printf "upgraded\n" >> "$__messages_out" +fi + +# changed configuration +if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no messages cause this is obvious +fi + + +# restart uwsgi +if [ "$do_restart" ]; then + cat << EOF +# Restart service +service uwsgi-netbox restart +EOF +fi diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst new file mode 100644 index 0000000..5bf3160 --- /dev/null +++ b/type/__netbox_uwsgi/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_uwsgi - run netbox with uwsgi + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. + Defaults to ``127.0.0.1:3031``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +installed + The uwsgi service was installed. + +upgraded + The uwsgi service was upgraded. + +configured + The uwsgi configuration got updated. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_uwsgi + + # with special bind + require="__netbox" __netbox_uwsgi --bind-to 0.0.0.0:3032 \ + --bind-to 0.0.0.0:3033 + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest new file mode 100755 index 0000000..779f93b --- /dev/null +++ b/type/__netbox_uwsgi/manifest @@ -0,0 +1,18 @@ +#!/bin/sh -e + +HOST="$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" + +# uwsgi config file +# TODO maybe patching with __key_value cause of .ini ? +__file /opt/netbox/uwsgi.ini \ + --mode 644 --source "$__object/files/uwsgi.ini" + +# install service file +__systemd_unit uwsgi-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_uwsgi/parameter/default/bind-to b/type/__netbox_uwsgi/parameter/default/bind-to new file mode 100644 index 0000000..c696456 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:3031 diff --git a/type/__netbox_uwsgi/parameter/optional_multiple b/type/__netbox_uwsgi/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to From 9d8b3ebe743685daac13d54ac3bf2449517c3318 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 1 Sep 2020 10:02:08 +0200 Subject: [PATCH 28/66] __netbox: add wrapper service to manage the wsgi services The wrapper service will "control" the services added from the __netbox_* types to provide a general interface. This is more dynamic than the alias approach used previously. Through this, it is possible to handle multiple wsgi services for netbox - if this works .. See as a reference: http://alesnosek.com/blog/2016/12/04/controlling-a-multi-service-application-with-systemd/ --- type/__netbox/files/netbox.service | 12 ++++++++++++ type/__netbox/gencode-remote | 1 + type/__netbox/manifest | 10 ++++++---- type/__netbox_gunicorn/files/netbox.service | 3 ++- type/__netbox_gunicorn/gencode-remote | 6 ------ type/__netbox_gunicorn/man.rst | 3 +-- type/__netbox_uwsgi/files/netbox.service | 3 ++- type/__netbox_uwsgi/gencode-remote | 6 ------ type/__netbox_uwsgi/man.rst | 3 +-- 9 files changed, 25 insertions(+), 22 deletions(-) create mode 100644 type/__netbox/files/netbox.service diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service new file mode 100644 index 0000000..88baaeb --- /dev/null +++ b/type/__netbox/files/netbox.service @@ -0,0 +1,12 @@ +[Unit] +Description=NetBox Service Wrapper +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 0153579..d598aa8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -98,6 +98,7 @@ fi if [ "$changes" = "yes" ]; then cat << EOF # Restart service. +service netbox restart service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 079ea48..95688d3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,7 +204,9 @@ $SECRET_KEY SECRET -# Upload systemd unit for worker service -__systemd_unit netbox-rq.service \ - --source "$__type/files/netbox-rq.service" \ - --enablement-state enabled +# Upload systemd unit for worker and wsgi service +for unit in netbox netbox-rq; do + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled +done diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 475ddc4..2154d27 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,6 +1,8 @@ [Unit] Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ +PartOf=netbox.service +After=netbox.service After=network-online.target Wants=network-online.target @@ -19,5 +21,4 @@ RestartSec=30 PrivateTmp=true [Install] -Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index e424c22..163bb1a 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -19,12 +19,6 @@ if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then printf "configured\n" >> "$__messages_out" fi -# application -if grep -q "^__netbox:" "$__messages_in"; then - do_restart=yes - # no message cause it should be obvious -fi - # restart gunicorn if [ "$do_restart" ]; then diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 43f5864..ac86261 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -37,8 +37,7 @@ updated $old to $new configured Configuration for gunicorn changed. -In both cases, and at messages from the `__netbox` type, it restarts the -service to using the up-to-date version. +In both cases, it restarts the service to use the up-to-date version. EXAMPLES diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index cfce752..bc10571 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -1,6 +1,8 @@ [Unit] Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ +PartOf=netbox.service +After=netbox.service After=network-online.target Wants=network-online.target @@ -19,5 +21,4 @@ RestartSec=30 PrivateTmp=true [Install] -Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 5cc82d1..3127b8b 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -19,12 +19,6 @@ if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then printf "configured\n" >> "$__messages_out" fi -# application -if grep -q "^__netbox:" "$__messages_in"; then - do_restart=yes - # no messages cause this is obvious -fi - # restart uwsgi if [ "$do_restart" ]; then diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 5bf3160..f328496 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -39,8 +39,7 @@ upgraded configured The uwsgi configuration got updated. -In both cases, and at messages from the `__netbox` type, it restarts the -service to using the up-to-date version. +In all cases, it restarts the service to use the up-to-date version. EXAMPLES From facb5a64d3824df095c6374a39f3a78482987046 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 1 Sep 2020 12:40:05 +0200 Subject: [PATCH 29/66] __netbox{,_gunicorn}: disabled shellcheck warnings Shellcheck warned about creating content for a python array. As the string will be printed literally into the config, the warning does not match to the current case. --- type/__netbox/manifest | 3 ++- type/__netbox_gunicorn/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 95688d3..aef50c1 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -38,9 +38,10 @@ export DATABASE_PORT # list of hosts ALLOWED_HOSTS="" while read -r hostname; do + # shellcheck disable=SC2089 ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," done < "$__object/parameter/host" -export ALLOWED_HOSTS +export ALLOWED_HOSTS # shellcheck disable=SC2090 if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index dab8181..d61cbe6 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -2,9 +2,10 @@ HOST="" while read -r host; do + # shellcheck disable=SC2089 HOST="$HOST '$host'," done < "$__object/parameter/bind-to" -export HOST +export HOST # shellcheck disable=SC2090 # process template mkdir "$__object/files" From dbc91cb33914e0b3c899dd169d61613f7a766c2a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Sep 2020 07:50:12 +0200 Subject: [PATCH 30/66] __netbox*: updated man pages --- type/__netbox/man.rst | 20 +++++++++++++++++--- type/__netbox_gunicorn/man.rst | 13 ++++++++++--- type/__netbox_uwsgi/man.rst | 13 ++++++++++--- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 6bb1e36..a3f283c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -12,8 +12,12 @@ This (singleton) type installs and configures a NetBox instance, a web application to help manage and document computer networks. It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. -Netbox will be run via `gnuicorn` as WSGI service. It setup systemd unit files -for the services `netbox` and `netbox-rq`. +It setup systemd unit files for the services `netbox` and `netbox-rq`. To +access the application through WSGI, uWSGI or Gunicorn can be used. The setup +can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. + +The Gunicorn setup is recommended from the NetBox documentation. Consult each +manual page to decide. The types must be called after the `__netbox` type. REQUIRED PARAMETERS @@ -36,6 +40,7 @@ host Hostname (domain or IP address) on which the application is served. Multiple hostnames are possible; given as multiple arguments. + OPTIONAL PARAMETERS ------------------- secret-key @@ -147,6 +152,7 @@ scripts-root needed. By default, it will be stored into the installation directory (``/opt/netbox/netbox/netbox/scripts``). + BOOLEAN PARAMETERS ------------------ redis-ssl @@ -171,6 +177,7 @@ update-notify Enables the NetBox version check for new upstream updates. It checks every 24 hours for new releases and notify the admin users in the gui if any. + MESSAGES -------- installed $VERSION @@ -199,6 +206,8 @@ EXAMPLES --ldap-group-base "ou=groups,dc=domain,dc=tld" \ --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + # using recommended gunicorn setup + require="__netbox" __netbox_gunicorn NOTES @@ -216,9 +225,14 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. + SEE ALSO -------- -- `NetBox documentation `_ +`NetBox documentation `_ + +:strong:`cdist-type__netbox_gunicorn`\ (7) +:strong:`cdist-type__netbox_uwsgi`\ (7) + AUTHORS ------- diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index ac86261..c51be0c 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -3,12 +3,16 @@ cdist-type__netbox_uwsgi(7) NAME ---- -cdist-type__netbox_gunicorn - run netbox with gunicorn +cdist-type__netbox_gunicorn - Run NetBox with Gunicorn DESCRIPTION ----------- -This space intentionally left blank. +This (singleton) type installs Gunicorn into the NetBox `python-venv` to host +the NetBox WSGI application. It provides the application as HTTP over the given +sockets. Static content must be served independent of Gunicorn. The Gunicorn +daemon is available as the `gunicorn-netbox` systemd service, but also +available via the `netbox` wrapper service. REQUIRED PARAMETERS @@ -58,7 +62,10 @@ EXAMPLES SEE ALSO -------- -:strong:`__netbox`\ (7) +`Gunicorn Documentation `_ + +:strong:`cdist-type__netbox`\ (7) +:strong:`cdist-type__netbox_uwsgi`\ (7) AUTHORS diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index f328496..9d5ced7 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -3,12 +3,16 @@ cdist-type__netbox_uwsgi(7) NAME ---- -cdist-type__netbox_uwsgi - run netbox with uwsgi +cdist-type__netbox_uwsgi - Run NetBox with uWSGI DESCRIPTION ----------- -This space intentionally left blank. +This (singleton) type installs uWSGI into the NetBox `python-venv`. It hosts +the NetBox WSGI application over the WSGI protocol. A further server must be +installed to provide it as HTTP. This application is available via the +`uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper +service, too. REQUIRED PARAMETERS @@ -58,7 +62,10 @@ EXAMPLES SEE ALSO -------- -:strong:`TODO`\ (7) +`uWSGI Documentation `_ + +:strong:`cdist-type__netbox`\ (7) +:strong:`cdist-type__netbox_gunicorn`\ (7) AUTHORS From 2805b6beffc61c380ebffe26a487bbdf587c992b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Sep 2020 09:15:45 +0200 Subject: [PATCH 31/66] __netbox{,_gunicorn}: fix shellcheck directives --- type/__netbox/manifest | 3 ++- type/__netbox_gunicorn/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index aef50c1..273423e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -41,7 +41,8 @@ while read -r hostname; do # shellcheck disable=SC2089 ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," done < "$__object/parameter/host" -export ALLOWED_HOSTS # shellcheck disable=SC2090 +# shellcheck disable=SC2090 +export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index d61cbe6..25ea452 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -5,7 +5,8 @@ while read -r host; do # shellcheck disable=SC2089 HOST="$HOST '$host'," done < "$__object/parameter/bind-to" -export HOST # shellcheck disable=SC2090 +# shellcheck disable=SC2090 +export HOST # process template mkdir "$__object/files" From 243e34f0a5e6d7e012c3eee3865cc995432f6f07 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 11:20:20 +0200 Subject: [PATCH 32/66] __netbox_uwsgi: enable further protocols to bind to Enables multiple protocols like fastcgi or HTTP to bind to. This makes it more flexible to use. Also, a little fix for __netbox was done: correctly output a error msg. --- type/__netbox/manifest | 2 +- type/__netbox_uwsgi/files/uwsgi.ini.sh | 29 +++++++++++++++++-- type/__netbox_uwsgi/man.rst | 29 +++++++++++++++---- .../parameter/optional_multiple | 4 +++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 273423e..cb6062e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -127,7 +127,7 @@ fi export SMTP_USE_SSL if [ -f "$__object/parameter/smtp-use-tls" ]; then if [ "$SMTP_USE_SSL" = "True" ]; then - echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" + echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" >&2 exit 2 fi SMTP_USE_TLS="True" diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 557a5fa..324b150 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -2,6 +2,18 @@ # Generates uwsgi config # see https://uwsgi-docs.readthedocs.io/en/latest/Options.html +# or https://uwsgi-docs-additions.readthedocs.io/en/latest/Options.html + +# params: +# 1: parameter name +# 2: parameter value file +# +# output: the lines for the configuration option +multi_options() { + while read -r line; do + printf "%s = %s\n" "$1" "$line" + done < "$2" +} # fix missing $__explorer # see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 @@ -13,8 +25,21 @@ cores="$(cat "$__explorer/cpu_cores")" cat << EOF [uwsgi] -; socket to bind -socket = $HOST +; socket(s) to bind +EOF + +# special protocol to bind +for param in $(find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print); do + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" + socket_changes=yes +done +# else, default bind to +if [ -z "$socket_changes" ]; then + multi_options "socket" "$__object/parameter/bind-to" +fi + + +cat << EOF ; processes and threads processes = $(( 2*cores + 1 )) diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 9d5ced7..7501573 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -9,10 +9,11 @@ cdist-type__netbox_uwsgi - Run NetBox with uWSGI DESCRIPTION ----------- This (singleton) type installs uWSGI into the NetBox `python-venv`. It hosts -the NetBox WSGI application over the WSGI protocol. A further server must be -installed to provide it as HTTP. This application is available via the -`uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper -service, too. +the NetBox WSGI application via the WSGI protocol. A further server must be +installed to provide it as HTTP and serve static content. It supports multiple +protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This +application is available via the `uwsgi-netbox` systemd service. It is +controllable via the `netbox` wrapper service, too. REQUIRED PARAMETERS @@ -24,7 +25,15 @@ OPTIONAL PARAMETERS ------------------- bind-to The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. - Defaults to ``127.0.0.1:3031``. + Defaults to ``127.0.0.1:3031``. Can be set multiple times. + +uwsgi-bind +http-bind +fastcgi-bind +scgi-bind + Bind the application to a specific protocol instead of implicit uwsgi via + ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must + be a UNIX/TCP socket. Can be set multiple times. BOOLEAN PARAMETERS @@ -55,10 +64,18 @@ EXAMPLES __netbox $args require="__netbox" __netbox_uwsgi - # with special bind + # with multiple binds + __netbox $args require="__netbox" __netbox_uwsgi --bind-to 0.0.0.0:3032 \ --bind-to 0.0.0.0:3033 + # with multiple protocols + # parameter `--bind-to` will be ignored + __netbox $args + require="__netbox" __netbox_uwsgi --uwsgi-bind 0.0.0.0:3031 \ + --http-bind 0.0.0.0:8080 \ + --fastcgi-bind 1.2.3.4:5678 + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/parameter/optional_multiple b/type/__netbox_uwsgi/parameter/optional_multiple index 331c077..3f3e7d4 100644 --- a/type/__netbox_uwsgi/parameter/optional_multiple +++ b/type/__netbox_uwsgi/parameter/optional_multiple @@ -1 +1,5 @@ bind-to +uwsgi-bind +http-bind +fastcgi-bind +scgi-bind From af91b6e3a1ad15f6579983e25570a0bda8796a9b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 11:26:15 +0200 Subject: [PATCH 33/66] __netbox_uwsgi: mark as singleton This should have been done much earlier .. :/ --- type/__netbox_uwsgi/singleton | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 type/__netbox_uwsgi/singleton diff --git a/type/__netbox_uwsgi/singleton b/type/__netbox_uwsgi/singleton new file mode 100644 index 0000000..e69de29 From 314a0d4d8ef6235c70fbf3d7b1421f1b34cb18c2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 13:53:05 +0200 Subject: [PATCH 34/66] __netbox: kill all running venv processes To avoid aborts because of the python venv could not be updated by killing all processes that uses the venv. It will be done all times to prevent any error, because it could not be reliably detected if the type installs or updates NetBox. --- type/__netbox/gencode-remote | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d598aa8..f200408 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -30,6 +30,17 @@ tar xf '$archive' cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" +EOF + + # Stop everything in the pyenv to update + cat << EOF +# Try to kill everything in the venv +systemctl stop netbox gunicorn-netbox uwsgi-netbox || true +ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true + +EOF + + cat << EOF # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' From 755bd9098ede2dbad6c557f7c4983037fe3aab22 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 15:18:49 +0200 Subject: [PATCH 35/66] __netbox*: update systemd service files The `uwsgi-netbox` service now works, also the `netbox` wrapper service. The PID file was removed from the Gunicorn service as it is not required and a bit more efford to move it to `/run/` due to permissions. Generally, all depend on `network.target` instead of `network-online.target` now, and signals for reload, stop and kill were added (especially required the uwsgi service). --- type/__netbox/files/netbox.service | 5 +++-- type/__netbox_gunicorn/files/netbox.service | 14 ++++++++------ type/__netbox_uwsgi/files/netbox.service | 16 +++++++++------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service index 88baaeb..738e178 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox/files/netbox.service @@ -1,12 +1,13 @@ [Unit] Description=NetBox Service Wrapper Documentation=https://netbox.readthedocs.io/en/stable/ -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=oneshot RemainAfterExit=yes +ExecStart=/bin/true [Install] WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 2154d27..55f3549 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -3,22 +3,24 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service After=netbox.service -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=simple User=netbox Group=netbox -PIDFile=/var/tmp/netbox.pid WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi +ExecStart=/opt/netbox/venv/bin/gunicorn --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi +# signals: https://docs.gunicorn.org/en/stable/signals.html +ExecReload=kill -HUP $MAINPID +ExecStop=kill -TERM $MAINPID +KillSignal=SIGQUIT Restart=on-failure RestartSec=30 -PrivateTmp=true [Install] -WantedBy=multi-user.target +WantedBy=netbox.service diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index bc10571..10ea734 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -3,22 +3,24 @@ Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service After=netbox.service -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=simple User=netbox Group=netbox -PIDFile=/var/tmp/netbox.pid -WorkingDirectory=/opt/netbox +WorkingDirectory=/opt/netbox/netbox/ -ExecStart=/opt/netbox/venv/bin/uwsgi --master --wsgi-file netbox/netbox/wsgi.py uwsgi.ini +ExecStart=/opt/netbox/venv/bin/uwsgi --master --module netbox.wsgi uwsgi.ini +# signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi +ExecReload=kill -HUP $MAINPID +ExecStop=kill -INT $MAINPID +KillSignal=SIGQUIT Restart=on-failure RestartSec=30 -PrivateTmp=true [Install] -WantedBy=multi-user.target +WantedBy=netbox.service From 5a403de0571b57c0da07824c91d703a21231e303 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 19:09:34 +0200 Subject: [PATCH 36/66] __netbox_uwsgi: fix shellcheck Fixed shellcheck cause of the find command. This required a bigger change due to variables and subshell. Now, input is used through a here-document. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 10 +++++++--- type/__netbox_uwsgi/manifest | 3 --- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 324b150..0f85804 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -29,10 +29,14 @@ cat << EOF EOF # special protocol to bind -for param in $(find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print); do +while read -r param; do multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" - socket_changes=yes -done + socket_changes="yes" + +done << INPUT # here-doc cause of SC2031 +$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print ) +INPUT + # else, default bind to if [ -z "$socket_changes" ]; then multi_options "socket" "$__object/parameter/bind-to" diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index 779f93b..3d9c1b4 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,8 +1,5 @@ #!/bin/sh -e -HOST="$__object/parameter/bind-to" -export HOST - # process template mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" From ffba3ae776ac4e4bd67d4812b8073c2e81a69cdd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 21:48:04 +0200 Subject: [PATCH 37/66] __netbox_uwsgi: add --serve-static parameter New option to directly serve static content via uWSGI. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 11 +++++++++++ type/__netbox_uwsgi/man.rst | 11 ++++++++++- type/__netbox_uwsgi/manifest | 6 ++++++ type/__netbox_uwsgi/parameter/boolean | 1 + 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 type/__netbox_uwsgi/parameter/boolean diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 0f85804..4abc59a 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -43,9 +43,20 @@ if [ -z "$socket_changes" ]; then fi +# multi-process settings cat << EOF ; processes and threads processes = $(( 2*cores + 1 )) threads = 2 EOF + + +# optional mapping of static content +if [ "$STATIC_MAP" != "" ]; then + cat << EOF + +; map static content +static-map = /static=/opt/netbox/netbox/static +EOF +fi diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 7501573..f5eff0b 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -38,7 +38,12 @@ scgi-bind BOOLEAN PARAMETERS ------------------ -None. +serve-static + Setup uWSGI to serve the static content, too. This is generally not + recommended for real production setups, as it is the job of the reverse + proxy server, who will thread it as static cachable content. This option + is only recommended for small setups or direct usage of the uWSGI socket + like using it as standalone HTTP server for NetBox. MESSAGES @@ -76,6 +81,10 @@ EXAMPLES --http-bind 0.0.0.0:8080 \ --fastcgi-bind 1.2.3.4:5678 + # as standalone server + __netbox $args + require="__netbox" __netbox_uwsgi --serve-static --http-bind 0.0.0.0:80 + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index 3d9c1b4..bb4f23f 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,5 +1,11 @@ #!/bin/sh -e +# *bind* parameters are directly processed in the gen script +if [ -f "$__object/parameter/serve-static" ]; then + STATIC_MAP="yes" + export STATIC_MAP +fi + # process template mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" diff --git a/type/__netbox_uwsgi/parameter/boolean b/type/__netbox_uwsgi/parameter/boolean new file mode 100644 index 0000000..aa08196 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/boolean @@ -0,0 +1 @@ +serve-static From 090a8f015e86f8dd0385b0b37a234a2538c254da Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:08:37 +0200 Subject: [PATCH 38/66] __netbox_uwsgi: fix uwsgi netbox service file Revert working directory changes as the configuration file still needs to be accessable. An absolute path would work, too, but it is not the preferred way. To still work with the python wsgi application, `--chdir` is used. --- type/__netbox_uwsgi/files/netbox.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index 10ea734..8d41cd0 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -11,9 +11,9 @@ Type=simple User=netbox Group=netbox -WorkingDirectory=/opt/netbox/netbox/ +WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/uwsgi --master --module netbox.wsgi uwsgi.ini +ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi ExecReload=kill -HUP $MAINPID ExecStop=kill -INT $MAINPID From 9d330a91bc5bdbdbe69a4db98ddad0eeaed344cb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:27:42 +0200 Subject: [PATCH 39/66] __netbox_{gunicorn,uwsgi}: add netbox ownership to config file --- type/__netbox_gunicorn/manifest | 3 ++- type/__netbox_uwsgi/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index 25ea452..b72b8fd 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -14,7 +14,8 @@ mkdir "$__object/files" # gunicorn config file __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__object/files/gunicorn.py" + --mode 644 --owner netbox \ + --source "$__object/files/gunicorn.py" # install service file __systemd_unit gunicorn-netbox.service \ diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index bb4f23f..b6c0308 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -13,7 +13,8 @@ mkdir "$__object/files" # uwsgi config file # TODO maybe patching with __key_value cause of .ini ? __file /opt/netbox/uwsgi.ini \ - --mode 644 --source "$__object/files/uwsgi.ini" + --mode 644 --owner netbox \ + --source "$__object/files/uwsgi.ini" # install service file __systemd_unit uwsgi-netbox.service \ From b10f6b71d051377f023fd6e96abbc00ff06e5d5d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:28:48 +0200 Subject: [PATCH 40/66] __netbox: avoid useless generated code-remote Because `set -e` got printed all the time, the type __netbox always had some generated code for the remote side. This line was removed because this is already done by cdist when executing the code-remote script. Rather, the exit-on-error option was set to some scirpts (two ..). --- type/__netbox/gencode-remote | 4 +--- type/__netbox/manifest | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index f200408..9ae6b5b 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -1,6 +1,4 @@ -#!/bin/sh - -echo "set -e" +#!/bin/sh -e old_version="$(cat "$__object/explorer/version")" VERSION=$(cat "$__object/parameter/version") diff --git a/type/__netbox/manifest b/type/__netbox/manifest index cb6062e..5f1762a 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e os=$(cat "$__global/explorer/os") From 3b07a660b3a7dd442e3184600fad25ed27769a5f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:55:43 +0200 Subject: [PATCH 41/66] __netbox_uwsgi: add hint to the --serve-static parameter --- type/__netbox_uwsgi/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index f5eff0b..6826cca 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -45,6 +45,11 @@ serve-static is only recommended for small setups or direct usage of the uWSGI socket like using it as standalone HTTP server for NetBox. + **Hint**: This parameter does not work in junction with the `__netbox` + parameter ``--basepath``. It is because this type does not know the + parameter value and this case is very unlikly to happen; although an + implementation is not difficult. + MESSAGES -------- From c9e4e8d7dce88c3f82a4bbf053b174d88a70a4ce Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 13:44:03 +0200 Subject: [PATCH 42/66] __netbox_{gunicorn,uwsgi}: add state parameter Adds the --state parameter to both types. With it, the transition between both types can be done smothly. --- type/__netbox_gunicorn/gencode-remote | 59 +++++++++++------ type/__netbox_gunicorn/man.rst | 40 ++++++++++-- type/__netbox_gunicorn/manifest | 64 ++++++++++++++----- .../__netbox_gunicorn/parameter/default/state | 1 + type/__netbox_gunicorn/parameter/optional | 1 + type/__netbox_uwsgi/gencode-remote | 58 +++++++++++------ type/__netbox_uwsgi/man.rst | 32 +++++++++- type/__netbox_uwsgi/manifest | 62 +++++++++++++----- type/__netbox_uwsgi/parameter/default/state | 1 + type/__netbox_uwsgi/parameter/optional | 1 + 10 files changed, 244 insertions(+), 75 deletions(-) create mode 100644 type/__netbox_gunicorn/parameter/default/state create mode 100644 type/__netbox_gunicorn/parameter/optional create mode 100644 type/__netbox_uwsgi/parameter/default/state create mode 100644 type/__netbox_uwsgi/parameter/optional diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 163bb1a..0da92f0 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -1,29 +1,50 @@ #!/bin/sh -e -curr_installed="$(cat "$__object/explorer/installed")" -should_installed="$(cat "$__object/explorer/should_installed")" +# control state +state="$(cat "$__object/parameter/state")" -# gunicorn version change -if [ "$curr_installed" != "$should_installed" ]; then - # (re)installing gunicorn - echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" +case "$state" in + # install gunicorn + enabled|disabled) + curr_installed="$(cat "$__object/explorer/installed")" + should_installed="$(cat "$__object/explorer/should_installed")" - do_restart=yes - printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ - >> "$__messages_out" -fi + # gunicorn version change + if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" -# configuration changes -if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then - do_restart=yes - printf "configured\n" >> "$__messages_out" -fi + if [ "$curr_installed" != "" ]; then + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" + else + printf "installed\n" >> "$__messages_out" + fi + do_restart=yes + fi + + # configuration changes + if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" + fi -# restart gunicorn -if [ "$do_restart" ]; then - cat << EOF + # restart gunicorn + if [ "$do_restart" ] && [ "$state" != "disabled" ]; then + cat << EOF # Restart service service gunicorn-netbox restart EOF -fi + fi + ;; + + # uninstall + absent) + # check if installed + if [ -s "$__object/explorer/installed" ]; then + # service already disabled + echo "/opt/netbox/venv/bin/pip3 uninstall -y gunicorn" + printf "uninstalled\n" >> "$__messages_out" + fi +esac diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index c51be0c..1c09d78 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -1,5 +1,5 @@ -cdist-type__netbox_uwsgi(7) -=========================== +cdist-type__netbox_gunicorn(7) +============================== NAME ---- @@ -22,6 +22,20 @@ None. OPTIONAL PARAMETERS ------------------- +state + Represents the state of the Gunciron application. Defaults to ``enabled``. + + enabled + The Gunicorn service is enabled and running. + disabled + The Gunicorn service is installed, but disabled. + absent + The uWSGI service is not installed and all configuration removed. + + This type does not guarantee anything about the running state of the + service. To be sure about the service is stopped or not, use the type + :strong:`cdist-type__systemd_service`\ (7) after this execution. + bind-to The hosts the gunicorn socket should be bind to. Formats are `IP`, `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple @@ -35,13 +49,20 @@ None. MESSAGES -------- -updated $old to $new +installed + The software was installed. + +upgraded $old to $new The version of the gunicorn software was updated from `$old` to `$new`. configured Configuration for gunicorn changed. -In both cases, it restarts the service to use the up-to-date version. +uninstalled + The Gunicorn application was removed. + +In all cases where the application is still present, it restarts the service to +use the up-to-date version. EXAMPLES @@ -59,6 +80,17 @@ EXAMPLES --bind-to 0.0.0.0:8001 \ --bind-to 1.2.3.4:5678 + # replace uwsgi with gunicorn + __netbox $args + require="__netbox" __netbox_uwsgi --state absent + # it should depend on __netbox_uwsgi if they use the same socket + require="__netbox_uwsgi" __netbox_gunicorn --state enabled + + # be sure the service is disabled + __netbox $args + require="__netbox" __netbox_gunicorn --state disabled + require="__netbox_gunicorn" __systemd_service gunicorn-netbox --state stopped + SEE ALSO -------- diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index b72b8fd..2e5ccce 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -1,23 +1,55 @@ #!/bin/sh -e +# __netbox_gunicorn/manifest -HOST="" -while read -r host; do - # shellcheck disable=SC2089 - HOST="$HOST '$host'," -done < "$__object/parameter/bind-to" -# shellcheck disable=SC2090 -export HOST +# Check states +state="" +unit_state="" +param_state="$(cat "$__object/parameter/state")" -# process template -mkdir "$__object/files" -"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" +case "$param_state" in + enabled|disabled) + state="present" + unit_state="$param_state" + ;; + + absent) + state="absent" + unit_state="disabled" + ;; + + *) + # does not exist + printf "The state '%s' does not exist, can't continue!\n" "$param_state" >&2 + exit 2 + ;; +esac + + +if [ "$state" = "present" ]; then + HOST="" + while read -r host; do + # shellcheck disable=SC2089 + HOST="$HOST '$host'," + done < "$__object/parameter/bind-to" + # shellcheck disable=SC2090 + export HOST + + # process template + mkdir "$__object/files" + "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + + # gunicorn config file + __file /opt/netbox/gunicorn.py \ + --mode 644 --owner netbox \ + --source "$__object/files/gunicorn.py" + +else + # absent config file + __file /opt/netbox/gunicorn.py --state absent +fi -# gunicorn config file -__file /opt/netbox/gunicorn.py \ - --mode 644 --owner netbox \ - --source "$__object/files/gunicorn.py" # install service file __systemd_unit gunicorn-netbox.service \ - --source "$__type/files/netbox.service" \ - --enablement-state enabled --restart + --state "$state" --enablement-state "$unit_state" \ + --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_gunicorn/parameter/default/state b/type/__netbox_gunicorn/parameter/default/state new file mode 100644 index 0000000..86981e6 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/state @@ -0,0 +1 @@ +enabled diff --git a/type/__netbox_gunicorn/parameter/optional b/type/__netbox_gunicorn/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional @@ -0,0 +1 @@ +state diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 3127b8b..9b6a603 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -1,29 +1,47 @@ #!/bin/sh -e -# not installed -if ! [ -s "$__object/explorer/installed" ]; then - echo "/opt/netbox/venv/bin/pip3 install uwsgi" - do_restart=yes - printf "installed\n" >> "$__messages_out" +# control state +state="$(cat "$__object/parameter/state")" -# updates available -elif [ -s "$__object/explorer/upgradeable" ]; then - echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" - do_restart=yes - printf "upgraded\n" >> "$__messages_out" -fi +case "$state" in + # install uwsgi + enabled|disabled) + # not installed + if ! [ -s "$__object/explorer/installed" ]; then + echo "/opt/netbox/venv/bin/pip3 install uwsgi" + do_restart=yes + printf "installed\n" >> "$__messages_out" -# changed configuration -if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then - do_restart=yes - printf "configured\n" >> "$__messages_out" -fi + # updates available + elif [ -s "$__object/explorer/upgradeable" ]; then + echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + do_restart=yes + printf "upgraded\n" >> "$__messages_out" + fi + + # changed configuration + if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" + fi -# restart uwsgi -if [ "$do_restart" ]; then - cat << EOF + # restart uwsgi + if [ "$do_restart" ] && [ "$state" != "disabled" ]; then + cat << EOF # Restart service service uwsgi-netbox restart EOF -fi + fi + ;; + + # uninstall + absent) + # check if installed + if [ -s "$__object/explorer/installed" ]; then + # service already disabled + echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi" + printf "uninstalled\n" >> "$__messages_out" + fi + ;; +esac diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 6826cca..3fb8515 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -23,6 +23,21 @@ None. OPTIONAL PARAMETERS ------------------- +state + Represents the state of the uWSGI application. Defaults to ``enabled``. + + enabled + The uWSGI service is enabled and running. + disabled + The uWSGI service is installed, but disabled. + absent + The uWSGI service is not installed and all configuration removed. + + This type does not guarantee anything about the running state of the + service. To be sure about the service is stopped or not, use the type + :strong:`cdist-type__systemd_service`\ (7) after this execution. + + bind-to The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. Defaults to ``127.0.0.1:3031``. Can be set multiple times. @@ -62,7 +77,11 @@ upgraded configured The uwsgi configuration got updated. -In all cases, it restarts the service to use the up-to-date version. +uninstalled + The uWSGI application was removed. + +In all cases where the application is still present, it restarts the service to +use the up-to-date version. EXAMPLES @@ -90,6 +109,17 @@ EXAMPLES __netbox $args require="__netbox" __netbox_uwsgi --serve-static --http-bind 0.0.0.0:80 + # replace gunicorn with uwsgi + __netbox $args + require="__netbox" __netbox_gunicorn --state absent + # it should depend on __netbox_gunicorn if they use the same socket + require="__netbox_gunicorn" __netbox_uwsgi --state enabled + + # be sure the service is disabled + __netbox $args + require="__netbox" __netbox_uwsgi --state disabled + require="__netbox_uwsgi" __systemd_service uwsgi-netbox --state stopped + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index b6c0308..c5885c9 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,22 +1,54 @@ #!/bin/sh -e +# __netbox_uwsgi/manifest -# *bind* parameters are directly processed in the gen script -if [ -f "$__object/parameter/serve-static" ]; then - STATIC_MAP="yes" - export STATIC_MAP +# Check states +state="" +unit_state="" +param_state="$(cat "$__object/parameter/state")" + +case "$param_state" in + enabled|disabled) + state="present" + unit_state="$param_state" + ;; + + absent) + state="absent" + unit_state="disabled" + ;; + + *) + # does not exist + printf "The state '%s' does not exist, can't continue!\n" "$param_state" >&2 + exit 2 + ;; +esac + + +if [ "$state" = "present" ]; then + # *bind* parameters are directly processed in the gen script + if [ -f "$__object/parameter/serve-static" ]; then + STATIC_MAP="yes" + export STATIC_MAP + fi + + # process template + mkdir "$__object/files" + "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" + + # uwsgi config file + # TODO maybe patching with __key_value cause of .ini ? + __file /opt/netbox/uwsgi.ini \ + --mode 644 --owner netbox \ + --source "$__object/files/uwsgi.ini" + +else + # absent config file + __file /opt/netbox/uwsgi.ini --state absent fi -# process template -mkdir "$__object/files" -"$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" - -# uwsgi config file -# TODO maybe patching with __key_value cause of .ini ? -__file /opt/netbox/uwsgi.ini \ - --mode 644 --owner netbox \ - --source "$__object/files/uwsgi.ini" # install service file __systemd_unit uwsgi-netbox.service \ - --source "$__type/files/netbox.service" \ - --enablement-state enabled --restart + --state "$state" --enablement-state "$unit_state" \ + --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/parameter/default/state b/type/__netbox_uwsgi/parameter/default/state new file mode 100644 index 0000000..86981e6 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/state @@ -0,0 +1 @@ +enabled diff --git a/type/__netbox_uwsgi/parameter/optional b/type/__netbox_uwsgi/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__netbox_uwsgi/parameter/optional @@ -0,0 +1 @@ +state From 0afc7136f8e4f304f179965f229ed368ce73abb6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 18:39:07 +0200 Subject: [PATCH 43/66] __netbox_uwsgi: fix if only --bind-to given If the here-document is empty because no file could be found, an empty line will be read. The new line of code will skip in those cases. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 4abc59a..7835a3e 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -30,6 +30,8 @@ EOF # special protocol to bind while read -r param; do + if [ -z "$param" ]; then continue; fi # ignore empty lines from the here-doc + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" socket_changes="yes" From eed3515424668b8684058b0d02368746ee5f5c06 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Sep 2020 17:10:59 +0200 Subject: [PATCH 44/66] __netbox: add --ldap-staff-group parameter Required to get "admin access". --- type/__netbox/files/ldap_config.py.sh | 15 +++++++++++---- type/__netbox/man.rst | 4 ++++ type/__netbox/manifest | 4 ++++ type/__netbox/parameter/optional | 1 + 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index a49d800..4e6b099 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -53,6 +53,8 @@ AUTH_LDAP_GROUP_TYPE = PosixGroupType() # Mirror LDAP group assignments. AUTH_LDAP_MIRROR_GROUPS = True +# For more granular permissions, map LDAP groups to Django groups. +AUTH_LDAP_FIND_GROUP_PERMS = True EOF if [ "$LDAP_REQUIRE_GROUP" != "" ]; then @@ -63,13 +65,18 @@ AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" EOF fi - if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then - cat << EOF + cat << EOF # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { - "is_superuser": "$LDAP_SUPERUSER_GROUP", -} EOF + # superuser + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + echo " \"is_superuser\": \"$LDAP_SUPERUSER_GROUP\"," fi + # staff user + if [ "$LDAP_STAFF_GROUP" != "" ]; then + echo " \"is_staff\": \"$LDAP_STAFF_GROUP\"," + fi + echo "}" fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index a3f283c..bc4bbc9 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -78,6 +78,10 @@ ldap-group-base ldap-require-group Group required to login. +ldap-staff-group + Make members of this group to "staff". This gives the users "Admin Access", + which means access to the "NetBox Administration" site. + ldap-superuser-group Make members of this groups superusers. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 5f1762a..3681ea0 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -89,6 +89,10 @@ if [ -f "$__object/parameter/ldap-superuser-group" ]; then LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") export LDAP_SUPERUSER_GROUP fi +if [ -f "$__object/parameter/ldap-staff-group" ]; then + LDAP_STAFF_GROUP="$(cat "$__object/parameter/ldap-staff-group")" + export LDAP_STAFF_GROUP +fi # export if base ldap parameters are used export USE_LDAP diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 47722e6..9495f7a 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -7,6 +7,7 @@ ldap-bind-password ldap-user-base ldap-group-base ldap-require-group +ldap-staff-group ldap-superuser-group redis-host redis-port From 3389752decfb48975f539b1702d37fe07d4e7755 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Sep 2020 17:23:22 +0200 Subject: [PATCH 45/66] __netbox: fix detection of the secret-key to use Mixed up a parameter. Should be correct now. --- type/__netbox/manifest | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 3681ea0..06d0c77 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -47,13 +47,13 @@ export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") elif [ -s "$__object/explorer/secretkey" ]; then + # take the key that is already used + SECRET_KEY="$(cat "$__object/explorer/secretkey")" +else # Can be done over netbox/generate_secret_key.py too, but it's to # complicated with the variable setup (can't generated right now!). # Generates a 50-character long key (without ' cause of python quotes) SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" -else - # take the key that is already used - SECRET_KEY="$(cat "$__object/explorer/secretkey")" fi export SECRET_KEY From 398a3da10eeaf67caa042b2a99a1a3e652b31e0f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 18:10:15 +0200 Subject: [PATCH 46/66] __netbox: fix gerneration random generated secret key Because `/dev/random` was used, the `cdist config` could hang a long time to get real random values. The pseudo-generated values through `/dev/urandom` are fully enought for the secret key. --- type/__netbox/manifest | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 06d0c77..4ad7901 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -50,10 +50,11 @@ elif [ -s "$__object/explorer/secretkey" ]; then # take the key that is already used SECRET_KEY="$(cat "$__object/explorer/secretkey")" else - # Can be done over netbox/generate_secret_key.py too, but it's to - # complicated with the variable setup (can't generated right now!). + # Can be done over netbox/generate_secret_key.py too, but it can't be + # generated right now where it's required (only if it's preloaded for + # this type to execute it now). # Generates a 50-character long key (without ' cause of python quotes) - SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" + SECRET_KEY="$(tr -cd '[:graph:]' < /dev/urandom | tr -d \' | head -c50)" fi export SECRET_KEY From 549feb87f96518a38ab7d3368bd3f2a2ed552d0d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 19:20:22 +0200 Subject: [PATCH 47/66] __netbox: fix --update-notify Template script for the `configuration.py` interpreted the boolean flag incorrectly and did the thing the user do not wanted. --- type/__netbox/files/configuration.py.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index ea73133..06a1c86 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -272,12 +272,12 @@ EOF if [ "$UPDATE_CHECK" != "" ]; then cat << EOF -RELEASE_CHECK_URL = None +RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' EOF else cat << EOF -RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' +RELEASE_CHECK_URL = None EOF fi From 5d437839f68da0c29787172ea3daa3170926f8d2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 19:52:03 +0200 Subject: [PATCH 48/66] __netbox: replaced secret key character set The used character set should now only contain characters used by the `netbox/generate_secret_key.py` program. Finially got correct escaping. --- type/__netbox/manifest | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 4ad7901..d754ae6 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -53,8 +53,9 @@ else # Can be done over netbox/generate_secret_key.py too, but it can't be # generated right now where it's required (only if it's preloaded for # this type to execute it now). - # Generates a 50-character long key (without ' cause of python quotes) - SECRET_KEY="$(tr -cd '[:graph:]' < /dev/urandom | tr -d \' | head -c50)" + # Generates a 50-character long key with the same character set like + # the helper script. Must escape the '-' to be no character range. + SECRET_KEY="$(tr -cd '!@#$%^&*(\-_=+)[:alnum:]' < /dev/urandom | head -c50)" fi export SECRET_KEY From dd167f075d25ecec4f1aca461658d2b93ff1ae8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 9 Sep 2020 19:08:46 +0200 Subject: [PATCH 49/66] __netbox*: fix service restart order Cause of corrupt databases if the services are restarted incorrectly, the order and dependencies are adjusted. Now, the `netbox-rq` service will be included in restarts of `netbox` and required for the WSGI servers that it must running. For these changes, the restart command of `__netbox` was adjusted. The other ones where edited too, to use the same command. All services now require redis and postgresql to be started before them to prevent any start order issues. If someone asked for what the RQ worker is required, see here: https://netbox.readthedocs.io/en/stable/additional-features/webhooks/#webhook-processing --- type/__netbox/files/netbox-rq.service | 7 ++++-- type/__netbox/files/netbox.service | 2 +- type/__netbox/gencode-remote | 7 +++--- type/__netbox/man.rst | 24 ++++++++++++++++++++- type/__netbox_gunicorn/files/netbox.service | 4 +++- type/__netbox_gunicorn/gencode-remote | 2 +- type/__netbox_uwsgi/files/netbox.service | 4 +++- type/__netbox_uwsgi/gencode-remote | 2 +- 8 files changed, 41 insertions(+), 11 deletions(-) diff --git a/type/__netbox/files/netbox-rq.service b/type/__netbox/files/netbox-rq.service index 77d7091..330e675 100644 --- a/type/__netbox/files/netbox-rq.service +++ b/type/__netbox/files/netbox-rq.service @@ -1,8 +1,11 @@ [Unit] Description=NetBox Request Queue Worker Documentation=https://netbox.readthedocs.io/en/stable/ -After=network-online.target -Wants=network-online.target +PartOf=netbox.service +Wants=network.target +After=netbox.service +After=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service index 738e178..68010e9 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox/files/netbox.service @@ -1,8 +1,8 @@ [Unit] Description=NetBox Service Wrapper Documentation=https://netbox.readthedocs.io/en/stable/ -After=network.target Wants=network.target +After=network.target [Service] Type=oneshot diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 9ae6b5b..0370949 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -105,9 +105,10 @@ fi # check for changes if [ "$changes" = "yes" ]; then + # to avoid database corruption at config changes, both services must be + # stopped at the same time (noted in the manual, too). cat << EOF -# Restart service. -service netbox restart -service netbox-rq restart +# Restart service. All required services are included with netbox.service. +systemctl restart netbox EOF fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index bc4bbc9..3330b1b 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -12,7 +12,11 @@ This (singleton) type installs and configures a NetBox instance, a web application to help manage and document computer networks. It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. -It setup systemd unit files for the services `netbox` and `netbox-rq`. To +It setup systemd unit files for the services `netbox` and `netbox-rq`. The +`netbox` service only wrap all netbox related services, e.g. restarting and +so one will be delegated to all related services. + +The application is still not accessable because WSGI server is required. To access the application through WSGI, uWSGI or Gunicorn can be used. The setup can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. @@ -229,6 +233,24 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. +If you change a configuration, the database may go corrupt if two instances of +the application are running with different configurations at the same time. +This most commonly happens when the WSGI server and RQ-worker restarts after a +configuration change. This occours in the following case for example: + +.. code-block:: sh + + systemctl restart gunicorn-netbox # WSGI-server already online with new + # configuration after this command. + systemctl restart netbox-rq # RQ-Worker still worked with the old + # configuration till here. + +This type handles the restart of both services correctly to avoid such database +corruptions. To safely manual restart the whole netbox instance manual, simply +restart all services in one ``systemctl restart netbox`` command, as it ensures +that first all services are shut down before starting one of them. The service +``netbox`` wraps all required services that are available. + SEE ALSO -------- diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 55f3549..dcf1a66 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -2,9 +2,11 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service +Requires=netbox-rq.service +Wants=network.target After=netbox.service After=network.target -Wants=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 0da92f0..4fae788 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -34,7 +34,7 @@ case "$state" in if [ "$do_restart" ] && [ "$state" != "disabled" ]; then cat << EOF # Restart service -service gunicorn-netbox restart +systemctl restart gunicorn-netbox EOF fi ;; diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index 8d41cd0..cd06e82 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -2,9 +2,11 @@ Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service +Requires=netbox-rq.service +Wants=network.target After=netbox.service After=network.target -Wants=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 9b6a603..c219643 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -30,7 +30,7 @@ case "$state" in if [ "$do_restart" ] && [ "$state" != "disabled" ]; then cat << EOF # Restart service -service uwsgi-netbox restart +systemctl restart uwsgi-netbox EOF fi ;; From e800f42a6d95b9378516252d53944383f673653d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 9 Sep 2020 20:33:20 +0200 Subject: [PATCH 50/66] __netbox: consistency with __systemd_unit --restart This commit brings consistency into the --restart parameter for systemd units. All units except the netbox wrapper service will be restarted on unit change. --- type/__netbox/manifest | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index d754ae6..f8ed5fc 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -213,8 +213,10 @@ SECRET # Upload systemd unit for worker and wsgi service -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done +# does not restart netbox on change cause it only restart all other services +__systemd_unit netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled --restart From b55186544f4cfeb9f371c4c0ff429bdc2a63c6cc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 28 Sep 2020 19:49:29 +0200 Subject: [PATCH 51/66] __netbox: adding seperate /opt/netbox/cdist/ directory Adding a place to keep all configuration stuff for cdist seperated from the installation directory. For safety reasons. --- type/__netbox/explorer/secretkey | 2 +- type/__netbox/explorer/version | 2 +- type/__netbox/gencode-remote | 26 +++++++------------------- type/__netbox/manifest | 17 ++++++++++------- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/type/__netbox/explorer/secretkey b/type/__netbox/explorer/secretkey index 9f0331c..7cce279 100755 --- a/type/__netbox/explorer/secretkey +++ b/type/__netbox/explorer/secretkey @@ -2,7 +2,7 @@ # Explorer will output the key if he exists. -secretkey="/opt/netbox/.secretkey" +secretkey="/opt/netbox/cdist/secretkey" if [ -f "$secretkey" ]; then cat "$secretkey" fi diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version index 5231e16..ee3dde8 100755 --- a/type/__netbox/explorer/version +++ b/type/__netbox/explorer/version @@ -1,5 +1,5 @@ #!/bin/sh -e # output version if exist -version_path="/opt/netbox/netbox/cdist/version" +version_path="/opt/netbox/cdist/version" if [ -f "$version_path" ]; then cat "$version_path"; fi diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 0370949..aa3864e 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -24,16 +24,13 @@ cd "\$tmpdir" curl -sS -L '$url' > '$archive' tar xf '$archive' -# Save cdist-upload configuration file. -cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" -cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" - EOF # Stop everything in the pyenv to update cat << EOF # Try to kill everything in the venv -systemctl stop netbox gunicorn-netbox uwsgi-netbox || true +systemctl -q --wait stop netbox gunicorn-netbox uwsgi-netbox || true +# don't know if this is required since using --wait ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true EOF @@ -42,11 +39,10 @@ EOF # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' -mkdir '$install_dir/cdist' cp '$src/requirements.txt' /opt/netbox/ # backup dependency info -cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' -cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' +ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. # forcing python3 to be sure (till python4 gets released ..) @@ -79,24 +75,16 @@ cd / rm -r "\$tmpdir" # Save version after successful installation -printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' +printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version EOF # meta printf "installed %s\n" "$VERSION" >> "$__messages_out" changes=yes +fi # check if configuration changed -elif grep -q "^__file/opt/netbox/" "$__messages_in"; then - # check if coping is required - if grep -q "^__file/opt/netbox/netbox/cdist/" "$__messages_in"; then - cat << EOF -# Copy configuration -cp '$install_dir/cdist/configuration.py' '$install_dir/netbox/configuration.py' -cp '$install_dir/cdist/ldap_config.py' '$install_dir/netbox/ldap_config.py' -EOF - fi - +if grep -q "^__file/opt/netbox/" "$__messages_in"; then # meta printf "configured\n" >> "$__messages_out" changes=yes diff --git a/type/__netbox/manifest b/type/__netbox/manifest index f8ed5fc..89b89ac 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -194,20 +194,23 @@ mkdir -p "$__object/files" "$__type/files/configuration.py.sh" > "$__object/files/configuration.py" "$__type/files/ldap_config.py.sh" > "$__object/files/ldap_config.py" -require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents -require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ +require="__user/netbox" __directory /opt/netbox/cdist +require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/configuration.py --mode 640 --owner netbox \ --source "$__object/files/configuration.py" if [ -f "$__object/parameter/ldap-server" ]; then - require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ --source "$__object/files/ldap_config.py" +else + require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/ldap_config.py --state absent fi # save secret -require="__user/netbox" __file /opt/netbox/.secretkey --mode 400 \ - --owner netbox --source - << SECRET +require="__directory/opt/netbox/cdist" __file /opt/netbox/cdist/secretkey \ + --mode 400 --owner netbox --source - << SECRET $SECRET_KEY SECRET From 050812305b282e65be51c3902dba691c4718ca4d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 28 Sep 2020 20:04:22 +0200 Subject: [PATCH 52/66] __netbox: add mange.py calls recommended from upstream Calls where added because there are used upstream in the `upgrade.sh`-Script, too. Upgrade-Script: https://github.com/netbox-community/netbox/blob/develop/upgrade.sh --- type/__netbox/gencode-remote | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index aa3864e..d7bc219 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -64,11 +64,17 @@ EOF # Set final permissions. chown -R netbox /opt/netbox +# NetBox manage scripts # Run database migrations. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate - # Generate static assets. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collectstatic --no-input +# Delete any stale content types +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py remove_stale_contenttypes --no-input +# Delete any expired user sessions +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py clearsessions +# Clear all cached data +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py invalidate all # Remove temporary working directory. cd / From 14f7cf88396f8a2870b4edfdb12f9b8448ac48b8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 4 Oct 2020 17:40:01 +0200 Subject: [PATCH 53/66] __netbox: set --data-root default value The default value is set to preserve all data saved by netbox. As explained in the manpage, it sets the directory for all data directories to `~netbox/data/` (would resolve to `/opt/netbox/data/`), so upgrades will not remove this data. --- type/__netbox/man.rst | 17 ++++++++++------- type/__netbox/manifest | 11 +++++------ type/__netbox/parameter/default/data-root | 1 + 3 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 type/__netbox/parameter/default/data-root diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 3330b1b..6d50268 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -131,7 +131,7 @@ data-root This parameter set's the media, reports and scripts root to subdirectories of the given directory. Values can be overwritten by special parameters like `--media-root` for example. Use this option if you want to store persistant - data of netbox on an other partition. A trailing shlash is not needed. + data of netbox on an other partition. A trailing slash is not needed. The data directories have following predefined sub-directory names: @@ -142,23 +142,26 @@ data-root scripts root: ``$data_root/scripts`` + To preserve all data from installation upgrades - which just replace the + installation directory - the data will be kept in the netbox home directory + rather than the installation directory by default (``/opt/netbox/data/``). + This way, no data will be deleted after the installation directory + replacement because it remains outside of the installation directory. + media-root The file path to where media files (like image attachments) are stored. Change this path if you require to store data on an other partiotion. - A trailing slash is not needed. By default, it will be stored into the - installation directory (``/opt/netbox/netbox/netbox/media``). + A trailing slash is not needed. Defaults to ``$data_root/media``. reports-root The file path of where custom reports are kept. Change this path if you require to store data on an other partition. A trailing slash is not - needed. By default, it will be stored into the installation directory - (``/opt/netbox/netbox/netbox/reports``). + needed. Defaults to ``$data_root/reports``. scripts-root The file path of where custom scripts are kept. Change this path if you require to store data on an other partition. A trailing slash is not - needed. By default, it will be stored into the installation directory - (``/opt/netbox/netbox/netbox/scripts``). + needed. Defaults to ``$data_root/scripts``. BOOLEAN PARAMETERS diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 89b89ac..fbcf253 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -161,12 +161,11 @@ else fi export LOGIN_REQUIRED -if [ -f "$__object/parameter/data-root" ]; then - data_root="$(cat "$__object/parameter/data-root")" - MEDIA_ROOT="$data_root/media" - REPORTS_ROOT="$data_root/reports" - SCRIPTS_ROOT="$data_root/scripts" -fi +data_root="$(cat "$__object/parameter/data-root")" +MEDIA_ROOT="$data_root/media" +REPORTS_ROOT="$data_root/reports" +SCRIPTS_ROOT="$data_root/scripts" + if [ -f "$__object/parameter/media-root" ]; then MEDIA_ROOT="$(cat "$__object/parameter/media-root")" fi diff --git a/type/__netbox/parameter/default/data-root b/type/__netbox/parameter/default/data-root new file mode 100644 index 0000000..45faab0 --- /dev/null +++ b/type/__netbox/parameter/default/data-root @@ -0,0 +1 @@ +/opt/netbox/data From 0c85b2d3fd3e9dee6a79091064e4fb7078adcdfb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 5 Oct 2020 18:26:47 +0200 Subject: [PATCH 54/66] __netbox: uninstall not anymore required packages This code tries to remove packages not needed anymore. As it presumably not removing dependencies, it's not that perfect pice of script. --- type/__netbox/gencode-remote | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d7bc219..1cd4939 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -36,18 +36,35 @@ ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || tr EOF cat << EOF +# backup requirement files +if [ -f /opt/netbox/requirements.txt]; then + cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt +else + # preseve file-not-found errors and warnings + touch /opt/netbox/old-requirements.txt +fi +cp '$src/requirements.txt' /opt/netbox/ + # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' -cp '$src/requirements.txt' /opt/netbox/ # backup dependency info ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' + # Setup & enter python virtualenv. # forcing python3 to be sure (till python4 gets released ..) virtualenv -p python3 /opt/netbox/venv +# Uninstall packages not required anymore +# if versions not shortend, they will be ignored by pip, but not by comm +# all of this could be done with grep, too, but it's still must be shortend with awk +awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" +awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" +comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ + | xargs /opt/netbox/venv/bin/pip3 uninstall -y + # Install python dependencies. # avoid gunicorn, because it will be done in an other type grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ @@ -57,7 +74,7 @@ EOF if [ -f "$__object/parameter/ldap-server" ]; then echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" else - echo "/opt/netbox/venv/bin/pip3 uninstall django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 uninstall -y django-auth-ldap" fi cat << EOF From 8f1b56026ca75b6a9d66dd40d40c239c34f6927b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 5 Oct 2020 19:02:05 +0200 Subject: [PATCH 55/66] __netbox: force link creation for config files This adds the force flag to `ln` to avoid aborts cause the link already exists and so forth. It also adds robustness to the execution. --- type/__netbox/gencode-remote | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 1cd4939..3b4e05c 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -48,9 +48,9 @@ cp '$src/requirements.txt' /opt/netbox/ # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' - -ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' -ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' +# force links to the cdist directory +ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. From 0b3bc1453073de6e45101e9386b5de1e8f93227a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 6 Oct 2020 17:29:31 +0200 Subject: [PATCH 56/66] __netbox: adjusted command flags Mostly added -q to the pip calls. Also forced the temp directory removal. --- type/__netbox/gencode-remote | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 3b4e05c..eb3bcd4 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -63,18 +63,18 @@ virtualenv -p python3 /opt/netbox/venv awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ - | xargs /opt/netbox/venv/bin/pip3 uninstall -y + | xargs /opt/netbox/venv/bin/pip3 uninstall -qy # Install python dependencies. # avoid gunicorn, because it will be done in an other type grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ - | xargs /opt/netbox/venv/bin/pip3 install + | xargs /opt/netbox/venv/bin/pip3 install -q EOF if [ -f "$__object/parameter/ldap-server" ]; then - echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 install -q django-auth-ldap" else - echo "/opt/netbox/venv/bin/pip3 uninstall -y django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 uninstall -qy django-auth-ldap" fi cat << EOF @@ -95,7 +95,7 @@ sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py invalid # Remove temporary working directory. cd / -rm -r "\$tmpdir" +rm -rf "\$tmpdir" # Save version after successful installation printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version From 3b780c4794b1c9df9e3f6582e7ad17ad1cfd0402 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 8 Oct 2020 19:54:48 +0200 Subject: [PATCH 57/66] __netbox: fixup small bugs and venv process killing Changed flag (force to ignore a non-existant directory), typo and swapped arguments are done. Also, the process to stop all processes from the virtal environment has changed: Now, it stops all potential services and ignore errors (because a service doesn't exist). After that, it sends a kill signal to all processes and then gracefully wait since there is no option to do that with systemd. --- type/__netbox/gencode-remote | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index eb3bcd4..1ef78cd 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -28,16 +28,24 @@ EOF # Stop everything in the pyenv to update cat << EOF -# Try to kill everything in the venv -systemctl -q --wait stop netbox gunicorn-netbox uwsgi-netbox || true -# don't know if this is required since using --wait -ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true +# Try to stop everything in the venv (ignore non-existant services) +systemctl -q stop netbox gunicorn-netbox uwsgi-netbox || true + +# kill and poll till all venv processes end +# at least if they are called with full path in the cmd (like in the services) +pids="\$( ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' )" +if [ "\$pids" ]; then + kill \$pids + while ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | grep -q . ; do + sleep 0.5 + done +fi EOF cat << EOF # backup requirement files -if [ -f /opt/netbox/requirements.txt]; then +if [ -f /opt/netbox/requirements.txt ]; then cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt else # preseve file-not-found errors and warnings @@ -46,7 +54,7 @@ fi cp '$src/requirements.txt' /opt/netbox/ # Deploy sources and restore configuration. -rm -r '$install_dir' +rm -rf '$install_dir' cp -r '$src/netbox' '$install_dir' # force links to the cdist directory ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' @@ -62,7 +70,7 @@ virtualenv -p python3 /opt/netbox/venv # all of this could be done with grep, too, but it's still must be shortend with awk awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" -comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ +comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" \ | xargs /opt/netbox/venv/bin/pip3 uninstall -qy # Install python dependencies. From 13e97d171bb700f577cdcfb03dde81a193b0c10c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:39:19 +0200 Subject: [PATCH 58/66] __netbox*: added systemd socket support The Gunicorn type now supports systemd sockets only. With uWSGI, you can choose between it and the native sockets based on the parameters chosen. This is done because it could not be implemented to have multiple protocols with the systemd sockets (so you may choose). The systemd socket unit file is generally available, so both types use the same script to generate the socket unit file. --- type/__netbox/files/netbox.socket.sh | 33 ++++++++++ type/__netbox_gunicorn/files/gunicorn.py.sh | 2 +- type/__netbox_gunicorn/files/netbox.service | 3 +- type/__netbox_gunicorn/files/netbox.socket.sh | 1 + type/__netbox_gunicorn/man.rst | 8 ++- type/__netbox_gunicorn/manifest | 22 +++---- type/__netbox_uwsgi/explorer/bind-capability | 12 ++++ .../{netbox.service => netbox.service.sh} | 18 +++++- type/__netbox_uwsgi/files/netbox.socket.sh | 1 + type/__netbox_uwsgi/files/uwsgi.ini.sh | 24 ++++---- type/__netbox_uwsgi/gencode-remote | 60 ++++++++++++++++++- type/__netbox_uwsgi/man.rst | 33 +++++++++- type/__netbox_uwsgi/manifest | 38 +++++++++++- .../__netbox_uwsgi/parameter/default/protocol | 1 + type/__netbox_uwsgi/parameter/optional | 1 + 15 files changed, 218 insertions(+), 39 deletions(-) create mode 100755 type/__netbox/files/netbox.socket.sh create mode 120000 type/__netbox_gunicorn/files/netbox.socket.sh create mode 100755 type/__netbox_uwsgi/explorer/bind-capability rename type/__netbox_uwsgi/files/{netbox.service => netbox.service.sh} (70%) mode change 100644 => 100755 create mode 120000 type/__netbox_uwsgi/files/netbox.socket.sh create mode 100644 type/__netbox_uwsgi/parameter/default/protocol diff --git a/type/__netbox/files/netbox.socket.sh b/type/__netbox/files/netbox.socket.sh new file mode 100755 index 0000000..03e3d44 --- /dev/null +++ b/type/__netbox/files/netbox.socket.sh @@ -0,0 +1,33 @@ +#!/bin/sh -e +# __netbox/files/netbox.socket.sh + +# This is shared between all WSGI-server types. + +# Arguments: +# 1: File which list all sockets to listen on (sepearated by \n) + +if [ $# -ne 1 ]; then + printf "netbox.socket.sh: argument \$1 missing or too much given!\n" >&2 + exit 1 +fi + + +cat << UNIT +[Unit] +Description=Socket for NetBox via $TYPE + +[Socket] +UNIT + +# read all sockets to listen to +while read line; do + printf "ListenStream=%s\n" "$line" +done < "$1" + +cat << UNIT +SocketUser=netbox +SocketGroup=www-data + +[Install] +WantedBy=sockets.target +UNIT diff --git a/type/__netbox_gunicorn/files/gunicorn.py.sh b/type/__netbox_gunicorn/files/gunicorn.py.sh index 862ba06..c1e6ee5 100755 --- a/type/__netbox_gunicorn/files/gunicorn.py.sh +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -13,7 +13,7 @@ cores="$(cat "$__explorer/cpu_cores")" cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = [$HOST ] +#bind = done via systemd socket 'gunicorn-netbox.socket' # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index dcf1a66..28b6b45 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -3,13 +3,14 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service Requires=netbox-rq.service +Requires=gunicorn-netbox.socket Wants=network.target After=netbox.service After=network.target After=redis-server.service postgresql.service [Service] -Type=simple +Type=notify User=netbox Group=netbox diff --git a/type/__netbox_gunicorn/files/netbox.socket.sh b/type/__netbox_gunicorn/files/netbox.socket.sh new file mode 120000 index 0000000..28ce920 --- /dev/null +++ b/type/__netbox_gunicorn/files/netbox.socket.sh @@ -0,0 +1 @@ +../../__netbox/files/netbox.socket.sh \ No newline at end of file diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 1c09d78..8860294 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -14,6 +14,9 @@ sockets. Static content must be served independent of Gunicorn. The Gunicorn daemon is available as the `gunicorn-netbox` systemd service, but also available via the `netbox` wrapper service. +It will use systemd socket activation to listen to the given sockets. This +should allow to bind to privileaged ports (all below 1024) and hot reloads. + REQUIRED PARAMETERS ------------------- @@ -38,8 +41,9 @@ state bind-to The hosts the gunicorn socket should be bind to. Formats are `IP`, - `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple - times. Defaults to ``127.0.0.1:8001``. + `IP:PORT`, `PATH` or anything other that systemd socket units will + understand as stream. Parameter can be set multiple times. Defaults + to ``127.0.0.1:8001``. BOOLEAN PARAMETERS diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index 2e5ccce..5748e9d 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -25,17 +25,10 @@ case "$param_state" in esac -if [ "$state" = "present" ]; then - HOST="" - while read -r host; do - # shellcheck disable=SC2089 - HOST="$HOST '$host'," - done < "$__object/parameter/bind-to" - # shellcheck disable=SC2090 - export HOST +mkdir "$__object/files" +if [ "$state" = "present" ]; then # process template - mkdir "$__object/files" "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" # gunicorn config file @@ -49,7 +42,16 @@ else fi -# install service file +TYPE="Gunicorn" +export TYPE + +"$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \ + > "$__object/files/netbox.socket" + +# install systemd files +__systemd_unit gunicorn-netbox.socket \ + --state "$state" --enablement-state "$unit_state" \ + --source "$__object/files/netbox.socket" --restart __systemd_unit gunicorn-netbox.service \ --state "$state" --enablement-state "$unit_state" \ --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/explorer/bind-capability b/type/__netbox_uwsgi/explorer/bind-capability new file mode 100755 index 0000000..c5c0365 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/bind-capability @@ -0,0 +1,12 @@ +#!/bin/sh -e +# explorer/bind-capablility + +# Checks if the uWSGI binary have the capability to bind to privileaged ports +# as a non-root user. It's required if no systemd sockets are used (cause of +# the use of multiple protocols etc.) + +binary="/opt/netbox/venv/bin/uwsgi" +# -v verifies if capability is set +if setcap -q -v CAP_NET_BIND_SERVICE+ep "$binary"; then + echo set +fi diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service.sh old mode 100644 new mode 100755 similarity index 70% rename from type/__netbox_uwsgi/files/netbox.service rename to type/__netbox_uwsgi/files/netbox.service.sh index cd06e82..3705769 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service.sh @@ -1,15 +1,26 @@ +#!/bin/sh -e + +cat << EOF [Unit] Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service Requires=netbox-rq.service +EOF + +# Add dependency to own socket +if [ "$(cat "$__object/files/systemd_socket")" = "yes" ]; then + echo "Requires=uwsgi-netbox.socket" +fi + +cat << EOF Wants=network.target After=netbox.service After=network.target After=redis-server.service postgresql.service [Service] -Type=simple +Type=notify User=netbox Group=netbox @@ -17,8 +28,8 @@ WorkingDirectory=/opt/netbox ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi -ExecReload=kill -HUP $MAINPID -ExecStop=kill -INT $MAINPID +ExecReload=kill -HUP \$MAINPID +ExecStop=kill -INT \$MAINPID KillSignal=SIGQUIT Restart=on-failure @@ -26,3 +37,4 @@ RestartSec=30 [Install] WantedBy=netbox.service +EOF diff --git a/type/__netbox_uwsgi/files/netbox.socket.sh b/type/__netbox_uwsgi/files/netbox.socket.sh new file mode 120000 index 0000000..28ce920 --- /dev/null +++ b/type/__netbox_uwsgi/files/netbox.socket.sh @@ -0,0 +1 @@ +../../__netbox/files/netbox.socket.sh \ No newline at end of file diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 7835a3e..4bae613 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -28,20 +28,16 @@ cat << EOF ; socket(s) to bind EOF -# special protocol to bind -while read -r param; do - if [ -z "$param" ]; then continue; fi # ignore empty lines from the here-doc - - multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" - socket_changes="yes" - -done << INPUT # here-doc cause of SC2031 -$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print ) -INPUT - -# else, default bind to -if [ -z "$socket_changes" ]; then - multi_options "socket" "$__object/parameter/bind-to" +if [ "$SYSTEMD_SOCKET" != "yes" ]; then + # special protocol to bind + find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print \ + | while read -r param; do + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" + done +else + # else, systemd will offer socket + echo "; sockets managed via 'uwsgi-netbox.socket'" + printf "protocol = %s\n" "$PROTOCOL" fi diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index c219643..7c3b826 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -3,18 +3,67 @@ # control state state="$(cat "$__object/parameter/state")" +# Set capabilities to aquire privileaged ports as netbox user. Two modes are +# available to efficiently set capabilites. Assumes libcap-bin is installed as +# default on debian systems. +# +# Arguments: +# 1: mode to detect if capabilites are required to set ('set' or 'correct') +set_bind_cap() { + cap_mode="" # reset variable from the execution before + + # check if capabilites are required after given mode + case "$1" in + # assumes capabilites are not set (cause of new binaries) + set) + if [ "$SYSTEMD_SOCKET" != "yes" ]; then + cap_mode="+ep" + fi + ;; + + # check if capabilities have changed + correct) + if [ -s "$__object/explorer/bind-capability" ]; then + # capabilites are set + if [ "$SYSTEMD_SOCKET" = "yes" ]; then + cap_mode="-ep" # unset + fi + else + # capabilities are unset + if [ "$SYSTEMD_SOCKET" != "yes" ]; then + cap_mode="+ep" # set + fi + fi + ;; + + # faulty mode + *) + echo "called set_bind_cap incorrect (\$1 missing)" >&2 + ;; + esac + + # set capabilities if any + if [ "$cap_mode" ]; then + printf "setcap -q CAP_NET_BIND_SERVICE%s /opt/netbox/venv/bin/uwsgi\n" "$cap_mode" + fi +} +SYSTEMD_SOCKET="$(cat "$__object/files/systemd_socket")" + + case "$state" in # install uwsgi enabled|disabled) # not installed if ! [ -s "$__object/explorer/installed" ]; then - echo "/opt/netbox/venv/bin/pip3 install uwsgi" + echo "/opt/netbox/venv/bin/pip3 install -q uwsgi" + set_bind_cap set do_restart=yes printf "installed\n" >> "$__messages_out" # updates available elif [ -s "$__object/explorer/upgradeable" ]; then - echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + echo "/opt/netbox/venv/bin/pip3 install -q --upgrade uwsgi" + set_bind_cap set do_restart=yes printf "upgraded\n" >> "$__messages_out" fi @@ -25,6 +74,11 @@ case "$state" in printf "configured\n" >> "$__messages_out" fi + # if no capabilities were set yet, check if any are required + if [ -z "$cap_mode" ]; then + set_bind_cap correct + fi + # restart uwsgi if [ "$do_restart" ] && [ "$state" != "disabled" ]; then @@ -40,7 +94,7 @@ EOF # check if installed if [ -s "$__object/explorer/installed" ]; then # service already disabled - echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi" + echo "/opt/netbox/venv/bin/pip3 uninstall -qy uwsgi" printf "uninstalled\n" >> "$__messages_out" fi ;; diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 3fb8515..d54401e 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -15,6 +15,19 @@ protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This application is available via the `uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper service, too. +**As uWSGI will be started as netbox user, it does not have privileges to +bind to a privileaged port (all ports below 1024).** Because uWSGI will +drop privileages anyway before binding to a port, solutions are to use +the systemd sockets to activate the ports as root or set linux kernel +capabilites to bind to such a privileaged port. + +As systemd sockets (or uwsgi itself) do not allow to distinguish multiple +sockets if different protocols are used for different sockets, this type does +not use systemd sockets if it is requested from the user. Using the +``--bind-to`` and ``--protocol`` parameters, it uses the systemd socket +activation. Else, it set the different sockets and protocols natively to uwsgi +and add kernel capabilities to be able to listen to privileaged ports. + REQUIRED PARAMETERS ------------------- @@ -39,8 +52,18 @@ state bind-to - The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. - Defaults to ``127.0.0.1:3031``. Can be set multiple times. + The socket uwsgi should bind to. Must be UNIX/TCP (or anything that + systemd sockets accept as stream). Defaults to ``127.0.0.1:3031``. Can be + set multiple times. The used protocol is defined by ``--protocol``. + + **By setting up the socket via this parameter, it uses systemd sockets to + handle these.** This parameter will be ignored if a more detailed paramter + is given (``--$proto-bind``). + +protocol + The protocol which should be used for the socket given by the ``--bind-to`` + parameter. Possible values are ``uwsgi``, ``http``, ``fastcgi`` and + ``scgi``. If nothing given, it defaults to ``uwsgi``. uwsgi-bind http-bind @@ -50,6 +73,12 @@ scgi-bind ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must be a UNIX/TCP socket. Can be set multiple times. + **By using such parameters instead of ``--bind-to``, no systemd sockets + will be used because it can not handle sockets for multiple protocols.** + Instead, the native socket binding will be used. It will add kernel + capabilites to bind to privileaged ports, too. This allow binds to ports + like 80 as netbox user. + BOOLEAN PARAMETERS ------------------ diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index c5885c9..7c593e8 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -25,15 +25,30 @@ case "$param_state" in esac +mkdir "$__object/files" + +# check if systemd sockets will be used +if [ -f "$__object/parameter/bind-to" ]; then + SYSTEMD_SOCKET="yes" +fi +if find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print -quit | grep -q .; then + SYSTEMD_SOCKET="no" +fi +echo "$SYSTEMD_SOCKET" > "$__object/files/systemd_socket" + if [ "$state" = "present" ]; then - # *bind* parameters are directly processed in the gen script + # already checked outside this if-clause + export SYSTEMD_SOCKET + + PROTOCOL="$(cat "$__object/parameter/protocol")" + export PROTOCOL + if [ -f "$__object/parameter/serve-static" ]; then STATIC_MAP="yes" export STATIC_MAP fi # process template - mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" # uwsgi config file @@ -48,7 +63,24 @@ else fi +# handle the systemd socket +if [ "$SYSTEMD_SOCKET" = "yes" ]; then + TYPE="uWSGI" + export TYPE + + # generate and set the socket unit + "$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \ + > "$__object/files/netbox.socket" + __systemd_unit uwsgi-netbox.socket \ + --state "$state" --enablement-state "$unit_state" \ + --source "$__object/files/netbox.socket" --restart +else + # remove the systemd socket unit + __systemd_unit uwsgi-netbox.socket --state absent +fi + # install service file +"$__type/files/netbox.service.sh" > "$__object/files/netbox.service" __systemd_unit uwsgi-netbox.service \ --state "$state" --enablement-state "$unit_state" \ - --source "$__type/files/netbox.service" --restart + --source "$__object/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/parameter/default/protocol b/type/__netbox_uwsgi/parameter/default/protocol new file mode 100644 index 0000000..caf986e --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/protocol @@ -0,0 +1 @@ +uwsgi diff --git a/type/__netbox_uwsgi/parameter/optional b/type/__netbox_uwsgi/parameter/optional index ff72b5c..3284ccc 100644 --- a/type/__netbox_uwsgi/parameter/optional +++ b/type/__netbox_uwsgi/parameter/optional @@ -1 +1,2 @@ state +protocol From 32c95ccb220cb3302d1c43cca5ace01fa66a6755 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:50:22 +0200 Subject: [PATCH 59/66] __netbox: don't call `pip3 uninstall` if nothing available To avoid an error from pip, it will be checked if there are packages to uninstall. This is better than just doing `|| true`. --- type/__netbox/gencode-remote | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 1ef78cd..07b0fcf 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -66,12 +66,16 @@ ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Uninstall packages not required anymore -# if versions not shortend, they will be ignored by pip, but not by comm +# if versions not be shortend, they will be ignored by pip, but not by comm # all of this could be done with grep, too, but it's still must be shortend with awk awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" -comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" \ - | xargs /opt/netbox/venv/bin/pip3 uninstall -qy +comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" > "\$tmpdir/pip-uninstall.txt" + +# only uninstall if something is available (to avoid errors cause of this) +if [ -s "\$tmpdir/pip-uninstall.txt" ]; then + /opt/netbox/venv/bin/pip3 uninstall -qy -r "\$tmpdir/pip-uninstall.txt" +fi # Install python dependencies. # avoid gunicorn, because it will be done in an other type @@ -86,6 +90,7 @@ EOF fi cat << EOF + # Set final permissions. chown -R netbox /opt/netbox From 72f2996c509e64a9f35f53d7b347ae65d1e6df59 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:56:54 +0200 Subject: [PATCH 60/66] __netbox*: fix shellcheck SC2162 --- type/__netbox/files/netbox.socket.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/netbox.socket.sh b/type/__netbox/files/netbox.socket.sh index 03e3d44..2ef9e81 100755 --- a/type/__netbox/files/netbox.socket.sh +++ b/type/__netbox/files/netbox.socket.sh @@ -20,7 +20,7 @@ Description=Socket for NetBox via $TYPE UNIT # read all sockets to listen to -while read line; do +while read -r line; do printf "ListenStream=%s\n" "$line" done < "$1" From 144cae3b796831d4c96e221611b25945f4462f68 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 10:16:25 +0200 Subject: [PATCH 61/66] __netbox: use __pyenv and remove obsolete netbox stop Now, the python venv is now created via `pyvenv` or `python3 -m venv` instead of the legacy `virtualenv`. For this, not all python processes from the venv need to be stopped. Migration from previous versions of this type may be difficult, but solvable if the venv will be recreated. --- type/__netbox/gencode-remote | 32 +++++++------------------------- type/__netbox/manifest | 2 ++ 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 07b0fcf..591717f 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -24,26 +24,7 @@ cd "\$tmpdir" curl -sS -L '$url' > '$archive' tar xf '$archive' -EOF - # Stop everything in the pyenv to update - cat << EOF -# Try to stop everything in the venv (ignore non-existant services) -systemctl -q stop netbox gunicorn-netbox uwsgi-netbox || true - -# kill and poll till all venv processes end -# at least if they are called with full path in the cmd (like in the services) -pids="\$( ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' )" -if [ "\$pids" ]; then - kill \$pids - while ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | grep -q . ; do - sleep 0.5 - done -fi - -EOF - - cat << EOF # backup requirement files if [ -f /opt/netbox/requirements.txt ]; then cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt @@ -61,9 +42,7 @@ ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' -# Setup & enter python virtualenv. -# forcing python3 to be sure (till python4 gets released ..) -virtualenv -p python3 /opt/netbox/venv +# virtualenv is given already by __pyvenv, just using it # Uninstall packages not required anymore # if versions not be shortend, they will be ignored by pip, but not by comm @@ -112,6 +91,7 @@ rm -rf "\$tmpdir" # Save version after successful installation printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version + EOF # meta @@ -127,10 +107,12 @@ if grep -q "^__file/opt/netbox/" "$__messages_in"; then fi -# check for changes +# Check for changes if [ "$changes" = "yes" ]; then - # to avoid database corruption at config changes, both services must be - # stopped at the same time (noted in the manual, too). + # After the upstream upgrade.sh script, it's ok to migrate while the + # application is running ;) + + # restarting after changes cat << EOF # Restart service. All required services are included with netbox.service. systemctl restart netbox diff --git a/type/__netbox/manifest b/type/__netbox/manifest index fbcf253..475cadb 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -187,6 +187,8 @@ fi # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home +# Generate python environment (user will be set by gencode-remote) +require="__user/netbox" __pyvenv /opt/netbox/venv/ # Generate and upload netbox configuration. mkdir -p "$__object/files" From 9def5700c8a1de7e59ed0ac3c40cdd1b7c484e05 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 10:21:52 +0200 Subject: [PATCH 62/66] __netbox: replace redis DEFAULT_TIMEOUT with RQ_DEFAULT_TIMEOUT The first one was deprecated by the second one. --- type/__netbox/files/configuration.py.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 06a1c86..31ebd05 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -36,7 +36,6 @@ REDIS = { # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '$REDIS_PASSWORD', 'DATABASE': $((REDIS_DBID_OFFSET + 0)), - 'DEFAULT_TIMEOUT': 300, 'SSL': $REDIS_SSL, }, 'caching': { @@ -47,10 +46,10 @@ REDIS = { # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '$REDIS_PASSWORD', 'DATABASE': $((REDIS_DBID_OFFSET + 1)), - 'DEFAULT_TIMEOUT': 300, 'SSL': $REDIS_SSL, } } +RQ_DEFAULT_TIMEOUT = 300 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and From 558021d2eeb10697d2dec0e7dc563fbf26e63ba0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 11:08:11 +0200 Subject: [PATCH 63/66] __netbox: reorder generated code for shorter transition Now, the venv and pip things are done before the program files are replaced, which should minimize the outage where the application is not available. --- type/__netbox/gencode-remote | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 591717f..5d4b7be 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -25,30 +25,22 @@ curl -sS -L '$url' > '$archive' tar xf '$archive' +# virtualenv is given already by __pyvenv, just using it + # backup requirement files if [ -f /opt/netbox/requirements.txt ]; then - cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt + mv /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt else # preseve file-not-found errors and warnings touch /opt/netbox/old-requirements.txt fi cp '$src/requirements.txt' /opt/netbox/ -# Deploy sources and restore configuration. -rm -rf '$install_dir' -cp -r '$src/netbox' '$install_dir' -# force links to the cdist directory -ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' -ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' - - -# virtualenv is given already by __pyvenv, just using it - # Uninstall packages not required anymore # if versions not be shortend, they will be ignored by pip, but not by comm # all of this could be done with grep, too, but it's still must be shortend with awk -awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" -awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" +awk -F== '{print \$1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" +awk -F== '{print \$1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" > "\$tmpdir/pip-uninstall.txt" # only uninstall if something is available (to avoid errors cause of this) @@ -70,9 +62,17 @@ EOF cat << EOF +# Deploy sources and restore configuration. +rm -rf '$install_dir' +cp -r '$src/netbox' '$install_dir' +# force links to the cdist directory +ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' + # Set final permissions. chown -R netbox /opt/netbox + # NetBox manage scripts # Run database migrations. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate From 46bff41f6d8bb9a0fd50ac7b659690f5e94377dc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 11:22:47 +0200 Subject: [PATCH 64/66] __netbox{,_uwsgi}: updated manpages Mostly added notes of what does not work that well. --- type/__netbox/man.rst | 28 +++++++++++++--------------- type/__netbox_uwsgi/man.rst | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 6d50268..f34ead7 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -16,7 +16,7 @@ It setup systemd unit files for the services `netbox` and `netbox-rq`. The `netbox` service only wrap all netbox related services, e.g. restarting and so one will be delegated to all related services. -The application is still not accessable because WSGI server is required. To +The application is still not accessable because a WSGI server is required. To access the application through WSGI, uWSGI or Gunicorn can be used. The setup can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. @@ -236,23 +236,21 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. -If you change a configuration, the database may go corrupt if two instances of -the application are running with different configurations at the same time. -This most commonly happens when the WSGI server and RQ-worker restarts after a -configuration change. This occours in the following case for example: +You may also be interested of writing a own type which handles the creation of +the super user. To do this non-interactivly, see the ansible role as `reference +`. -.. code-block:: sh +If you change the secret key while the netbox instance is running, there is a +time frame where the access to the application corrupts the whole database. +Then, you need to restore a backup or wipe the database. - systemctl restart gunicorn-netbox # WSGI-server already online with new - # configuration after this command. - systemctl restart netbox-rq # RQ-Worker still worked with the old - # configuration till here. +Currently, the cause is not clear, but it should work if you do not touch +netbox while the configuration is done (do not shut it down, too). It only +applies for changes of the secret key, which not happen normally. -This type handles the restart of both services correctly to avoid such database -corruptions. To safely manual restart the whole netbox instance manual, simply -restart all services in one ``systemctl restart netbox`` command, as it ensures -that first all services are shut down before starting one of them. The service -``netbox`` wraps all required services that are available. +Maybe the `--restart` flag for the `__systemd_unit` types is not the best idea, +but avoids that the changes will not be applied. It could be solved if the type +would send messages from his actions. SEE ALSO diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index d54401e..c5e51ca 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -129,6 +129,7 @@ EXAMPLES # with multiple protocols # parameter `--bind-to` will be ignored + # avoids systemd sockets, but can handle multiple protocols __netbox $args require="__netbox" __netbox_uwsgi --uwsgi-bind 0.0.0.0:3031 \ --http-bind 0.0.0.0:8080 \ @@ -150,6 +151,20 @@ EXAMPLES require="__netbox_uwsgi" __systemd_service uwsgi-netbox --state stopped +NOTES +----- +If systemd sockets are used, uwsgi can not be reloaded because it does not +handle the socket correctly. It works by completly restarting uwsgi (because +it is near the same cause of the systemd socket) or tweaking the service unit +with the line ``StandardInput=socket``, which limits you to only one address +to bind to (else, the service will not start). + +Maybe someone is interested in enabling log files, because the "log to stdout" +is not the fanciest approach (because it is shown in the journal). See the +`uwsgi documentation ` for +reference. + + SEE ALSO -------- `uWSGI Documentation `_ From c1d5d5b088724c22c8b8e79f00a1b82ed88436f2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 08:52:43 +0200 Subject: [PATCH 65/66] __netbox: remove dependency to package virtualenv This is not required any more because it was completly replaced by `python3-venv`. The venv handling is already given to `__pyvenv`. --- type/__netbox/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 475cadb..4be49bd 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -6,7 +6,7 @@ case "$os" in debian|ubuntu) # Install netbox dependencies. for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl sudo; do __package $pkg done From 87e115da98425c69d8bb37397ccdf47c0f2a4dc2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 25 Oct 2020 19:13:10 +0100 Subject: [PATCH 66/66] __netbox: some manpage fixes Mostly invalid links .. --- type/__netbox/man.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index f34ead7..135304c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -100,8 +100,7 @@ redis-password redis-dbid-offset Offset to set the redis database id's. The `tasks` database id is - `offset + 0` and `caching` is `offset + 1`. The offset defaults - to ``0``. + `offset + 0` and `caching` is `offset + 1`. The offset defaults to ``0``. smtp-host Host of the SMTP email server. Defaults to ``localhost``. @@ -172,12 +171,12 @@ redis-ssl smtp-use-tls Uses TLS to connect to the SMTP email server. `See documentation - `_ for more information. smtp-use-ssl Uses implicit TLS with the SMTP email server. `See documentation - `_ for more information. login-required @@ -229,16 +228,16 @@ is a more good way to inject python code for dynamic configuration variables, you are welcome to contribute! - `Possible optional settings - ` + `_ If you not setup ldap authentification, you may be interested into how to `setting up a super user -` +`_ directly on the machine to be able to access and use NetBox. You may also be interested of writing a own type which handles the creation of the super user. To do this non-interactivly, see the ansible role as `reference -`. +`_. If you change the secret key while the netbox instance is running, there is a time frame where the access to the application corrupts the whole database.