From 05a6b8aa42e0c3d34dd9f4512a8e631c7c28f05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 15 May 2020 07:57:16 +0200 Subject: [PATCH 001/156] Import __matterbridge type from ungleich's dot-cdist --- .../files/matterbridge.service.sh | 18 ++++ type/__matterbridge/man.rst | 56 ++++++++++ type/__matterbridge/manifest | 101 ++++++++++++++++++ type/__matterbridge/parameter/required | 2 + type/__matterbridge/singleton | 0 5 files changed, 177 insertions(+) create mode 100755 type/__matterbridge/files/matterbridge.service.sh create mode 100644 type/__matterbridge/man.rst create mode 100755 type/__matterbridge/manifest create mode 100644 type/__matterbridge/parameter/required create mode 100644 type/__matterbridge/singleton diff --git a/type/__matterbridge/files/matterbridge.service.sh b/type/__matterbridge/files/matterbridge.service.sh new file mode 100755 index 0000000..9dbd1cb --- /dev/null +++ b/type/__matterbridge/files/matterbridge.service.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +cat <`_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__matterbridge/manifest b/type/__matterbridge/manifest new file mode 100755 index 0000000..56f470a --- /dev/null +++ b/type/__matterbridge/manifest @@ -0,0 +1,101 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "$__global/explorer/os") +case "$os" in + debian) + # This type assume systemd for service installation. + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +# Required parameters. +VERSION=$(cat "$__object/parameter/version") +if [ -f "$__object/parameter/config" ]; then + CONFIG="$(cat "$__object/parameter/config")" + if [ "$CONFIG" = "-" ]; then + CONFIG=$(cat "$__object/stdin") + fi +fi + +# Hardcoded values used in templates. +export BINARY_PATH=/usr/local/bin/matterbridge +export CONFIG_PATH=/etc/matterbridge/matterbridge.toml +export USER=matterbridge +export GROUP=$USER + +# Internal variables. +artefact="matterbridge-$VERSION-linux-64bit" +checksum_file="checksums.txt" +release_download_url=https://github.com/42wim/matterbridge/releases/download +binary_url="$release_download_url/v$VERSION/$artefact" +checksum_file_url="$release_download_url/v$VERSION/$checksum_file" +config_dir=$(dirname $CONFIG_PATH) +systemd_unit_path='/etc/systemd/system/matterbridge.service' + +# Check if curl is available. +if [ ! -x "$(which curl)" ]; then + echo "curl is required for this type, but could not be found. Exiting." &>2 + exit 1 +fi + +# Initialize working directory. +mkdir -p "$__object/files" + +# Download and check matterbridge binary. +curl -L $binary_url -o "$__object/files/$artefact" +curl -Ls $checksum_file_url | grep $artefact > "$__object/files/$checksum_file" +ls $__object/files/ >&2 +cat $__object/files/checksums.txt >&2 +(cd "$__object/files"; sha256sum --check $checksum_file) +if [ $? -ne 0 ]; then + echo "Matterbridge binary checksum failed." >&2 + exit 1 +fi + +# Create service user. +__user $USER --home "/var/lib/$USER" + +# Deploy matterbridge binary. +require="__user/$USER" __file "$BINARY_PATH" \ + --source "$__object/files/$artefact" \ + --owner "$USER" --mode 755 + +# Generate and deploy configuration file. +"$__type/files/matterbridge.service.sh" > "$__object/files/matterbridge.service" + +require="__user/$USER" __directory "$config_dir" \ + --owner "$USER" --mode 0755 --parents \ + +require="__directory/$config_dir" __file "$CONFIG_PATH" \ + --owner "$USER" \ + --mode 0640 \ + --source "$CONFIG" + +__file "$systemd_unit_path" \ + --source "$__object/files/matterbridge.service" + +# Deal with init system. +require="__file/$systemd_unit_path" __start_on_boot matterbridge +require="__file/$BINARY_PATH __file/$CONFIG_PATH __file/$systemd_unit_path" __service matterbridge --action restart diff --git a/type/__matterbridge/parameter/required b/type/__matterbridge/parameter/required new file mode 100644 index 0000000..ed5d8b3 --- /dev/null +++ b/type/__matterbridge/parameter/required @@ -0,0 +1,2 @@ +version +config diff --git a/type/__matterbridge/singleton b/type/__matterbridge/singleton new file mode 100644 index 0000000..e69de29 From 43562ff09c7ab44f42e37893df5dc48af1b1e15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 15 May 2020 08:08:45 +0200 Subject: [PATCH 002/156] Make __matterbridge shellcheck-friendly --- type/__matterbridge/manifest | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/type/__matterbridge/manifest b/type/__matterbridge/manifest index 56f470a..ef02112 100755 --- a/type/__matterbridge/manifest +++ b/type/__matterbridge/manifest @@ -55,8 +55,8 @@ config_dir=$(dirname $CONFIG_PATH) systemd_unit_path='/etc/systemd/system/matterbridge.service' # Check if curl is available. -if [ ! -x "$(which curl)" ]; then - echo "curl is required for this type, but could not be found. Exiting." &>2 +if ! command -v curl; then + echo "curl is required for this type, but could not be found. Exiting." >&2 exit 1 fi @@ -64,12 +64,9 @@ fi mkdir -p "$__object/files" # Download and check matterbridge binary. -curl -L $binary_url -o "$__object/files/$artefact" -curl -Ls $checksum_file_url | grep $artefact > "$__object/files/$checksum_file" -ls $__object/files/ >&2 -cat $__object/files/checksums.txt >&2 -(cd "$__object/files"; sha256sum --check $checksum_file) -if [ $? -ne 0 ]; then +curl -L "$binary_url" -o "$__object/files/$artefact" +curl -Ls "$checksum_file_url" | grep "$artefact" > "$__object/files/$checksum_file" +if ! (cd "$__object/files"; sha256sum --check $checksum_file); then echo "Matterbridge binary checksum failed." >&2 exit 1 fi From f4671691bee30b89054a60523f9177ac15fa8790 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 29 May 2020 17:33:23 +0200 Subject: [PATCH 003/156] Rename __root_mail_dma to __dma to prepare for new types --- type/{__root_mail_dma => __dma}/files/aliases | 0 type/{__root_mail_dma => __dma}/gencode-remote | 0 type/{__root_mail_dma => __dma}/man.rst | 6 +++--- type/{__root_mail_dma => __dma}/manifest | 0 type/{__root_mail_dma => __dma}/parameter/boolean | 0 type/{__root_mail_dma => __dma}/parameter/optional | 0 type/{__root_mail_dma => __dma}/parameter/required | 0 type/{__root_mail_dma => __dma}/parameter/required_multiple | 0 type/{__root_mail_dma => __dma}/singleton | 0 9 files changed, 3 insertions(+), 3 deletions(-) rename type/{__root_mail_dma => __dma}/files/aliases (100%) rename type/{__root_mail_dma => __dma}/gencode-remote (100%) rename type/{__root_mail_dma => __dma}/man.rst (94%) rename type/{__root_mail_dma => __dma}/manifest (100%) rename type/{__root_mail_dma => __dma}/parameter/boolean (100%) rename type/{__root_mail_dma => __dma}/parameter/optional (100%) rename type/{__root_mail_dma => __dma}/parameter/required (100%) rename type/{__root_mail_dma => __dma}/parameter/required_multiple (100%) rename type/{__root_mail_dma => __dma}/singleton (100%) diff --git a/type/__root_mail_dma/files/aliases b/type/__dma/files/aliases similarity index 100% rename from type/__root_mail_dma/files/aliases rename to type/__dma/files/aliases diff --git a/type/__root_mail_dma/gencode-remote b/type/__dma/gencode-remote similarity index 100% rename from type/__root_mail_dma/gencode-remote rename to type/__dma/gencode-remote diff --git a/type/__root_mail_dma/man.rst b/type/__dma/man.rst similarity index 94% rename from type/__root_mail_dma/man.rst rename to type/__dma/man.rst index ecf2885..a10c6c2 100644 --- a/type/__root_mail_dma/man.rst +++ b/type/__dma/man.rst @@ -1,9 +1,9 @@ -cdist-type__root_mail_dma(7) +cdist-type__dma(7) ============================ NAME ---- -cdist-type__root_mail_dma - Setup root email with the DragonFly Mail Agent +cdist-type__dma - Setup the DragonFly Mail Agent as the MTA. DESCRIPTION @@ -57,7 +57,7 @@ EXAMPLES # Send root email to both our BOFH and the nice-admin. # That way they can figure things out together. - __root_mail_dma \ + __dma \ --root-email bofh@domain.tld \ --root-email nice-admin@domain.tld \ --smart-host mx1.domain.tld \ diff --git a/type/__root_mail_dma/manifest b/type/__dma/manifest similarity index 100% rename from type/__root_mail_dma/manifest rename to type/__dma/manifest diff --git a/type/__root_mail_dma/parameter/boolean b/type/__dma/parameter/boolean similarity index 100% rename from type/__root_mail_dma/parameter/boolean rename to type/__dma/parameter/boolean diff --git a/type/__root_mail_dma/parameter/optional b/type/__dma/parameter/optional similarity index 100% rename from type/__root_mail_dma/parameter/optional rename to type/__dma/parameter/optional diff --git a/type/__root_mail_dma/parameter/required b/type/__dma/parameter/required similarity index 100% rename from type/__root_mail_dma/parameter/required rename to type/__dma/parameter/required diff --git a/type/__root_mail_dma/parameter/required_multiple b/type/__dma/parameter/required_multiple similarity index 100% rename from type/__root_mail_dma/parameter/required_multiple rename to type/__dma/parameter/required_multiple diff --git a/type/__root_mail_dma/singleton b/type/__dma/singleton similarity index 100% rename from type/__root_mail_dma/singleton rename to type/__dma/singleton From a491e8739efb6753d52ede2ee58ba5a93247d1b4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 29 May 2020 17:33:40 +0200 Subject: [PATCH 004/156] Add __dma_auth type --- type/__dma_auth/gencode-remote | 20 +++++++++++++ type/__dma_auth/man.rst | 53 ++++++++++++++++++++++++++++++++++ type/__dma_auth/manifest | 30 +++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100755 type/__dma_auth/gencode-remote create mode 100644 type/__dma_auth/man.rst create mode 100755 type/__dma_auth/manifest diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote new file mode 100755 index 0000000..77ad9d2 --- /dev/null +++ b/type/__dma_auth/gencode-remote @@ -0,0 +1,20 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + diff --git a/type/__dma_auth/man.rst b/type/__dma_auth/man.rst new file mode 100644 index 0000000..cd4f647 --- /dev/null +++ b/type/__dma_auth/man.rst @@ -0,0 +1,53 @@ +cdist-type__dma_auth(7) +======================= + +NAME +---- +cdist-type__dma_auth - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __dma_auth + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Dennis Camera + + +COPYING +------- +Copyright \(C) 2020 Dennis Camera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__dma_auth/manifest b/type/__dma_auth/manifest new file mode 100755 index 0000000..e0c809a --- /dev/null +++ b/type/__dma_auth/manifest @@ -0,0 +1,30 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac From 98496aa8e5c8463b59263f20daa162bed9b63dfa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 29 May 2020 17:33:52 +0200 Subject: [PATCH 005/156] Add __mail_alias type --- type/__mail_alias/gencode-remote | 20 ++++++++++++ type/__mail_alias/man.rst | 53 ++++++++++++++++++++++++++++++++ type/__mail_alias/manifest | 30 ++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100755 type/__mail_alias/gencode-remote create mode 100644 type/__mail_alias/man.rst create mode 100755 type/__mail_alias/manifest diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote new file mode 100755 index 0000000..77ad9d2 --- /dev/null +++ b/type/__mail_alias/gencode-remote @@ -0,0 +1,20 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst new file mode 100644 index 0000000..d9563a6 --- /dev/null +++ b/type/__mail_alias/man.rst @@ -0,0 +1,53 @@ +cdist-type__mail_alias(7) +========================= + +NAME +---- +cdist-type__mail_alias - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __mail_alias + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Dennis Camera + + +COPYING +------- +Copyright \(C) 2020 Dennis Camera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__mail_alias/manifest b/type/__mail_alias/manifest new file mode 100755 index 0000000..e0c809a --- /dev/null +++ b/type/__mail_alias/manifest @@ -0,0 +1,30 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac From 3adc4f160998776033934d65ba22f1666c9e0b4b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 30 May 2020 17:24:26 +0200 Subject: [PATCH 006/156] [type/__mail_alias] Implement type --- type/__mail_alias/explorer/aliases | 59 ++++++++++ .../{manifest => explorer/aliases_file} | 38 ++++-- type/__mail_alias/gencode-remote | 108 ++++++++++++++++++ type/__mail_alias/man.rst | 21 +++- type/__mail_alias/parameter/default/state | 1 + type/__mail_alias/parameter/optional | 1 + type/__mail_alias/parameter/optional_multiple | 1 + 7 files changed, 215 insertions(+), 14 deletions(-) create mode 100755 type/__mail_alias/explorer/aliases rename type/__mail_alias/{manifest => explorer/aliases_file} (56%) mode change 100755 => 100644 create mode 100644 type/__mail_alias/parameter/default/state create mode 100644 type/__mail_alias/parameter/optional create mode 100644 type/__mail_alias/parameter/optional_multiple diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases new file mode 100755 index 0000000..ce1a439 --- /dev/null +++ b/type/__mail_alias/explorer/aliases @@ -0,0 +1,59 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Find aliases for a given name and print the aliases line separated + +aliases_file=$("${__type_explorer}/aliases_file") +test -r "${aliases_file}" || exit 0 + +name=$__object_id + +awk -F ':[[:blank:]]*' ' +function print_aliases (aliases, matches) { + split(aliases, matches, /,[[:blank:]]*/) + for (i in matches) { + gsub(/^[[:blank:]]*|[[:blank:]]*$/, "", matches[i]) + print matches[i] + } +} + +/^#/ { + # comment + select = 0; cont = 0; next +} + +{ + cont = ($0 ~ /\\$/) + if (cont) sub(/[[:blank:]]*\\$/, "", $0) +} + +/^[[:blank:]]/ || cont { + # continuation line + if (select) print_aliases($0) + next +} + +$1 == ENVIRON["__object_id"] { + select = 1 + print_aliases($2) + next +} + +{ select = 0 } +' "${aliases_file}" diff --git a/type/__mail_alias/manifest b/type/__mail_alias/explorer/aliases_file old mode 100755 new mode 100644 similarity index 56% rename from type/__mail_alias/manifest rename to type/__mail_alias/explorer/aliases_file index e0c809a..f7c4596 --- a/type/__mail_alias/manifest +++ b/type/__mail_alias/explorer/aliases_file @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,14 +17,36 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer tries to find the correct aliases file. +found() { echo "$*"; exit 0; } -os=$(cat "$__global/explorer/os") +check_file() { + if test -f "$1" + then + found "$1" + fi +} -case "$os" in - *) - printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; +case $("$__explorer/os") +in + (freebsd|openbsd|solaris) + check_file /etc/mail/aliases + + # default + found /etc/mail/aliases + ;; + (debian|devuan|ubuntu) + check_file /etc/aliases + + # default + found /etc/aliases + ;; + (*) + check_file /etc/mail/aliases + check_file /etc/aliases + + # default + found /etc/mail/aliases + ;; esac diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 77ad9d2..a93dff2 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -18,3 +18,111 @@ # along with cdist. If not, see . # +state_should=$(cat "${__object}/parameter/state") + +case $state_should +in + (present) + if cmp "${__object}/explorer/aliases" "${__object}/parameter/alias" + then + # all good! + exit 0 + fi + + echo "set aliases" >>"$__messages_out" + mode=1 + ;; + (absent) + # nothing to do if no aliases found. + test -s "${__object}/explorer/aliases" || exit 0 + + echo "delete aliases" >>"$__messages_out" + mode=0 + ;; + (*) + printf 'Invalid --state given: %s\n' "$state_should" >&2 + exit 1 +esac + +aliases_file=$(cat "${__object}/explorer/aliases_file") + +if test -z "${aliases_file}" +then + echo 'Could not determine aliases file path.' >&2 + exit 1 +fi + +# "export" variables to remote +printf 'mode=%u\n' "${mode}" +printf "aliases_file='%s'\n" "${aliases_file}" + +cat <<'EOF' +awk -F ':[[:blank:]]*' -v mode="${mode}" ' +function sepafter(f, default, _) { + _ = substr($0, length($f) + 1, index(substr($0, length($f)+1), $(f+1)) - 1) + if (_) return _ + else return default +} + +function write_aliases() { + if (aliases_written) return + + printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") + while ((getline < aliases_should_file) > 0) { + if (aliases_written) printf ", " + printf "%s", $0 + aliases_written = 1 + } + printf "\n" + close(aliases_should_file) +} + +BEGIN { + aliases_should_file = (ENVIRON["__object"] "/parameter/alias") +} + +/^#/ { + # comment + select = 0; cont = 0 + print + next +} + +{ + cont = ($0 ~ /\\$/) + if (cont) sub(/[[:blank:]]*\\$/, "", $0) +} + +/^[[:blank:]]/ || cont { + # continuation line + if (select) next +} + +$1 == ENVIRON["__object_id"] { + in_list = 1 + if (mode) write_aliases() + next +} + +{ + in_list = 0 + print +} + +END { + # if the last line as an alias definition, the separator will be reused + if (mode && !aliases_written) write_aliases() +} +' <"${aliases_file}" >"${aliases_file}.tmp" || { + echo 'Generating new aliases file failed!' >&2 + exit 1 +} + +if ! cmp "${aliases_file}" "${aliases_file}.tmp" +then + mv "${aliases_file}.tmp" "${aliases_file}" + newaliases +else + rm "${aliases_file}.tmp" +fi +EOF diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst index d9563a6..d6c7873 100644 --- a/type/__mail_alias/man.rst +++ b/type/__mail_alias/man.rst @@ -3,12 +3,12 @@ cdist-type__mail_alias(7) NAME ---- -cdist-type__mail_alias - TODO +cdist-type__mail_alias - Manage mail aliases. DESCRIPTION ----------- -This space intentionally left blank. +This cdist type allows you to configure mail aliases (/etc/mail/aliases). REQUIRED PARAMETERS @@ -18,7 +18,14 @@ None. OPTIONAL PARAMETERS ------------------- -None. +state + 'present' or 'absent', defaults to 'present' +alias + the aliases where mail for the given user should be redirected to. + This parameter can be specified multiple times to redirect to more than one + recipient. + See the `aliases(5)` man page for the different forms this parameter can + take.. BOOLEAN PARAMETERS @@ -31,13 +38,15 @@ EXAMPLES .. code-block:: sh - # TODO - __mail_alias + # Redirect root mail to a "real" email address + __mail_alias root --alias admin@example.com + # Disable redirection of mail for joe + __mail_alias joe --state absent SEE ALSO -------- -:strong:`TODO`\ (7) +:strong:`aliases`\ (5) AUTHORS diff --git a/type/__mail_alias/parameter/default/state b/type/__mail_alias/parameter/default/state new file mode 100644 index 0000000..e7f6134 --- /dev/null +++ b/type/__mail_alias/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/type/__mail_alias/parameter/optional b/type/__mail_alias/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__mail_alias/parameter/optional @@ -0,0 +1 @@ +state diff --git a/type/__mail_alias/parameter/optional_multiple b/type/__mail_alias/parameter/optional_multiple new file mode 100644 index 0000000..d077ed8 --- /dev/null +++ b/type/__mail_alias/parameter/optional_multiple @@ -0,0 +1 @@ +alias From a5f3f3cdafe77f5aae075d1a0c053a9a36475463 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 30 May 2020 18:27:13 +0200 Subject: [PATCH 007/156] [type/__dma_auth] Implement type --- type/__dma_auth/{manifest => explorer/logins} | 29 ++++++--- type/__dma_auth/gencode-remote | 62 +++++++++++++++++++ type/__dma_auth/man.rst | 27 +++++--- type/__dma_auth/parameter/default/state | 1 + type/__dma_auth/parameter/optional | 2 + type/__dma_auth/parameter/required | 2 + 6 files changed, 106 insertions(+), 17 deletions(-) rename type/__dma_auth/{manifest => explorer/logins} (55%) mode change 100755 => 100644 create mode 100644 type/__dma_auth/parameter/default/state create mode 100644 type/__dma_auth/parameter/optional create mode 100644 type/__dma_auth/parameter/required diff --git a/type/__dma_auth/manifest b/type/__dma_auth/explorer/logins old mode 100755 new mode 100644 similarity index 55% rename from type/__dma_auth/manifest rename to type/__dma_auth/explorer/logins index e0c809a..0ed6bc1 --- a/type/__dma_auth/manifest +++ b/type/__dma_auth/explorer/logins @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,14 +17,25 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer lines matching user + server in /etc/dma/auth.conf and reports +# their cksum. +test -r /etc/dma/auth.conf || exit 0 -os=$(cat "$__global/explorer/os") +if test -f "${__object}/parameter/login" +then + login=$(cat "${__object}/parameter/login") +else + login=$__object_id +fi +server=$(cat "${__object}/parameter/server") -case "$os" in - *) - printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; -esac +regex=$(printf '^%s|%s:' "$login" "$server") + +grep -e "${regex}" /etc/dma/auth.conf \ +| while read -r line + do + echo "${line}" \ + | cksum - \ + | cut -d ' ' -f 1 + done diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 77ad9d2..0951c16 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,3 +18,65 @@ # along with cdist. If not, see . # +logins=$(cat "${__object}/explorer/logins") +state_should=$(cat "${__object}/parameter/state") + +if test -f "${__object}/parameter/login" +then + login=$(cat "${__object}/parameter/login") +else + login=$__object_id +fi +password=$(cat "${__object}/parameter/password") +server=$(cat "${__object}/parameter/server") + +case $state_should +in + (present) + conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "${password}") + cksum_should=$(echo "${conf_line}" | cksum - | cut -d ' ' -f 1) + if echo "$logins" | grep -qxF "${cksum_should}" + then + # correct line already present -> nothing to do + exit 0 + fi + + mode=1 + ;; + (absent) + if test -z "$logins" + then + # no logins present -> nothing to do + exit 0 + fi + + # NOTE: password is not needed to delete + conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "") + + mode=0 + ;; + (*) + printf 'Invalid --state: %s' "${state_should}" >&2 + exit 1 + ;; +esac + +cat </etc/dma/auth.conf.tmp \ +&& mv /etc/dma/auth.conf.tmp /etc/dma/auth.conf +EOF diff --git a/type/__dma_auth/man.rst b/type/__dma_auth/man.rst index cd4f647..bd077d6 100644 --- a/type/__dma_auth/man.rst +++ b/type/__dma_auth/man.rst @@ -3,23 +3,29 @@ cdist-type__dma_auth(7) NAME ---- -cdist-type__dma_auth - TODO +cdist-type__dma_auth - Configure SMTP logins for the DragonFly Mail Agent MTA. DESCRIPTION ----------- -This space intentionally left blank. +This cdist type allows you to set up credentials to log in to remote SMTP +servers. REQUIRED PARAMETERS ------------------- -None. +password + The user's password (in plain text.) +server + The SMTP server on which the login is valid. OPTIONAL PARAMETERS ------------------- -None. - +login + The user's LOGIN name on the SMTP server. Defaults to `__object_id`. +state + Either `present` or `absent`. Defaults to `present`. BOOLEAN PARAMETERS ------------------ @@ -31,13 +37,18 @@ EXAMPLES .. code-block:: sh - # TODO - __dma_auth + # Set the password for smarthost + __dma_auth joe --server smarthost --password hunter2 + # Set credentials for user at an external provider + __dma_auth paul@example.com --server mail.provider.com --password letmein + + # Delete credentials for example.com + __dma_auth paul --server example.com --state absent SEE ALSO -------- -:strong:`TODO`\ (7) +:strong:`cdist-type__dma`\ (7), :strong:`dma`\ (8) AUTHORS diff --git a/type/__dma_auth/parameter/default/state b/type/__dma_auth/parameter/default/state new file mode 100644 index 0000000..e7f6134 --- /dev/null +++ b/type/__dma_auth/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/type/__dma_auth/parameter/optional b/type/__dma_auth/parameter/optional new file mode 100644 index 0000000..c35dbef --- /dev/null +++ b/type/__dma_auth/parameter/optional @@ -0,0 +1,2 @@ +login +state diff --git a/type/__dma_auth/parameter/required b/type/__dma_auth/parameter/required new file mode 100644 index 0000000..8f1a1c9 --- /dev/null +++ b/type/__dma_auth/parameter/required @@ -0,0 +1,2 @@ +password +server From 988f277ad63c8c3b465b5e31571e470009c6ee9a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 30 May 2020 22:07:20 +0200 Subject: [PATCH 008/156] [type/__mail_alias] Fixes (mostly mawk compatibility) --- type/__mail_alias/explorer/aliases | 12 +++++------- type/__mail_alias/explorer/aliases_file | 0 type/__mail_alias/gencode-remote | 11 ++++++----- 3 files changed, 11 insertions(+), 12 deletions(-) mode change 100644 => 100755 type/__mail_alias/explorer/aliases_file diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index ce1a439..66940d5 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -22,13 +22,11 @@ aliases_file=$("${__type_explorer}/aliases_file") test -r "${aliases_file}" || exit 0 -name=$__object_id - -awk -F ':[[:blank:]]*' ' -function print_aliases (aliases, matches) { - split(aliases, matches, /,[[:blank:]]*/) +awk -F ':[ \t]*' ' +function print_aliases(aliases, matches) { + split(aliases, matches, /,[ \t]*/) for (i in matches) { - gsub(/^[[:blank:]]*|[[:blank:]]*$/, "", matches[i]) + gsub(/^[ \t]*|[ \t]*$/, "", matches[i]) print matches[i] } } @@ -40,7 +38,7 @@ function print_aliases (aliases, matches) { { cont = ($0 ~ /\\$/) - if (cont) sub(/[[:blank:]]*\\$/, "", $0) + if (cont) sub(/[ \t]*\\$/, "", $0) } /^[[:blank:]]/ || cont { diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file old mode 100644 new mode 100755 diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index a93dff2..7778536 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -23,7 +23,7 @@ state_should=$(cat "${__object}/parameter/state") case $state_should in (present) - if cmp "${__object}/explorer/aliases" "${__object}/parameter/alias" + if cmp -s "${__object}/explorer/aliases" "${__object}/parameter/alias" then # all good! exit 0 @@ -57,7 +57,8 @@ printf 'mode=%u\n' "${mode}" printf "aliases_file='%s'\n" "${aliases_file}" cat <<'EOF' -awk -F ':[[:blank:]]*' -v mode="${mode}" ' +test -f "${aliases_file}" || touch "${aliases_file}" +awk -F ':[ \t]*' -v mode="${mode}" ' function sepafter(f, default, _) { _ = substr($0, length($f) + 1, index(substr($0, length($f)+1), $(f+1)) - 1) if (_) return _ @@ -90,10 +91,10 @@ BEGIN { { cont = ($0 ~ /\\$/) - if (cont) sub(/[[:blank:]]*\\$/, "", $0) + if (cont) sub(/[ \t]*\\$/, "", $0) } -/^[[:blank:]]/ || cont { +/^[ \t]/ || cont { # continuation line if (select) next } @@ -118,7 +119,7 @@ END { exit 1 } -if ! cmp "${aliases_file}" "${aliases_file}.tmp" +if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" then mv "${aliases_file}.tmp" "${aliases_file}" newaliases From 59059a200a27cd750794439544e3aa5a45b65f09 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 31 May 2020 11:57:54 +0200 Subject: [PATCH 009/156] [type/__dma_auth] Use host as key --- .../__dma_auth/explorer/{logins => auth_conf} | 48 +++++++++-------- type/__dma_auth/explorer/authusers | 51 +++++++++++++++++++ type/__dma_auth/gencode-remote | 26 +++++----- type/__dma_auth/man.rst | 16 +++--- type/__dma_auth/parameter/optional | 2 +- type/__dma_auth/parameter/required | 2 +- 6 files changed, 102 insertions(+), 43 deletions(-) rename type/__dma_auth/explorer/{logins => auth_conf} (54%) mode change 100644 => 100755 create mode 100755 type/__dma_auth/explorer/authusers diff --git a/type/__dma_auth/explorer/logins b/type/__dma_auth/explorer/auth_conf old mode 100644 new mode 100755 similarity index 54% rename from type/__dma_auth/explorer/logins rename to type/__dma_auth/explorer/auth_conf index 0ed6bc1..cef0aca --- a/type/__dma_auth/explorer/logins +++ b/type/__dma_auth/explorer/auth_conf @@ -17,25 +17,33 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer lines matching user + server in /etc/dma/auth.conf and reports -# their cksum. +# This explorer determines the path of dma's auth.conf file -test -r /etc/dma/auth.conf || exit 0 +# No dma.conf -> use default +test -f /etc/dma/dma.conf || { + echo /etc/dma/auth.conf + exit 0 +} +test -r /etc/dma/dma.conf || { + echo 'Cannot read /etc/dma/dma.conf' >&2 + exit 1 +} -if test -f "${__object}/parameter/login" -then - login=$(cat "${__object}/parameter/login") -else - login=$__object_id -fi -server=$(cat "${__object}/parameter/server") - -regex=$(printf '^%s|%s:' "$login" "$server") - -grep -e "${regex}" /etc/dma/auth.conf \ -| while read -r line - do - echo "${line}" \ - | cksum - \ - | cut -d ' ' -f 1 - done +# Get AUTHPATH from dma.conf +awk -F'[ \t]' ' +{ + sub(/#.*$/, "", $0) # remove comments + if (!$0) next # ignore empty lines +} +$1 == "AUTHPATH" { + # Store authpath. In dma conf parsing last wins. + if ($2) authpath = substr($0, index($0, " ") + 1) +} +END { + if (authpath) { + print authpath + exit 0 + } else exit 1 +} +' /etc/dma/dma.conf \ +|| echo /etc/dma/auth.conf # default diff --git a/type/__dma_auth/explorer/authusers b/type/__dma_auth/explorer/authusers new file mode 100755 index 0000000..5fc6b4e --- /dev/null +++ b/type/__dma_auth/explorer/authusers @@ -0,0 +1,51 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# This explorer looks for lines matching the server parameter in dma's auth.conf +# and reports the login and server fields (password is stripped off) + +auth_conf=$("${__type_explorer}/auth_conf") +test -r "${auth_conf}" || exit 0 + +if test -f "${__object}/parameter/server" +then + server=$(cat "${__object}/parameter/server") +else + server=$__object_id +fi + +awk -F'\n' -v server="${server}" ' +BEGIN { + DP = "[: \t]" # copied from dma/conf.c +} + +# skip comments and empty lines +/^#/ || /^$/ { next } + +{ + login = substr($0, 1, index($0, "|") - 1) + host = substr($0, length(login) + 2) + if (match(host, DP)) { + host = substr(host, 1, RSTART - 1) + endpos = length(login) + RSTART + } else endpos = length +} + +host == server { print substr($0, 0, endpos) } +' "${auth_conf}" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 0951c16..989e176 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,24 +18,24 @@ # along with cdist. If not, see . # -logins=$(cat "${__object}/explorer/logins") +authusers=$(cat "${__object}/explorer/authusers") state_should=$(cat "${__object}/parameter/state") -if test -f "${__object}/parameter/login" +if test -f "${__object}/parameter/server" then - login=$(cat "${__object}/parameter/login") + server=$(cat "${__object}/parameter/server") else - login=$__object_id + server=$__object_id fi +login=$(cat "${__object}/parameter/login") password=$(cat "${__object}/parameter/password") -server=$(cat "${__object}/parameter/server") case $state_should in (present) conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "${password}") cksum_should=$(echo "${conf_line}" | cksum - | cut -d ' ' -f 1) - if echo "$logins" | grep -qxF "${cksum_should}" + if echo "$authusers" | grep -qxF "${cksum_should}" then # correct line already present -> nothing to do exit 0 @@ -44,11 +44,8 @@ in mode=1 ;; (absent) - if test -z "$logins" - then - # no logins present -> nothing to do - exit 0 - fi + # no logins present -> nothing to do + test -n "$authusers" || exit 0 # NOTE: password is not needed to delete conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "") @@ -66,11 +63,14 @@ read -r CONF_LINE <<'EOL' ${conf_line} EOL export CONF_LINE +export mode=${mode} +EOF -awk -F: -v print=$mode ' +cat <<'EOF' +awk -F: -v mode=$mode ' BEGIN { split(ENVIRON["CONF_LINE"], conf, ":") } $1 == conf[1] { - if (print && !found) { + if (mode && !found) { # remove duplicates print ENVIRON["CONF_LINE"] found = 1 diff --git a/type/__dma_auth/man.rst b/type/__dma_auth/man.rst index bd077d6..9c3ad7a 100644 --- a/type/__dma_auth/man.rst +++ b/type/__dma_auth/man.rst @@ -14,16 +14,16 @@ servers. REQUIRED PARAMETERS ------------------- +login + The user's LOGIN name on the SMTP server. password The user's password (in plain text.) -server - The SMTP server on which the login is valid. OPTIONAL PARAMETERS ------------------- -login - The user's LOGIN name on the SMTP server. Defaults to `__object_id`. +server + The SMTP server on which the login is valid. Defaults to `__object_id`. state Either `present` or `absent`. Defaults to `present`. @@ -38,13 +38,13 @@ EXAMPLES .. code-block:: sh # Set the password for smarthost - __dma_auth joe --server smarthost --password hunter2 + __dma_auth smarthost.example.com --login joe --password hunter2 # Set credentials for user at an external provider - __dma_auth paul@example.com --server mail.provider.com --password letmein + __dma_auth mail.provider.com --login paul@example.com --password letmein - # Delete credentials for example.com - __dma_auth paul --server example.com --state absent + # Delete credentials for example.com (for all users) + __dma_auth example.com --login '' --password '' --state absent SEE ALSO -------- diff --git a/type/__dma_auth/parameter/optional b/type/__dma_auth/parameter/optional index c35dbef..3e42ed3 100644 --- a/type/__dma_auth/parameter/optional +++ b/type/__dma_auth/parameter/optional @@ -1,2 +1,2 @@ -login +server state diff --git a/type/__dma_auth/parameter/required b/type/__dma_auth/parameter/required index 8f1a1c9..ae3c622 100644 --- a/type/__dma_auth/parameter/required +++ b/type/__dma_auth/parameter/required @@ -1,2 +1,2 @@ +login password -server From b848fca9299befe36b78624cf622d6df286db053 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 31 May 2020 15:01:40 +0200 Subject: [PATCH 010/156] [type/__dma_auth] Finish code to rewrite auth.conf --- type/__dma_auth/explorer/authusers | 12 +++- type/__dma_auth/gencode-remote | 90 +++++++++++++++++++++--------- 2 files changed, 73 insertions(+), 29 deletions(-) diff --git a/type/__dma_auth/explorer/authusers b/type/__dma_auth/explorer/authusers index 5fc6b4e..db83482 100755 --- a/type/__dma_auth/explorer/authusers +++ b/type/__dma_auth/explorer/authusers @@ -18,7 +18,7 @@ # along with cdist. If not, see . # # This explorer looks for lines matching the server parameter in dma's auth.conf -# and reports the login and server fields (password is stripped off) +# and reports the login and server fields (password is cksummed) auth_conf=$("${__type_explorer}/auth_conf") test -r "${auth_conf}" || exit 0 @@ -47,5 +47,11 @@ BEGIN { } else endpos = length } -host == server { print substr($0, 0, endpos) } -' "${auth_conf}" +host == server { print endpos, $0 } +' "${auth_conf}" \ +| while read pos line + do + printf '%s:%s\n' \ + "$(printf '%s' "$line" | cut -c $((-pos)))" \ + "$(printf '%s' "$line" | cut -c $((pos+2))- | cksum | cut -d' ' -f1)" + done diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 989e176..d75d611 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,7 +18,6 @@ # along with cdist. If not, see . # -authusers=$(cat "${__object}/explorer/authusers") state_should=$(cat "${__object}/parameter/state") if test -f "${__object}/parameter/server" @@ -28,27 +27,26 @@ else server=$__object_id fi login=$(cat "${__object}/parameter/login") -password=$(cat "${__object}/parameter/password") case $state_should in (present) - conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "${password}") - cksum_should=$(echo "${conf_line}" | cksum - | cut -d ' ' -f 1) - if echo "$authusers" | grep -qxF "${cksum_should}" + line_should=$(printf '%s|%s:%s\n' \ + "${login}" "${server}" \ + "$(cksum "${__object}/parameter/password" | cut -d' ' -f1)") + if grep -qxF "${line_should}" "${__object}/explorer/authusers" then # correct line already present -> nothing to do exit 0 fi + test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1 } + mode=1 ;; (absent) - # no logins present -> nothing to do - test -n "$authusers" || exit 0 - - # NOTE: password is not needed to delete - conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "") + # no matching logins present -> nothing to do + test -s "${__object}/explorer/authusers" || exit 0 mode=0 ;; @@ -58,25 +56,65 @@ in ;; esac +auth_conf=$(cat "${__object}/explorer/auth_conf") + +if test -z "${auth_conf}" +then + echo 'Cannot determine path of dma auth.conf' >&2 + exit 1 +fi + cat </etc/dma/auth.conf.tmp \ -&& mv /etc/dma/auth.conf.tmp /etc/dma/auth.conf + +BEGIN { + DP = "[: \t]" # copied from dma/conf.c +} + +# skip comments and empty lines +/^#/ || /^$/ { print; next } + +{ + login = substr($0, 1, index($0, "|") - 1) + host = substr($0, length(login) + 2) + if (match(host, DP)) { + host = substr(host, 1, RSTART - 1) + endpos = length(login) + RSTART + } else endpos = length +} + +host == ENVIRON["server"] { + if (mode) { + if (login == ENVIRON["login"] && !written) { + printf "%s%s\n", substr($0, 1, endpos+1), getpw() + written = 1 + next + } + } else if (!ENVIRON["login"] || login == ENVIRON["login"]) next +} + +{ print } + +END { + if (mode && !written) { + printf "%s|%s:%s\n", ENVIRON["login"], ENVIRON["server"], getpw() + } +} +' <"${auth_conf}" >"${auth_conf}.tmp" \ + && mv "${auth_conf}.tmp" "${auth_conf}" EOF From 3f72ca134108163f3208b709f84037312d460e5c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 31 May 2020 15:18:11 +0200 Subject: [PATCH 011/156] [type/__dma_auth] Send messages --- type/__dma/gencode-remote | 45 +++++++++++++++++++++------------- type/__dma_auth/gencode-remote | 11 ++++++++- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 2961c09..2e3a80d 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -1,20 +1,31 @@ #!/bin/sh -e -if [ -f "${__object}/parameter/send-test-email" ]; then - SEND_EMAIL="YES" +if test -f "${__object}/parameter/send-test-email" +then + modified=false + + if grep -q '^__mail_alias/root:' "${__messages_in}" + then + modified=true + elif grep -q '^__dma_auth/' "${__messages_in}" + then + modified=true + elif grep -q '^__dma/' "${__messages_in}" + then + modified=true + fi + + if $modified + then + cat <<-EOF + sendmail root <&2; exit 1 } mode=1 - ;; + + if test -s "${__object}/explorer/authusers" + then + printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + else + printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + fi + ;; (absent) # no matching logins present -> nothing to do test -s "${__object}/explorer/authusers" || exit 0 mode=0 + + printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" ;; (*) printf 'Invalid --state: %s' "${state_should}" >&2 From b87b67597efb8349709e8f5153bb73bb08f39337 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 31 May 2020 15:06:06 +0200 Subject: [PATCH 012/156] [type/__dma] Remove mail aliases functionality --- type/__dma/files/aliases | 68 -------------------------- type/__dma/man.rst | 22 ++------- type/__dma/manifest | 22 +-------- type/__dma/parameter/required_multiple | 1 - 4 files changed, 6 insertions(+), 107 deletions(-) delete mode 100644 type/__dma/files/aliases delete mode 100644 type/__dma/parameter/required_multiple diff --git a/type/__dma/files/aliases b/type/__dma/files/aliases deleted file mode 100644 index d341318..0000000 --- a/type/__dma/files/aliases +++ /dev/null @@ -1,68 +0,0 @@ -# Based off FreeBSD's /etc/aliases -# -# >>>>>>>>>> The program "newaliases" must be run after -# >> NOTE >> this file is updated for any changes to -# >>>>>>>>>> show through to sendmail. -# -# -# See also RFC 2142, `MAILBOX NAMES FOR COMMON SERVICES, ROLES -# AND FUNCTIONS', May 1997 -# http://tools.ietf.org/html/rfc2142 - -# Pretty much everything else in this file points to "root", so -# you would do well in either reading root's mailbox or forwarding -# root's email from here. - -# root: me@my.domain - - -# Basic system aliases -- these MUST be present -MAILER-DAEMON: postmaster -postmaster: root - -# General redirections for pseudo accounts -_dhcp: root -_pflogd: root -auditdistd: root -bin: root -bind: root -daemon: root -games: root -hast: root -kmem: root -mailnull: postmaster -man: root -news: root -nobody: root -operator: root -pop: root -proxy: root -smmsp: postmaster -sshd: root -system: root -toor: root -tty: root -usenet: news -uucp: root - -# Well-known aliases -- these should be filled in! -manager: root -dumper: root - -# BUSINESS-RELATED MAILBOX NAMES -info: root -marketing: root -sales: root -support: root - -# NETWORK OPERATIONS MAILBOX NAMES -abuse: root -noc: root -security: root - -# SUPPORT MAILBOX NAMES FOR SPECIFIC INTERNET SERVICES -ftp: root -ftp-bugs: ftp -hostmaster: root -webmaster: root -www: webmaster diff --git a/type/__dma/man.rst b/type/__dma/man.rst index a10c6c2..af9298e 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -19,18 +19,9 @@ email server configured in the `smart-host` parameter. REQUIRED PARAMETERS ------------------- smart-host - The destination email server. The addresses passed in `root-email` must be - either local to the `smart-host` or it must be configured to act as a relay - for the host being configured by this type. - - -REQUIRED MULTIPLE PARAMETERS ----------------------------- -root-email - Destination email address. Can be specified multiple times or just once - with each address separated by commas. - This will be setup in `/etc/aliases` as the destination for the local - root mailbox. + The email server used to send email. + It must be configured to act as a relay for the host being configured by + this type so that mail can be sent to users non-local to the smart-host. BOOLEAN PARAMETERS @@ -55,11 +46,7 @@ EXAMPLES .. code-block:: sh - # Send root email to both our BOFH and the nice-admin. - # That way they can figure things out together. __dma \ - --root-email bofh@domain.tld \ - --root-email nice-admin@domain.tld \ --smart-host mx1.domain.tld \ --send-test-email @@ -73,11 +60,12 @@ SEE ALSO AUTHORS ------- Evilham +Dennis Camera COPYING ------- -Copyright \(C) 2020 Evilham. You can redistribute it +Copyright \(C) 2020 Evilham and Dennis Camera. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/type/__dma/manifest b/type/__dma/manifest index abcaa5b..e07fbfc 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -1,8 +1,7 @@ #!/bin/sh -e -os="$(cat "${__global}/explorer/os")" +os=$(cat "${__global}/explorer/os") -root_email="$(tr '\n' ',' < "${__object}/parameter/root-email" | sed -E 's/,+$//')" smart_host="$(cat "${__object}/parameter/smart-host")" if [ -f "${__object}/parameter/mailname" ]; then @@ -21,14 +20,12 @@ else esac fi -aliases_file=/etc/mail/aliases case ${os} in debian|devuan|ubuntu) # Debian-like requires installing DMA __package dma # Moving forward without DMA doesn't make much sense export require="__package/dma" - aliases_file=/etc/aliases ;; freebsd) # Disable sendmail + stop if necessary @@ -134,20 +131,3 @@ EOF __file /etc/dma/dma.conf --mode 0644 --source '-' < Date: Mon, 1 Jun 2020 17:07:35 +0200 Subject: [PATCH 013/156] [type/__dma] Implement config file updates --- type/__dma/explorer/conf | 30 ++++ type/__dma/gencode-remote | 197 ++++++++++++++++++++++++- type/__dma/man.rst | 24 +++ type/__dma/manifest | 165 ++++++--------------- type/__dma/parameter/default/security | 1 + type/__dma/parameter/optional | 2 + type/__dma/parameter/optional_multiple | 1 + 7 files changed, 302 insertions(+), 118 deletions(-) create mode 100755 type/__dma/explorer/conf create mode 100644 type/__dma/parameter/default/security create mode 100644 type/__dma/parameter/optional_multiple diff --git a/type/__dma/explorer/conf b/type/__dma/explorer/conf new file mode 100755 index 0000000..129e3c3 --- /dev/null +++ b/type/__dma/explorer/conf @@ -0,0 +1,30 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# This explorer looks for lines matching the server parameter in dma's auth.conf +# and reports the login and server fields (password is cksummed) + +CONF_PATH=/etc/dma # set in Makefile +dma_conf="${CONF_PATH:?}/dma.conf" + +test -f "${dma_conf}" || exit 0 + +grep -v -e '^[ \t]*#\|^$' "${dma_conf}" \ +| sed -e 's/[ \t]*#.*$//' \ +| sort -s -k 1,1 diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 2e3a80d..1091aeb 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -1,5 +1,200 @@ #!/bin/sh -e +CONF_PATH=/etc/dma # set in Makefile + +# Generate config +conf_should=$( + if test -s "${__object}/parameter/smart-host" + then + printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smart-host")" + #printf 'AUTHPATH %s\n' "$(cat "${__type}/ + fi + + case $(cat "${__object}/parameter/security") + in + (ssl|tls) + default_smtp_port=465 + echo 'SECURETRANSFER' + ;; + (starttls) + default_smtp_port=587 + echo 'SECURETRANSFER' + echo 'STARTTLS' + ;; + (opportunistic) + default_smtp_port=25 # XXX: correct? + echo 'SECURETRANSFER' + echo 'STARTTLS' + echo 'OPPORTUNISTIC_TLS' + ;; + (insecure) + default_smtp_port=25 + echo 'INSECURE' + ;; + esac + + if test -s "${__object}/parameter/port" + then + printf 'PORT %u\n' "$(cat "${__object}/parameter/port")" + elif test "${default_smtp_port}" -ne 25 + then + printf 'PORT %u\n' "${default_smtp_port}" + fi + + if test -f "${__object}/parameter/masquerade" + then + while read -r line + do + printf 'MASQUERADE %s\n' "${line}" + done <"${__object}/parameter/masquerade" + fi + + +) +conf_should=$(echo "$conf_should" | sort -s -k 1,1) + +config_updated=false +if ! echo "$conf_should" | cmp -s "${__object}/explorer/conf" - +then + # config needs to be updated + echo "dma_conf='${CONF_PATH:?}/dma.conf'" + cat <<'EOF' +awk -F '\n' ' +function comment_line(line) { return match(line, /^[ \t]*#+[ \t]*/) } +function empty_line(line) { return match(line, /^[ \t]*$/) } +function is_word(s) { return s ~ /^[A-Z_]+$/ } + +function first(line, sep) { + if (!sep) sep = SUBSEP + return index(line, sep) ? substr(line, 0, index(line, sep)) : line +} + +function rest(line, sep) { + if (!sep) sep = SUBSEP + if (index(line, sep)) + return substr(line, index(line, sep) + 1) +} + +function conf_pop(word, value) { + if (!(word in conf)) return 0 + if (!value) { + if (index(conf[word], SUBSEP)) # more than one element? + value = substr(conf[word], 0, index(conf[word], SUBSEP)) + else + value = conf[word] + } + + if (index(conf[word], SUBSEP)) { + if (index(conf[word], value SUBSEP) != 1) return 0 + conf[word] = substr(conf[word], length(value) + 2) + } else { + if (conf[word] != value) return 0 + delete conf[word] + } + return value +} + +function print_conf(word, value) { + printf "%s", word + if (value) printf " %s", value + printf "\n" +} + +function print_confs(word, value) { + if (!(word in conf)) return + if (conf[word]) { + while (value = conf_pop(word)) + print_conf(word, value) + } else { + print_conf(word) + delete conf[word] + } +} + +BEGIN { + while (getline < "/dev/stdin") { + word = first($0, " ") + if ((word in conf)) + conf[word] = conf[word] SUBSEP rest($0, " ") + else + conf[word] = rest($0, " ") + } +} + +# first pass, gather information +NR == FNR { + if (comment_line($0)) { + # comment line + word = first(substr($0, RLENGTH + 1), " ") + if (is_word(word)) last_occ["#" word] = FNR + } else { + word = first($0, " ") + if (is_word(word)) last_occ[word] = FNR + } +} + +NR > FNR && FNR == 1 { + # before second pass prepare hashes + + for (k in last_occ) + if (k ~ /^\#/ && (substr(k, 2) in last_occ)) + delete last_occ[k] + + for (k in last_occ) { + line_map[last_occ[k]] = k + } +} + +# second pass, output new config +NR > FNR { + if (comment_line($0) || empty_line($0)) { + # comment or empty line + print + + if ((FNR in line_map)) { + if (line_map[FNR] ~ /^\#/) { + # the "matching" comment line is here + k = substr(line_map[FNR], 2) + if ((k in conf)) print_confs(k) + } + + if (("INSECURE" in conf) && line_map[FNR] ~ /^\#?SECURE$/) { + # INSECURE goes where SECURE comment is + print_confs("INSECURE") + } + } + } else { + sub(/[ \t]*\#.*$/, "", $0) # ignore comments + word = first($0, " ") + + if ((word in conf) && rest($0, " ") == first(conf[word])) { + # keep config options we want + conf_pop(word) + print + } + + if ((FNR in line_map) && line_map[FNR] == word) { + # rest of config options should be here + print_confs(word) + } + } +} + +END { + # print rest of config options + for (word in conf) print_confs(word) +} +' "${dma_conf}" "${dma_conf}" <<'EOF' >"${dma_conf}.tmp" \ + && mv "${dma_conf}.tmp" "${dma_conf}" +EOF + echo "${conf_should}" + echo 'EOF' + + config_updated=true + echo 'config updated' >>"${__messages_out}" +fi + + if test -f "${__object}/parameter/send-test-email" then modified=false @@ -10,7 +205,7 @@ then elif grep -q '^__dma_auth/' "${__messages_in}" then modified=true - elif grep -q '^__dma/' "${__messages_in}" + elif $config_updated then modified=true fi diff --git a/type/__dma/man.rst b/type/__dma/man.rst index af9298e..cb3c35f 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -39,7 +39,31 @@ mailname If not defined, it defaults to `/etc/mailname` on Debian-derived Operating Systems and to `__target_host` otherwise. See `dma(8)` for more information. +masquerade + Masquerade the envelope-from addresses with this address/hostname. + Use this setting if mails are not accepted by destination mail servers + because your sender domain is invalid. + This option can be used multiple times. + For more information see the `dma(8)` man page. +port + The port on which to deliver email. + If not provided, a sensible default port will be used based on the + `--security` argument. +security + Configures whether and how DMA should use secure connections. + ssl/tls + Enable TLS/SSL secured transfer. + starttls + Use STARTTLS to establish a secure connection. + opportunistic (default) + Will try to establish a secure connection using STARTTLS, but allow + unencrypted transfer if STARTTLS fails. + Most useful when dma is used without a smarthost, delivering remote + messages directly to the outside mail exchangers. + insecure + allow plain text SMTP login over an insecure connection. + Should really not be used anymore! EXAMPLES -------- diff --git a/type/__dma/manifest b/type/__dma/manifest index e07fbfc..814e3ef 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -2,54 +2,57 @@ os=$(cat "${__global}/explorer/os") -smart_host="$(cat "${__object}/parameter/smart-host")" - -if [ -f "${__object}/parameter/mailname" ]; then - mailname="$(cat "${__object}/parameter/mailname")" +# mailname: default behaviour is different on certain systems +if test -f "${__object}/parameter/mailname" +then + mailname=$(cat "${__object}/parameter/mailname") else - # default mailname behaviour is different in certain systems - case ${os} in - debian|devuan|ubuntu) - # Debian-like default to /etc/mailname - mailname="/etc/mailname" - ;; - *) - # Otherwise let's use the hostname - mailname="${__target_host}" - ;; - esac + # Otherwise use the hostname + mailname=$(cat "${__global}/explorer/hostname") fi -case ${os} in - debian|devuan|ubuntu) - # Debian-like requires installing DMA - __package dma - # Moving forward without DMA doesn't make much sense - export require="__package/dma" - ;; - freebsd) - # Disable sendmail + stop if necessary - __key_value \ - --file "/etc/rc.conf" \ - --comment "# Disable sendmail " \ - --key "sendmail_enable" \ - --delimiter "=" \ - --value "NONE" \ - --onchange "service sendmail onestop || true" \ - "sendmail_enable" +case $os +in + (debian|devuan|ubuntu) + # On Debian-like systems use /etc/mailname + if test -f "${__object}/parameter/mailname" + then + echo "$mailname" | __file '/etc/mailname' --state present \ + --mode 0644 --owner root --group root --source - + fi + + mailname='/etc/mailname' + ;; +esac + +# Install DMA +case $os +in + (debian|devuan|ubuntu) + __package dma --state present + export require='__package/dma' + ;; + (freebsd) + # Stop sendmail if necessary + __process 'sendmail' --name 'sendmail.*' --state absent \ + --stop '/etc/rc.d/sendmail onestop' + + # ... and disable it + __key_value 'rcconf-sendmail-enable' --file '/etc/rc.conf' \ + --key 'sendmail_enable' --delimiter '=' --value '"NONE"' \ + --exact_delimiter + # Setup mailwrapper accordingly - __file /etc/mail/mailer.conf \ - --mode 0644 \ - --source '-' < /dev/stderr <&2 Your OS (${os}) is not supported yet. Maybe adding support is as simple as adapting the packages or allowing it, @@ -57,77 +60,5 @@ we highly encourage you to open a PR with the necessary changes. See: https://code.ungleich.ch/ungleich-public/cdist-contrib/ EOF exit 1 - ;; + ;; esac - -DMA_CONF="$(cat < Date: Mon, 1 Jun 2020 17:16:04 +0200 Subject: [PATCH 014/156] [type/__dma] Detect AUTHPATH --- type/__dma/explorer/auth_conf | 1 + type/__dma/gencode-remote | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 120000 type/__dma/explorer/auth_conf diff --git a/type/__dma/explorer/auth_conf b/type/__dma/explorer/auth_conf new file mode 120000 index 0000000..db038ae --- /dev/null +++ b/type/__dma/explorer/auth_conf @@ -0,0 +1 @@ +../../__dma_auth/explorer/auth_conf \ No newline at end of file diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 1091aeb..105edba 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -49,7 +49,10 @@ conf_should=$( done <"${__object}/parameter/masquerade" fi - + if test -s "${__object}/explorer/auth_conf" + then + printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" + fi ) conf_should=$(echo "$conf_should" | sort -s -k 1,1) From 4fdddfd738dac87a1e33e359e1ede6ee22a16918 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 17:23:31 +0200 Subject: [PATCH 015/156] [type/__dma] Add --defer, --full-bounce, and --null-client --- type/__dma/gencode-remote | 20 ++++++++++++++++++++ type/__dma/man.rst | 8 ++++++++ type/__dma/parameter/boolean | 3 +++ 3 files changed, 31 insertions(+) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 105edba..1f252a4 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -49,6 +49,26 @@ conf_should=$( done <"${__object}/parameter/masquerade" fi + if test -f "${__object}/parameter/defer" + then + echo 'DEFER' + fi + + if test -f "${__object}/parameter/full-bounce" + then + echo 'FULLBOUNCE' + fi + + if test -f "${__object}/parameter/null-client" + then + test -s "${__object}/parameter/smart-host" || { + echo '--null-client requires a --smart-host to be defined' >&2 + exit 1 + } + + echo 'NULLCLIENT' + fi + if test -s "${__object}/explorer/auth_conf" then printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" diff --git a/type/__dma/man.rst b/type/__dma/man.rst index cb3c35f..a8ba546 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -26,6 +26,14 @@ smart-host BOOLEAN PARAMETERS ------------------ +defer + If enabled, the mail queue has to be manually flushed with the `-q` option. +full-bounce + Enable if the bounce message should include the complete original message, + not just the headers. +null-client + Enable to bypass aliases and local delivery, and instead forward all mails + to the defined `--smart-host`. send-test-email If present, after setup this type will send an email to root, to allow you to easily test your setup. diff --git a/type/__dma/parameter/boolean b/type/__dma/parameter/boolean index d1af563..ede7dda 100644 --- a/type/__dma/parameter/boolean +++ b/type/__dma/parameter/boolean @@ -1 +1,4 @@ +defer +full-bounce +null-client send-test-email From 99d58672c4987757429fc783b66d96eb6fa90f2b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 19:20:05 +0200 Subject: [PATCH 016/156] [type/__dma_auth] Add semicolon --- type/__dma_auth/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 48e7dce..b6b7f63 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -40,7 +40,7 @@ in exit 0 fi - test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1 } + test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1; } mode=1 From 0657ac4f115ee575f0bb7780fa9ac759c472ec97 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 19:21:41 +0200 Subject: [PATCH 017/156] [type/__dma] Fix mailname --- type/__dma/gencode-remote | 31 +++++++++++++++++++++++++------ type/__dma/manifest | 27 +-------------------------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 1f252a4..4100d39 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -2,12 +2,36 @@ CONF_PATH=/etc/dma # set in Makefile +# Determine mailname +if test -f "${__object}/parameter/mailname" +then + mailname=$(cat "${__object}/parameter/mailname") +else + case $(cat "${__global}/explorer/os") + in + (debian|devuan|ubuntu) + # On Debian-like systems use /etc/mailname unless --mailname is used + mailname='/etc/mailname' + ;; + (*) + mailname=$__target_fqdn + ;; + esac +fi + + # Generate config conf_should=$( if test -s "${__object}/parameter/smart-host" then printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smart-host")" - #printf 'AUTHPATH %s\n' "$(cat "${__type}/ + fi + + printf 'MAILNAME %s\n' "${mailname}" + + if test -s "${__object}/explorer/auth_conf" + then + printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" fi case $(cat "${__object}/parameter/security") @@ -68,11 +92,6 @@ conf_should=$( echo 'NULLCLIENT' fi - - if test -s "${__object}/explorer/auth_conf" - then - printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" - fi ) conf_should=$(echo "$conf_should" | sort -s -k 1,1) diff --git a/type/__dma/manifest b/type/__dma/manifest index 814e3ef..60038b1 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -1,32 +1,7 @@ #!/bin/sh -e -os=$(cat "${__global}/explorer/os") - -# mailname: default behaviour is different on certain systems -if test -f "${__object}/parameter/mailname" -then - mailname=$(cat "${__object}/parameter/mailname") -else - # Otherwise use the hostname - mailname=$(cat "${__global}/explorer/hostname") -fi - -case $os -in - (debian|devuan|ubuntu) - # On Debian-like systems use /etc/mailname - if test -f "${__object}/parameter/mailname" - then - echo "$mailname" | __file '/etc/mailname' --state present \ - --mode 0644 --owner root --group root --source - - fi - - mailname='/etc/mailname' - ;; -esac - # Install DMA -case $os +case $(cat "${__global}/explorer/os") in (debian|devuan|ubuntu) __package dma --state present From 7183bb3cd191dfece36e32db86eb6ee42b49bb5d Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 19:24:32 +0200 Subject: [PATCH 018/156] [type/__dma] Fixes for FreeBSD --- type/__dma/gencode-remote | 13 ++++++++----- type/__dma/manifest | 3 +++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 4100d39..1b00f04 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -108,7 +108,7 @@ function is_word(s) { return s ~ /^[A-Z_]+$/ } function first(line, sep) { if (!sep) sep = SUBSEP - return index(line, sep) ? substr(line, 0, index(line, sep)) : line + return index(line, sep) ? substr(line, 1, index(line, sep) - 1) : line } function rest(line, sep) { @@ -121,7 +121,7 @@ function conf_pop(word, value) { if (!(word in conf)) return 0 if (!value) { if (index(conf[word], SUBSEP)) # more than one element? - value = substr(conf[word], 0, index(conf[word], SUBSEP)) + value = substr(conf[word], 1, index(conf[word], SUBSEP) - 1) else value = conf[word] } @@ -256,10 +256,13 @@ then then cat <<-EOF sendmail root < Date: Mon, 1 Jun 2020 20:25:10 +0200 Subject: [PATCH 019/156] [type/__dma_auth] Fix SC2162 --- type/__dma_auth/explorer/authusers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__dma_auth/explorer/authusers b/type/__dma_auth/explorer/authusers index db83482..c76667b 100755 --- a/type/__dma_auth/explorer/authusers +++ b/type/__dma_auth/explorer/authusers @@ -49,7 +49,7 @@ BEGIN { host == server { print endpos, $0 } ' "${auth_conf}" \ -| while read pos line +| while read -r pos line do printf '%s:%s\n' \ "$(printf '%s' "$line" | cut -c $((-pos)))" \ From bf822f3f8ca53852554750003296b9163122c424 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 20:26:52 +0200 Subject: [PATCH 020/156] [type/__dma] Fix SC2154 --- type/__dma/manifest | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/type/__dma/manifest b/type/__dma/manifest index 4a78dab..7abd7c8 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -1,7 +1,9 @@ #!/bin/sh -e +os=$(cat "${__global}/explorer/os") + # Install DMA -case $(cat "${__global}/explorer/os") +case $os in (debian|devuan|ubuntu) __package dma --state present From 3adb5ac4cabe65b24c0a5139a62b38b35b9191bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 14:58:43 +0200 Subject: [PATCH 021/156] Import __ungleich_unbound type for ungleich's dot-cdist --- type/__ungleich_unbound/files/unbound.conf.sh | 1018 +++++++++++++++++ type/__ungleich_unbound/man.rst | 74 ++ type/__ungleich_unbound/manifest | 79 ++ type/__ungleich_unbound/parameter/boolean | 2 + .../parameter/optional_multiple | 2 + type/__ungleich_unbound/parameter/required | 1 + .../parameter/required_multiple | 1 + type/__ungleich_unbound/singleton | 0 8 files changed, 1177 insertions(+) create mode 100755 type/__ungleich_unbound/files/unbound.conf.sh create mode 100644 type/__ungleich_unbound/man.rst create mode 100755 type/__ungleich_unbound/manifest create mode 100644 type/__ungleich_unbound/parameter/boolean create mode 100644 type/__ungleich_unbound/parameter/optional_multiple create mode 100644 type/__ungleich_unbound/parameter/required create mode 100644 type/__ungleich_unbound/parameter/required_multiple create mode 100644 type/__ungleich_unbound/singleton diff --git a/type/__ungleich_unbound/files/unbound.conf.sh b/type/__ungleich_unbound/files/unbound.conf.sh new file mode 100755 index 0000000..e713843 --- /dev/null +++ b/type/__ungleich_unbound/files/unbound.conf.sh @@ -0,0 +1,1018 @@ +#!/bin/sh + +generate_interface() { + for i in $INTERFACES; do + echo " interface: $i" + done +} + +generate_access_control() { + echo "$ACCESS_CONTROLS" | while read entry; do + echo " access-control: $entry" + done +} + +generate_forward_addr() { + for addr in $FORWARD_ADDRS; do + echo " forward-addr: $addr" + done +} + +cat << EOF +# +# THIS FILE HAS BEEN GENERATED BY CDIST, DO NOT EDIT BY HAND. +# +# See unbound.conf(5) man page, version 1.9.1. +# +# this is a comment. + +#Use this to include other text into the file. +#include: "otherfile.conf" + +# The server clause sets the main parameters. +server: + # whitespace is not necessary, but looks cleaner. + + # verbosity number, 0 is least verbose. 1 is default. + verbosity: 1 + + # print statistics to the log (for every thread) every N seconds. + # Set to "" or 0 to disable. Default is disabled. + # statistics-interval: 0 + + # enable shm for stats, default no. if you enable also enable + # statistics-interval, every time it also writes stats to the + # shared memory segment keyed with shm-key. + # shm-enable: no + + # shm for stats uses this key, and key+1 for the shared mem segment. + # shm-key: 11777 + + # enable cumulative statistics, without clearing them after printing. + # statistics-cumulative: no + + # enable extended statistics (query types, answer codes, status) + # printed from unbound-control. default off, because of speed. + # extended-statistics: no + + # number of threads to create. 1 disables threading. + # num-threads: 1 + + # specify the interfaces to answer queries from by ip-address. + # The default is to listen to localhost (127.0.0.1 and ::1). + # specify 0.0.0.0 and ::0 to bind to all available interfaces. + # specify every interface[@port] on a new 'interface:' labelled line. + # The listen interfaces are not changed on reload, only on restart. + # interface: 192.0.2.153 + # interface: 192.0.2.154 + # interface: 192.0.2.154@5003 + # interface: 2001:DB8::5 +$(generate_interface) + + # enable this feature to copy the source address of queries to reply. + # Socket options are not supported on all platforms. experimental. + # interface-automatic: no + + # port to answer queries from + # port: 53 + + # specify the interfaces to send outgoing queries to authoritative + # server from by ip-address. If none, the default (all) interface + # is used. Specify every interface on a 'outgoing-interface:' line. + # outgoing-interface: 192.0.2.153 + # outgoing-interface: 2001:DB8::5 + # outgoing-interface: 2001:DB8::6 + + # Specify a netblock to use remainder 64 bits as random bits for + # upstream queries. Uses freebind option (Linux). + # outgoing-interface: 2001:DB8::/64 + # Also (Linux:) ip -6 addr add 2001:db8::/64 dev lo + # And: ip -6 route add local 2001:db8::/64 dev lo + # And set prefer-ip6: yes to use the ip6 randomness from a netblock. + # Set this to yes to prefer ipv6 upstream servers over ipv4. + # prefer-ip6: no + + # number of ports to allocate per thread, determines the size of the + # port range that can be open simultaneously. About double the + # num-queries-per-thread, or, use as many as the OS will allow you. + # outgoing-range: 4096 + + # permit unbound to use this port number or port range for + # making outgoing queries, using an outgoing interface. + # outgoing-port-permit: 32768 + + # deny unbound the use this of port number or port range for + # making outgoing queries, using an outgoing interface. + # Use this to make sure unbound does not grab a UDP port that some + # other server on this computer needs. The default is to avoid + # IANA-assigned port numbers. + # If multiple outgoing-port-permit and outgoing-port-avoid options + # are present, they are processed in order. + # outgoing-port-avoid: "3200-3208" + + # number of outgoing simultaneous tcp buffers to hold per thread. + # outgoing-num-tcp: 10 + + # number of incoming simultaneous tcp buffers to hold per thread. + # incoming-num-tcp: 10 + + # buffer size for UDP port 53 incoming (SO_RCVBUF socket option). + # 0 is system default. Use 4m to catch query spikes for busy servers. + # so-rcvbuf: 0 + + # buffer size for UDP port 53 outgoing (SO_SNDBUF socket option). + # 0 is system default. Use 4m to handle spikes on very busy servers. + # so-sndbuf: 0 + + # use SO_REUSEPORT to distribute queries over threads. + # so-reuseport: yes + + # use IP_TRANSPARENT so the interface: addresses can be non-local + # and you can config non-existing IPs that are going to work later on + # (uses IP_BINDANY on FreeBSD). + # ip-transparent: no + + # use IP_FREEBIND so the interface: addresses can be non-local + # and you can bind to nonexisting IPs and interfaces that are down. + # Linux only. On Linux you also have ip-transparent that is similar. + # ip-freebind: no + + # EDNS reassembly buffer to advertise to UDP peers (the actual buffer + # is set with msg-buffer-size). 1472 can solve fragmentation (timeouts) + # edns-buffer-size: 4096 + + # Maximum UDP response size (not applied to TCP response). + # Suggested values are 512 to 4096. Default is 4096. 65536 disables it. + # max-udp-size: 4096 + + # max memory to use for stream(tcp and tls) waiting result buffers. + # stream-wait-size: 4m + + # buffer size for handling DNS data. No messages larger than this + # size can be sent or received, by UDP or TCP. In bytes. + # msg-buffer-size: 65552 + + # the amount of memory to use for the message cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # msg-cache-size: 4m + + # the number of slabs to use for the message cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # msg-cache-slabs: 4 + + # the number of queries that a thread gets to service. + # num-queries-per-thread: 1024 + + # if very busy, 50% queries run to completion, 50% get timeout in msec + # jostle-timeout: 200 + + # msec to wait before close of port on timeout UDP. 0 disables. + # delay-close: 0 + + # msec for waiting for an unknown server to reply. Increase if you + # are behind a slow satellite link, to eg. 1128. + # unknown-server-time-limit: 376 + + # the amount of memory to use for the RRset cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # rrset-cache-size: 4m + + # the number of slabs to use for the RRset cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # rrset-cache-slabs: 4 + + # the time to live (TTL) value lower bound, in seconds. Default 0. + # If more than an hour could easily give trouble due to stale data. + # cache-min-ttl: 0 + + # the time to live (TTL) value cap for RRsets and messages in the + # cache. Items are not cached for longer. In seconds. + # cache-max-ttl: 86400 + + # the time to live (TTL) value cap for negative responses in the cache + # cache-max-negative-ttl: 3600 + + # the time to live (TTL) value for cached roundtrip times, lameness and + # EDNS version information for hosts. In seconds. + # infra-host-ttl: 900 + + # minimum wait time for responses, increase if uplink is long. In msec. + # infra-cache-min-rtt: 50 + + # the number of slabs to use for the Infrastructure cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # infra-cache-slabs: 4 + + # the maximum number of hosts that are cached (roundtrip, EDNS, lame). + # infra-cache-numhosts: 10000 + + # define a number of tags here, use with local-zone, access-control. + # repeat the define-tag statement to add additional tags. + # define-tag: "tag1 tag2 tag3" + + # Enable IPv4, "yes" or "no". + do-ip4: $DO_IP4 + + # Enable IPv6, "yes" or "no". + do-ip6: $DO_IP6 + + # Enable UDP, "yes" or "no". + # do-udp: yes + + # Enable TCP, "yes" or "no". + # do-tcp: yes + + # upstream connections use TCP only (and no UDP), "yes" or "no" + # useful for tunneling scenarios, default no. + # tcp-upstream: no + + # upstream connections also use UDP (even if do-udp is no). + # useful if if you want UDP upstream, but don't provide UDP downstream. + # udp-upstream-without-downstream: no + + # Maximum segment size (MSS) of TCP socket on which the server + # responds to queries. Default is 0, system default MSS. + # tcp-mss: 0 + + # Maximum segment size (MSS) of TCP socket for outgoing queries. + # Default is 0, system default MSS. + # outgoing-tcp-mss: 0 + + # Idle TCP timeout, connection closed in milliseconds + # tcp-idle-timeout: 30000 + + # Enable EDNS TCP keepalive option. + # edns-tcp-keepalive: no + + # Timeout for EDNS TCP keepalive, in msec. + # edns-tcp-keepalive-timeout: 120000 + + # Use systemd socket activation for UDP, TCP, and control sockets. + # use-systemd: no + + # Detach from the terminal, run in background, "yes" or "no". + # Set the value to "no" when unbound runs as systemd service. + # do-daemonize: yes + + # control which clients are allowed to make (recursive) queries + # to this server. Specify classless netblocks with /size and action. + # By default everything is refused, except for localhost. + # Choose deny (drop message), refuse (polite error reply), + # allow (recursive ok), allow_setrd (recursive ok, rd bit is forced on), + # allow_snoop (recursive and nonrecursive ok) + # deny_non_local (drop queries unless can be answered from local-data) + # refuse_non_local (like deny_non_local but polite error reply). + # access-control: 0.0.0.0/0 refuse + # access-control: 127.0.0.0/8 allow + # access-control: ::0/0 refuse + # access-control: ::1 allow + # access-control: ::ffff:127.0.0.1 allow +$(generate_access_control) + + # tag access-control with list of tags (in "" with spaces between) + # Clients using this access control element use localzones that + # are tagged with one of these tags. + # access-control-tag: 192.0.2.0/24 "tag2 tag3" + + # set action for particular tag for given access control element + # if you have multiple tag values, the tag used to lookup the action + # is the first tag match between access-control-tag and local-zone-tag + # where "first" comes from the order of the define-tag values. + # access-control-tag-action: 192.0.2.0/24 tag3 refuse + + # set redirect data for particular tag for access control element + # access-control-tag-data: 192.0.2.0/24 tag2 "A 127.0.0.1" + + # Set view for access control element + # access-control-view: 192.0.2.0/24 viewname + + # if given, a chroot(2) is done to the given directory. + # i.e. you can chroot to the working directory, for example, + # for extra security, but make sure all files are in that directory. + # + # If chroot is enabled, you should pass the configfile (from the + # commandline) as a full path from the original root. After the + # chroot has been performed the now defunct portion of the config + # file path is removed to be able to reread the config after a reload. + # + # All other file paths (working dir, logfile, roothints, and + # key files) can be specified in several ways: + # o as an absolute path relative to the new root. + # o as a relative path to the working directory. + # o as an absolute path relative to the original root. + # In the last case the path is adjusted to remove the unused portion. + # + # The pid file can be absolute and outside of the chroot, it is + # written just prior to performing the chroot and dropping permissions. + # + # Additionally, unbound may need to access /dev/random (for entropy). + # How to do this is specific to your OS. + # + # If you give "" no chroot is performed. The path must not end in a /. + # chroot: "" + + # if given, user privileges are dropped (after binding port), + # and the given username is assumed. Default is user "unbound". + # If you give "" no privileges are dropped. + # username: "unbound" + + # the working directory. The relative files in this config are + # relative to this directory. If you give "" the working directory + # is not changed. + # If you give a server: directory: dir before include: file statements + # then those includes can be relative to the working directory. + # directory: "" + + # the log file, "" means log to stderr. + # Use of this option sets use-syslog to "no". + # logfile: "" + + # Log to syslog(3) if yes. The log facility LOG_DAEMON is used to + # log to. If yes, it overrides the logfile. + # use-syslog: yes + + # Log identity to report. if empty, defaults to the name of argv[0] + # (usually "unbound"). + # log-identity: "" + + # print UTC timestamp in ascii to logfile, default is epoch in seconds. + # log-time-ascii: no + + # print one line with time, IP, name, type, class for every query. + # log-queries: no + + # print one line per reply, with time, IP, name, type, class, rcode, + # timetoresolve, fromcache and responsesize. + # log-replies: no + + # log with tag 'query' and 'reply' instead of 'info' for + # filtering log-queries and log-replies from the log. + # log-tag-queryreply: no + + # log the local-zone actions, like local-zone type inform is enabled + # also for the other local zone types. + # log-local-actions: no + + # print log lines that say why queries return SERVFAIL to clients. + # log-servfail: no + + # file to read root hints from. + # get one from https://www.internic.net/domain/named.cache + root-hints: /usr/share/dns-root-hints/named.root + + # enable to not answer id.server and hostname.bind queries. + # hide-identity: no + + # enable to not answer version.server and version.bind queries. + # hide-version: no + + # enable to not answer trustanchor.unbound queries. + # hide-trustanchor: no + + # the identity to report. Leave "" or default to return hostname. + # identity: "" + + # the version to report. Leave "" or default to return package version. + # version: "" + + # the target fetch policy. + # series of integers describing the policy per dependency depth. + # The number of values in the list determines the maximum dependency + # depth the recursor will pursue before giving up. Each integer means: + # -1 : fetch all targets opportunistically, + # 0: fetch on demand, + # positive value: fetch that many targets opportunistically. + # Enclose the list of numbers between quotes (""). + # target-fetch-policy: "3 2 1 0 0" + + # Harden against very small EDNS buffer sizes. + # harden-short-bufsize: no + + # Harden against unseemly large queries. + # harden-large-queries: no + + # Harden against out of zone rrsets, to avoid spoofing attempts. + # harden-glue: yes + + # Harden against receiving dnssec-stripped data. If you turn it + # off, failing to validate dnskey data for a trustanchor will + # trigger insecure mode for that zone (like without a trustanchor). + # Default on, which insists on dnssec data for trust-anchored zones. + # harden-dnssec-stripped: yes + + # Harden against queries that fall under dnssec-signed nxdomain names. + # harden-below-nxdomain: yes + + # Harden the referral path by performing additional queries for + # infrastructure data. Validates the replies (if possible). + # Default off, because the lookups burden the server. Experimental + # implementation of draft-wijngaards-dnsext-resolver-side-mitigation. + # harden-referral-path: no + + # Harden against algorithm downgrade when multiple algorithms are + # advertised in the DS record. If no, allows the weakest algorithm + # to validate the zone. + # harden-algo-downgrade: no + + # Sent minimum amount of information to upstream servers to enhance + # privacy. Only sent minimum required labels of the QNAME and set QTYPE + # to A when possible. + # qname-minimisation: yes + + # QNAME minimisation in strict mode. Do not fall-back to sending full + # QNAME to potentially broken nameservers. A lot of domains will not be + # resolvable when this option in enabled. + # This option only has effect when qname-minimisation is enabled. + # qname-minimisation-strict: no + + # Aggressive NSEC uses the DNSSEC NSEC chain to synthesize NXDOMAIN + # and other denials, using information from previous NXDOMAINs answers. + # aggressive-nsec: no + + # Use 0x20-encoded random bits in the query to foil spoof attempts. + # This feature is an experimental implementation of draft dns-0x20. + # use-caps-for-id: no + + # Domains (and domains in them) without support for dns-0x20 and + # the fallback fails because they keep sending different answers. + # caps-whitelist: "licdn.com" + # caps-whitelist: "senderbase.org" + + # Enforce privacy of these addresses. Strips them away from answers. + # It may cause DNSSEC validation to additionally mark it as bogus. + # Protects against 'DNS Rebinding' (uses browser as network proxy). + # Only 'private-domain' and 'local-data' names are allowed to have + # these private addresses. No default. + # private-address: 10.0.0.0/8 + # private-address: 172.16.0.0/12 + # private-address: 192.168.0.0/16 + # private-address: 169.254.0.0/16 + # private-address: fd00::/8 + # private-address: fe80::/10 + # private-address: ::ffff:0:0/96 + + # Allow the domain (and its subdomains) to contain private addresses. + # local-data statements are allowed to contain private addresses too. + # private-domain: "example.com" + + # If nonzero, unwanted replies are not only reported in statistics, + # but also a running total is kept per thread. If it reaches the + # threshold, a warning is printed and a defensive action is taken, + # the cache is cleared to flush potential poison out of it. + # A suggested value is 10000000, the default is 0 (turned off). + # unwanted-reply-threshold: 0 + + # Do not query the following addresses. No DNS queries are sent there. + # List one address per entry. List classless netblocks with /size, + # do-not-query-address: 127.0.0.1/8 + # do-not-query-address: ::1 + + # if yes, the above default do-not-query-address entries are present. + # if no, localhost can be queried (for testing and debugging). + # do-not-query-localhost: yes + + # if yes, perform prefetching of almost expired message cache entries. + # prefetch: no + + # if yes, perform key lookups adjacent to normal lookups. + # prefetch-key: no + + # deny queries of type ANY with an empty response. + # deny-any: no + + # if yes, Unbound rotates RRSet order in response. + # rrset-roundrobin: no + + # if yes, Unbound doesn't insert authority/additional sections + # into response messages when those sections are not required. + # minimal-responses: yes + + # true to disable DNSSEC lameness check in iterator. + # disable-dnssec-lame-check: no + + # module configuration of the server. A string with identifiers + # separated by spaces. Syntax: "[dns64] [validator] iterator" + # most modules have to be listed at the beginning of the line, + # except cachedb(just before iterator), and python (at the beginning, + # or, just before the iterator). + module-config: "dns64 validator iterator" + + # File with trusted keys, kept uptodate using RFC5011 probes, + # initial file like trust-anchor-file, then it stores metadata. + # Use several entries, one per domain name, to track multiple zones. + # + # If you want to perform DNSSEC validation, run unbound-anchor before + # you start unbound (i.e. in the system boot scripts). And enable: + # Please note usage of unbound-anchor root anchor is at your own risk + # and under the terms of our LICENSE (see that file in the source). + # auto-trust-anchor-file: "" + + # trust anchor signaling sends a RFC8145 key tag query after priming. + # trust-anchor-signaling: yes + + # Root key trust anchor sentinel (draft-ietf-dnsop-kskroll-sentinel) + # root-key-sentinel: yes + + # File with DLV trusted keys. Same format as trust-anchor-file. + # There can be only one DLV configured, it is trusted from root down. + # DLV is going to be decommissioned. Please do not use it any more. + # dlv-anchor-file: "dlv.isc.org.key" + + # File with trusted keys for validation. Specify more than one file + # with several entries, one file per entry. + # Zone file format, with DS and DNSKEY entries. + # Note this gets out of date, use auto-trust-anchor-file please. + trust-anchor-file: "/usr/share/dnssec-root/trusted-key.key" + + # Trusted key for validation. DS or DNSKEY. specify the RR on a + # single line, surrounded by "". TTL is ignored. class is IN default. + # Note this gets out of date, use auto-trust-anchor-file please. + # (These examples are from August 2007 and may not be valid anymore). + # trust-anchor: "nlnetlabs.nl. DNSKEY 257 3 5 AQPzzTWMz8qSWIQlfRnPckx2BiVmkVN6LPupO3mbz7FhLSnm26n6iG9N Lby97Ji453aWZY3M5/xJBSOS2vWtco2t8C0+xeO1bc/d6ZTy32DHchpW 6rDH1vp86Ll+ha0tmwyy9QP7y2bVw5zSbFCrefk8qCUBgfHm9bHzMG1U BYtEIQ==" + # trust-anchor: "jelte.nlnetlabs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A" + + # File with trusted keys for validation. Specify more than one file + # with several entries, one file per entry. Like trust-anchor-file + # but has a different file format. Format is BIND-9 style format, + # the trusted-keys { name flag proto algo "key"; }; clauses are read. + # you need external update procedures to track changes in keys. + # trusted-keys-file: "" + + # Ignore chain of trust. Domain is treated as insecure. + # domain-insecure: "example.com" + + # Override the date for validation with a specific fixed date. + # Do not set this unless you are debugging signature inception + # and expiration. "" or "0" turns the feature off. -1 ignores date. + # val-override-date: "" + + # The time to live for bogus data, rrsets and messages. This avoids + # some of the revalidation, until the time interval expires. in secs. + # val-bogus-ttl: 60 + + # The signature inception and expiration dates are allowed to be off + # by 10% of the signature lifetime (expir-incep) from our local clock. + # This leeway is capped with a minimum and a maximum. In seconds. + # val-sig-skew-min: 3600 + # val-sig-skew-max: 86400 + + # Should additional section of secure message also be kept clean of + # unsecure data. Useful to shield the users of this validator from + # potential bogus data in the additional section. All unsigned data + # in the additional section is removed from secure messages. + # val-clean-additional: yes + + # Turn permissive mode on to permit bogus messages. Thus, messages + # for which security checks failed will be returned to clients, + # instead of SERVFAIL. It still performs the security checks, which + # result in interesting log files and possibly the AD bit in + # replies if the message is found secure. The default is off. + # val-permissive-mode: no + + # Ignore the CD flag in incoming queries and refuse them bogus data. + # Enable it if the only clients of unbound are legacy servers (w2008) + # that set CD but cannot validate themselves. + # ignore-cd-flag: no + + # Serve expired responses from cache, with TTL 0 in the response, + # and then attempt to fetch the data afresh. + # serve-expired: no + # + # Limit serving of expired responses to configured seconds after + # expiration. 0 disables the limit. + # serve-expired-ttl: 0 + # + # Set the TTL of expired records to the serve-expired-ttl value after a + # failed attempt to retrieve the record from upstream. This makes sure + # that the expired records will be served as long as there are queries + # for it. + # serve-expired-ttl-reset: no + + # Have the validator log failed validations for your diagnosis. + # 0: off. 1: A line per failed user query. 2: With reason and bad IP. + # val-log-level: 0 + + # It is possible to configure NSEC3 maximum iteration counts per + # keysize. Keep this table very short, as linear search is done. + # A message with an NSEC3 with larger count is marked insecure. + # List in ascending order the keysize and count values. + # val-nsec3-keysize-iterations: "1024 150 2048 500 4096 2500" + + # instruct the auto-trust-anchor-file probing to add anchors after ttl. + # add-holddown: 2592000 # 30 days + + # instruct the auto-trust-anchor-file probing to del anchors after ttl. + # del-holddown: 2592000 # 30 days + + # auto-trust-anchor-file probing removes missing anchors after ttl. + # If the value 0 is given, missing anchors are not removed. + # keep-missing: 31622400 # 366 days + + # debug option that allows very small holddown times for key rollover, + # otherwise the RFC mandates probe intervals must be at least 1 hour. + # permit-small-holddown: no + + # the amount of memory to use for the key cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # key-cache-size: 4m + + # the number of slabs to use for the key cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # key-cache-slabs: 4 + + # the amount of memory to use for the negative cache (used for DLV). + # plain value in bytes or you can append k, m or G. default is "1Mb". + # neg-cache-size: 1m + + # By default, for a number of zones a small default 'nothing here' + # reply is built-in. Query traffic is thus blocked. If you + # wish to serve such zone you can unblock them by uncommenting one + # of the nodefault statements below. + # You may also have to use domain-insecure: zone to make DNSSEC work, + # unless you have your own trust anchors for this zone. + # local-zone: "localhost." nodefault + # local-zone: "127.in-addr.arpa." nodefault + # local-zone: "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa." nodefault + # local-zone: "onion." nodefault + # local-zone: "test." nodefault + # local-zone: "invalid." nodefault + # local-zone: "10.in-addr.arpa." nodefault + # local-zone: "16.172.in-addr.arpa." nodefault + # local-zone: "17.172.in-addr.arpa." nodefault + # local-zone: "18.172.in-addr.arpa." nodefault + # local-zone: "19.172.in-addr.arpa." nodefault + # local-zone: "20.172.in-addr.arpa." nodefault + # local-zone: "21.172.in-addr.arpa." nodefault + # local-zone: "22.172.in-addr.arpa." nodefault + # local-zone: "23.172.in-addr.arpa." nodefault + # local-zone: "24.172.in-addr.arpa." nodefault + # local-zone: "25.172.in-addr.arpa." nodefault + # local-zone: "26.172.in-addr.arpa." nodefault + # local-zone: "27.172.in-addr.arpa." nodefault + # local-zone: "28.172.in-addr.arpa." nodefault + # local-zone: "29.172.in-addr.arpa." nodefault + # local-zone: "30.172.in-addr.arpa." nodefault + # local-zone: "31.172.in-addr.arpa." nodefault + # local-zone: "168.192.in-addr.arpa." nodefault + # local-zone: "0.in-addr.arpa." nodefault + # local-zone: "254.169.in-addr.arpa." nodefault + # local-zone: "2.0.192.in-addr.arpa." nodefault + # local-zone: "100.51.198.in-addr.arpa." nodefault + # local-zone: "113.0.203.in-addr.arpa." nodefault + # local-zone: "255.255.255.255.in-addr.arpa." nodefault + # local-zone: "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa." nodefault + # local-zone: "d.f.ip6.arpa." nodefault + # local-zone: "8.e.f.ip6.arpa." nodefault + # local-zone: "9.e.f.ip6.arpa." nodefault + # local-zone: "a.e.f.ip6.arpa." nodefault + # local-zone: "b.e.f.ip6.arpa." nodefault + # local-zone: "8.b.d.0.1.0.0.2.ip6.arpa." nodefault + # And for 64.100.in-addr.arpa. to 127.100.in-addr.arpa. + + # If unbound is running service for the local host then it is useful + # to perform lan-wide lookups to the upstream, and unblock the + # long list of local-zones above. If this unbound is a dns server + # for a network of computers, disabled is better and stops information + # leakage of local lan information. + # unblock-lan-zones: no + + # The insecure-lan-zones option disables validation for + # these zones, as if they were all listed as domain-insecure. + # insecure-lan-zones: no + + # a number of locally served zones can be configured. + # local-zone: + # local-data: "" + # o deny serves local data (if any), else, drops queries. + # o refuse serves local data (if any), else, replies with error. + # o static serves local data, else, nxdomain or nodata answer. + # o transparent gives local data, but resolves normally for other names + # o redirect serves the zone data for any subdomain in the zone. + # o nodefault can be used to normally resolve AS112 zones. + # o typetransparent resolves normally for other types and other names + # o inform acts like transparent, but logs client IP address + # o inform_deny drops queries and logs client IP address + # o inform_redirect redirects queries and logs client IP address + # o always_transparent, always_refuse, always_nxdomain, resolve in + # that way but ignore local data for that name + # o noview breaks out of that view towards global local-zones. + # + # defaults are localhost address, reverse for 127.0.0.1 and ::1 + # and nxdomain for AS112 zones. If you configure one of these zones + # the default content is omitted, or you can omit it with 'nodefault'. + # + # If you configure local-data without specifying local-zone, by + # default a transparent local-zone is created for the data. + # + # You can add locally served data with + # local-zone: "local." static + # local-data: "mycomputer.local. IN A 192.0.2.51" + # local-data: 'mytext.local TXT "content of text record"' + # + # You can override certain queries with + # local-data: "adserver.example.com A 127.0.0.1" + # + # You can redirect a domain to a fixed address with + # (this makes example.com, www.example.com, etc, all go to 192.0.2.3) + # local-zone: "example.com" redirect + # local-data: "example.com A 192.0.2.3" + # + # Shorthand to make PTR records, "IPv4 name" or "IPv6 name". + # You can also add PTR records using local-data directly, but then + # you need to do the reverse notation yourself. + # local-data-ptr: "192.0.2.3 www.example.com" + + # tag a localzone with a list of tag names (in "" with spaces between) + # local-zone-tag: "example.com" "tag2 tag3" + + # add a netblock specific override to a localzone, with zone type + # local-zone-override: "example.com" 192.0.2.0/24 refuse + + # service clients over TLS (on the TCP sockets), with plain DNS inside + # the TLS stream. Give the certificate to use and private key. + # default is "" (disabled). requires restart to take effect. + # tls-service-key: "path/to/privatekeyfile.key" + # tls-service-pem: "path/to/publiccertfile.pem" + # tls-port: 853 + + # cipher setting for TLSv1.2 + # tls-ciphers: "DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256" + # cipher setting for TLSv1.3 + # tls-ciphersuites: "TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_8_SHA256:TLS_AES_128_CCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256" + + # Add the secret file for TLS Session Ticket. + # Secret file must be 80 bytes of random data. + # First key use to encrypt and decrypt TLS session tickets. + # Other keys use to decrypt only. + # requires restart to take effect. + # tls-session-ticket-keys: "path/to/secret_file1" + # tls-session-ticket-keys: "path/to/secret_file2" + + # request upstream over TLS (with plain DNS inside the TLS stream). + # Default is no. Can be turned on and off with unbound-control. + # tls-upstream: no + + # Certificates used to authenticate connections made upstream. + # tls-cert-bundle: "" + + # Add system certs to the cert bundle, from the Windows Cert Store + # tls-win-cert: no + + # Also serve tls on these port numbers (eg. 443, ...), by listing + # tls-additional-port: portno for each of the port numbers. + + # DNS64 prefix. Must be specified when DNS64 is use. + # Enable dns64 in module-config. Used to synthesize IPv6 from IPv4. + dns64-prefix: $DNS64_PREFIX + + # DNS64 ignore AAAA records for these domains and use A instead. + # dns64-ignore-aaaa: "example.com" + + # ratelimit for uncached, new queries, this limits recursion effort. + # ratelimiting is experimental, and may help against randomqueryflood. + # if 0(default) it is disabled, otherwise state qps allowed per zone. + # ratelimit: 0 + + # ratelimits are tracked in a cache, size in bytes of cache (or k,m). + # ratelimit-size: 4m + # ratelimit cache slabs, reduces lock contention if equal to cpucount. + # ratelimit-slabs: 4 + + # 0 blocks when ratelimited, otherwise let 1/xth traffic through + # ratelimit-factor: 10 + + # override the ratelimit for a specific domain name. + # give this setting multiple times to have multiple overrides. + # ratelimit-for-domain: example.com 1000 + # override the ratelimits for all domains below a domain name + # can give this multiple times, the name closest to the zone is used. + # ratelimit-below-domain: com 1000 + + # global query ratelimit for all ip addresses. + # feature is experimental. + # if 0(default) it is disabled, otherwise states qps allowed per ip address + # ip-ratelimit: 0 + + # ip ratelimits are tracked in a cache, size in bytes of cache (or k,m). + # ip-ratelimit-size: 4m + # ip ratelimit cache slabs, reduces lock contention if equal to cpucount. + # ip-ratelimit-slabs: 4 + + # 0 blocks when ip is ratelimited, otherwise let 1/xth traffic through + # ip-ratelimit-factor: 10 + + # Limit the number of connections simultaneous from a netblock + # tcp-connection-limit: 192.0.2.0/24 12 + + # select from the fastest servers this many times out of 1000. 0 means + # the fast server select is disabled. prefetches are not sped up. + # fast-server-permil: 0 + # the number of servers that will be used in the fast server selection. + # fast-server-num: 3 + + # Specific options for ipsecmod. unbound needs to be configured with + # --enable-ipsecmod for these to take effect. + # + # Enable or disable ipsecmod (it still needs to be defined in + # module-config above). Can be used when ipsecmod needs to be + # enabled/disabled via remote-control(below). + # ipsecmod-enabled: yes + # + # Path to executable external hook. It must be defined when ipsecmod is + # listed in module-config (above). + # ipsecmod-hook: "./my_executable" + # + # When enabled unbound will reply with SERVFAIL if the return value of + # the ipsecmod-hook is not 0. + # ipsecmod-strict: no + # + # Maximum time to live (TTL) for cached A/AAAA records with IPSECKEY. + # ipsecmod-max-ttl: 3600 + # + # Reply with A/AAAA even if the relevant IPSECKEY is bogus. Mainly used for + # testing. + # ipsecmod-ignore-bogus: no + # + # Domains for which ipsecmod will be triggered. If not defined (default) + # all domains are treated as being whitelisted. + # ipsecmod-whitelist: "example.com" + # ipsecmod-whitelist: "nlnetlabs.nl" + + +# Python config section. To enable: +# o use --with-pythonmodule to configure before compiling. +# o list python in the module-config string (above) to enable. +# It can be at the start, it gets validated results, or just before +# the iterator and process before DNSSEC validation. +# o and give a python-script to run. +python: + # Script file to load + # python-script: "/ubmodule-tst.py" + +# Remote control config section. +remote-control: + # Enable remote control with unbound-control(8) here. + # set up the keys and certificates with unbound-control-setup. + # control-enable: no + + # what interfaces are listened to for remote control. + # give 0.0.0.0 and ::0 to listen to all interfaces. + # set to an absolute path to use a unix local name pipe, certificates + # are not used for that, so key and cert files need not be present. + # control-interface: 127.0.0.1 + # control-interface: ::1 + + # port number for remote control operations. + # control-port: 8953 + + # for localhost, you can disable use of TLS by setting this to "no" + # For local sockets this option is ignored, and TLS is not used. + # control-use-cert: "yes" + + # unbound server key file. + # server-key-file: "/unbound_server.key" + + # unbound server certificate file. + # server-cert-file: "/unbound_server.pem" + + # unbound-control key file. + # control-key-file: "/unbound_control.key" + + # unbound-control certificate file. + # control-cert-file: "/unbound_control.pem" + +# Stub zones. +# Create entries like below, to make all queries for 'example.com' and +# 'example.org' go to the given list of nameservers. list zero or more +# nameservers by hostname or by ipaddress. If you set stub-prime to yes, +# the list is treated as priming hints (default is no). +# With stub-first yes, it attempts without the stub if it fails. +# Consider adding domain-insecure: name and local-zone: name nodefault +# to the server: section if the stub is a locally served zone. +# stub-zone: +# name: "example.com" +# stub-addr: 192.0.2.68 +# stub-prime: no +# stub-first: no +# stub-tls-upstream: no +# stub-no-cache: no +# stub-zone: +# name: "example.org" +# stub-host: ns.example.com. + +# Forward zones +# Create entries like below, to make all queries for 'example.com' and +# 'example.org' go to the given list of servers. These servers have to handle +# recursion to other nameservers. List zero or more nameservers by hostname +# or by ipaddress. Use an entry with name "." to forward all queries. +# If you enable forward-first, it attempts without the forward if it fails. +# forward-zone: +# name: "example.com" +# forward-addr: 192.0.2.68 +# forward-addr: 192.0.2.73@5355 # forward to port 5355. +# forward-first: no +# forward-tls-upstream: no +# forward-no-cache: no +# forward-zone: +# name: "example.org" +# forward-host: fwd.example.com + +forward-zone: + name: "." +$(generate_forward_addr) + +# Authority zones +# The data for these zones is kept locally, from a file or downloaded. +# The data can be served to downstream clients, or used instead of the +# upstream (which saves a lookup to the upstream). The first example +# has a copy of the root for local usage. The second serves example.org +# authoritatively. zonefile: reads from file (and writes to it if you also +# download it), master: fetches with AXFR and IXFR, or url to zonefile. +# With allow-notify: you can give additional (apart from masters) sources of +# notifies. +# auth-zone: +# name: "." +# master: 199.9.14.201 # b.root-servers.net +# master: 192.33.4.12 # c.root-servers.net +# master: 199.7.91.13 # d.root-servers.net +# master: 192.5.5.241 # f.root-servers.net +# master: 192.112.36.4 # g.root-servers.net +# master: 193.0.14.129 # k.root-servers.net +# master: 192.0.47.132 # xfr.cjr.dns.icann.org +# master: 192.0.32.132 # xfr.lax.dns.icann.org +# master: 2001:500:200::b # b.root-servers.net +# master: 2001:500:2::c # c.root-servers.net +# master: 2001:500:2d::d # d.root-servers.net +# master: 2001:500:2f::f # f.root-servers.net +# master: 2001:500:12::d0d # g.root-servers.net +# master: 2001:7fd::1 # k.root-servers.net +# master: 2620:0:2830:202::132 # xfr.cjr.dns.icann.org +# master: 2620:0:2d0:202::132 # xfr.lax.dns.icann.org +# fallback-enabled: yes +# for-downstream: no +# for-upstream: yes +# auth-zone: +# name: "example.org" +# for-downstream: yes +# for-upstream: yes +# zonefile: "example.org.zone" + +# Views +# Create named views. Name must be unique. Map views to requests using +# the access-control-view option. Views can contain zero or more local-zone +# and local-data options. Options from matching views will override global +# options. Global options will be used if no matching view is found. +# With view-first yes, it will try to answer using the global local-zone and +# local-data elements if there is no view specific match. +# view: +# name: "viewname" +# local-zone: "example.com" redirect +# local-data: "example.com A 192.0.2.3" +# local-data-ptr: "192.0.2.3 www.example.com" +# view-first: no +# view: +# name: "anotherview" +# local-zone: "example.com" refuse + +# DNSCrypt +# Caveats: +# 1. the keys/certs cannot be produced by unbound. You can use dnscrypt-wrapper +# for this: https://github.com/cofyc/dnscrypt-wrapper/blob/master/README.md#usage +# 2. dnscrypt channel attaches to an interface. you MUST set interfaces to +# listen on `dnscrypt-port` with the follo0wing snippet: +# server: +# interface: 0.0.0.0@443 +# interface: ::0@443 +# +# Finally, `dnscrypt` config has its own section. +# dnscrypt: +# dnscrypt-enable: yes +# dnscrypt-port: 443 +# dnscrypt-provider: 2.dnscrypt-cert.example.com. +# dnscrypt-secret-key: /path/unbound-conf/keys1/1.key +# dnscrypt-secret-key: /path/unbound-conf/keys2/1.key +# dnscrypt-provider-cert: /path/unbound-conf/keys1/1.cert +# dnscrypt-provider-cert: /path/unbound-conf/keys2/1.cert + +# CacheDB +# Enable external backend DB as auxiliary cache. Specify the backend name +# (default is "testframe", which has no use other than for debugging and +# testing) and backend-specific options. The 'cachedb' module must be +# included in module-config, just before the iterator module. +# cachedb: +# backend: "testframe" +# # secret seed string to calculate hashed keys +# secret-seed: "default" +# +# # For "redis" backend: +# # redis server's IP address or host name +# redis-server-host: 127.0.0.1 +# # redis server's TCP port +# redis-server-port: 6379 +# # timeout (in ms) for communication with the redis server +# redis-timeout: 100 +EOF diff --git a/type/__ungleich_unbound/man.rst b/type/__ungleich_unbound/man.rst new file mode 100644 index 0000000..417b343 --- /dev/null +++ b/type/__ungleich_unbound/man.rst @@ -0,0 +1,74 @@ +cdist-type__ungleich_unbound(7) +=============================== + +NAME +---- +cdist-type__ungleich_unbound - unbound server deployment for ungleich + + +DESCRIPTION +----------- +This unbound (dns resolver and cache) deployment provides DNS64 and fetch +answers from specified upstrean DNS server. This is a singleton type. + +REQUIRED PARAMETERS +------------------- +dns64_prefix + IPv6 prefix used for DNS64. + +forward_addr + DNS servers used to lookup names, can be provided multiple times. It can be + either an IPv4 or IPv6 address but no domain name. + +OPTIONAL PARAMETERS +------------------- +interface + Interface to listen on, can be provided multiple times. Defaults to + '127.0.0.1' and '::1'. + +access_control + Controls which clients are allowed queries to the unbound service (everything + but localhost is refused by default), can be provided multiple times. The + format is described in unbound.conf(5). + +BOOLEAN PARAMETERS +------------------ +disable-ip4 + Do not answer or issue queries over IPv4. Cannot be used alongside the + `--disable-ip6` flag. + +disable-ip6 + Do not answer or issue queries over IPv6. Cannot be used alongside the + `--disable-ip4` flag. + +EXAMPLES +-------- + +.. code-block:: sh + + __ungleich_unbound \ + --interface '::0' \ + --dns64_prefix '2a0a:e5c0:2:10::/96' \ + --forward_addr '2a0a:e5c0:2:1::5' \ + --forward_addr '2a0a:e5c0:2:1::6' \ + --access_control '::0/0 deny' \ + --access_control '2a0a:e5c0::/29 allow' \ + --access_control '2a09:2940::/29 allow' \ + --ip6 + +SEE ALSO +-------- +- `unbound.conf(5) `_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__ungleich_unbound/manifest b/type/__ungleich_unbound/manifest new file mode 100755 index 0000000..525a981 --- /dev/null +++ b/type/__ungleich_unbound/manifest @@ -0,0 +1,79 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + alpine) + __package unbound --state present + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +# Required parameters: +export DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") +export FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") + +# Optional parameters: +if [ -f "$__object/parameter/interface" ]; then + export INTERFACES=$(cat "$__object/parameter/interface") +fi + +if [ -f "$__object/parameter/access_control" ]; then + export ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") +fi + +# Boolean parameters: +if [ -f "$__object/parameter/disable_ip4" ] && \ + [ -f "$__object/parameter/disable_ip6" ]; then + echo "--disable-ip4 and --disable-ip6 cannot be used at the same time." >&2 + exit 1 +fi + +if [ -f "$__object/parameter/disable_ip4" ]; then + export DO_IP4='no' +else + export DO_IP4='yes' +fi + +if [ -f "$__object/parameter/disable_ip6" ]; then + export DO_IP6='no' +else + export DO_IP6='yes' +fi + +# Generate and deploy configuration files. +source_file="$__object/files/unbound.conf" +target_file="/etc/unbound/unbound.conf" + +mkdir -p "$__object/files" +"$__type/files/unbound.conf.sh" > $source_file +require="__package/unbound" __file $target_file \ + --source $source_file \ + --owner root \ + --mode 644 + +# Restart unbound server after reconfiguration. +require="__file/$target_file" __service unbound --action restart diff --git a/type/__ungleich_unbound/parameter/boolean b/type/__ungleich_unbound/parameter/boolean new file mode 100644 index 0000000..a2d5910 --- /dev/null +++ b/type/__ungleich_unbound/parameter/boolean @@ -0,0 +1,2 @@ +disable_ip6 +disable_ip4 diff --git a/type/__ungleich_unbound/parameter/optional_multiple b/type/__ungleich_unbound/parameter/optional_multiple new file mode 100644 index 0000000..ecacd1d --- /dev/null +++ b/type/__ungleich_unbound/parameter/optional_multiple @@ -0,0 +1,2 @@ +access_control +interface diff --git a/type/__ungleich_unbound/parameter/required b/type/__ungleich_unbound/parameter/required new file mode 100644 index 0000000..4ad232f --- /dev/null +++ b/type/__ungleich_unbound/parameter/required @@ -0,0 +1 @@ +dns64_prefix diff --git a/type/__ungleich_unbound/parameter/required_multiple b/type/__ungleich_unbound/parameter/required_multiple new file mode 100644 index 0000000..776abf5 --- /dev/null +++ b/type/__ungleich_unbound/parameter/required_multiple @@ -0,0 +1 @@ +forward_addr diff --git a/type/__ungleich_unbound/singleton b/type/__ungleich_unbound/singleton new file mode 100644 index 0000000..e69de29 From 6b38b248e2227dc29fca104db29f2656c4310202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 14:59:42 +0200 Subject: [PATCH 022/156] Rename __ungleich_unbound into __unbound --- type/{__ungleich_unbound => __unbound}/files/unbound.conf.sh | 0 type/{__ungleich_unbound => __unbound}/man.rst | 2 +- type/{__ungleich_unbound => __unbound}/manifest | 0 type/{__ungleich_unbound => __unbound}/parameter/boolean | 0 .../parameter/optional_multiple | 0 type/{__ungleich_unbound => __unbound}/parameter/required | 0 .../parameter/required_multiple | 0 type/{__ungleich_unbound => __unbound}/singleton | 0 8 files changed, 1 insertion(+), 1 deletion(-) rename type/{__ungleich_unbound => __unbound}/files/unbound.conf.sh (100%) rename type/{__ungleich_unbound => __unbound}/man.rst (98%) rename type/{__ungleich_unbound => __unbound}/manifest (100%) rename type/{__ungleich_unbound => __unbound}/parameter/boolean (100%) rename type/{__ungleich_unbound => __unbound}/parameter/optional_multiple (100%) rename type/{__ungleich_unbound => __unbound}/parameter/required (100%) rename type/{__ungleich_unbound => __unbound}/parameter/required_multiple (100%) rename type/{__ungleich_unbound => __unbound}/singleton (100%) diff --git a/type/__ungleich_unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh similarity index 100% rename from type/__ungleich_unbound/files/unbound.conf.sh rename to type/__unbound/files/unbound.conf.sh diff --git a/type/__ungleich_unbound/man.rst b/type/__unbound/man.rst similarity index 98% rename from type/__ungleich_unbound/man.rst rename to type/__unbound/man.rst index 417b343..6d113b0 100644 --- a/type/__ungleich_unbound/man.rst +++ b/type/__unbound/man.rst @@ -1,4 +1,4 @@ -cdist-type__ungleich_unbound(7) +cdist-type__unbound(7) =============================== NAME diff --git a/type/__ungleich_unbound/manifest b/type/__unbound/manifest similarity index 100% rename from type/__ungleich_unbound/manifest rename to type/__unbound/manifest diff --git a/type/__ungleich_unbound/parameter/boolean b/type/__unbound/parameter/boolean similarity index 100% rename from type/__ungleich_unbound/parameter/boolean rename to type/__unbound/parameter/boolean diff --git a/type/__ungleich_unbound/parameter/optional_multiple b/type/__unbound/parameter/optional_multiple similarity index 100% rename from type/__ungleich_unbound/parameter/optional_multiple rename to type/__unbound/parameter/optional_multiple diff --git a/type/__ungleich_unbound/parameter/required b/type/__unbound/parameter/required similarity index 100% rename from type/__ungleich_unbound/parameter/required rename to type/__unbound/parameter/required diff --git a/type/__ungleich_unbound/parameter/required_multiple b/type/__unbound/parameter/required_multiple similarity index 100% rename from type/__ungleich_unbound/parameter/required_multiple rename to type/__unbound/parameter/required_multiple diff --git a/type/__ungleich_unbound/singleton b/type/__unbound/singleton similarity index 100% rename from type/__ungleich_unbound/singleton rename to type/__unbound/singleton From 3d3a3ef8024a5dc201253bfa5a67b1508510e0d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 15:08:04 +0200 Subject: [PATCH 023/156] Fix shellcheck warnings in newly imported __unbound type --- type/__unbound/files/unbound.conf.sh | 6 +++--- type/__unbound/manifest | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index e713843..effea65 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -7,7 +7,7 @@ generate_interface() { } generate_access_control() { - echo "$ACCESS_CONTROLS" | while read entry; do + echo "$ACCESS_CONTROLS" | while read -r entry; do echo " access-control: $entry" done } @@ -983,12 +983,12 @@ $(generate_forward_addr) # 1. the keys/certs cannot be produced by unbound. You can use dnscrypt-wrapper # for this: https://github.com/cofyc/dnscrypt-wrapper/blob/master/README.md#usage # 2. dnscrypt channel attaches to an interface. you MUST set interfaces to -# listen on `dnscrypt-port` with the follo0wing snippet: +# listen on 'dnscrypt-port' with the follo0wing snippet: # server: # interface: 0.0.0.0@443 # interface: ::0@443 # -# Finally, `dnscrypt` config has its own section. +# Finally, 'dnscrypt' config has its own section. # dnscrypt: # dnscrypt-enable: yes # dnscrypt-port: 443 diff --git a/type/__unbound/manifest b/type/__unbound/manifest index 525a981..a25cdec 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -33,16 +33,20 @@ case "$os" in esac # Required parameters: -export DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") -export FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") +DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") +export DNS64_PREFIX +FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") +export FORWARD_ADDRS # Optional parameters: if [ -f "$__object/parameter/interface" ]; then - export INTERFACES=$(cat "$__object/parameter/interface") + INTERFACES=$(cat "$__object/parameter/interface") + export INTERFACES fi if [ -f "$__object/parameter/access_control" ]; then - export ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") + ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") + export ACCESS_CONTROLS fi # Boolean parameters: @@ -69,9 +73,9 @@ source_file="$__object/files/unbound.conf" target_file="/etc/unbound/unbound.conf" mkdir -p "$__object/files" -"$__type/files/unbound.conf.sh" > $source_file -require="__package/unbound" __file $target_file \ - --source $source_file \ +"$__type/files/unbound.conf.sh" > "$source_file" +require="__package/unbound" __file "$target_file" \ + --source "$source_file" \ --owner root \ --mode 644 From 89f0dc90ead21d338c271cb283cd10d1ebe57ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 15:09:07 +0200 Subject: [PATCH 024/156] Add changelog entry for __unbound type --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2ebad0..a61c82d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ # cdist-contrib changes +* 2020-06-03: New type: __unbound (Timothée Floure) * 2020-04-28: New type: __find_exec (Ander Punnar) From de4508cb0695cf0f7a6561ac7d8b92add1bc313d Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 6 Jun 2020 21:45:40 +0200 Subject: [PATCH 025/156] Mark __dma_auth and __mail_alias as nonparallel Both types modify a single file, so they shouldn't be run at the same time. --- type/__dma_auth/nonparallel | 0 type/__mail_alias/nonparallel | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 type/__dma_auth/nonparallel create mode 100644 type/__mail_alias/nonparallel diff --git a/type/__dma_auth/nonparallel b/type/__dma_auth/nonparallel new file mode 100644 index 0000000..e69de29 diff --git a/type/__mail_alias/nonparallel b/type/__mail_alias/nonparallel new file mode 100644 index 0000000..e69de29 From 7b9ffb4a413693556a6c19258ba3a17d1614613f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:46:18 +0200 Subject: [PATCH 026/156] __unbound: wire remote control configuration --- type/__unbound/files/unbound.conf.sh | 12 ++++++------ type/__unbound/gencode-remote | 8 ++++++++ type/__unbound/man.rst | 6 ++++++ type/__unbound/manifest | 17 +++++++++++++++++ type/__unbound/parameter/boolean | 1 + type/__unbound/parameter/default/rc_interface | 1 + type/__unbound/parameter/optional | 1 + 7 files changed, 40 insertions(+), 6 deletions(-) create mode 100755 type/__unbound/gencode-remote create mode 100644 type/__unbound/parameter/default/rc_interface create mode 100644 type/__unbound/parameter/optional diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index effea65..46d2824 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -857,14 +857,14 @@ python: remote-control: # Enable remote control with unbound-control(8) here. # set up the keys and certificates with unbound-control-setup. - # control-enable: no + control-enable: $RC_ENABLE # what interfaces are listened to for remote control. # give 0.0.0.0 and ::0 to listen to all interfaces. # set to an absolute path to use a unix local name pipe, certificates # are not used for that, so key and cert files need not be present. # control-interface: 127.0.0.1 - # control-interface: ::1 + control-interface: $RC_INTERFACE # port number for remote control operations. # control-port: 8953 @@ -874,16 +874,16 @@ remote-control: # control-use-cert: "yes" # unbound server key file. - # server-key-file: "/unbound_server.key" + server-key-file: "$RC_SERVER_KEY_FILE" # unbound server certificate file. - # server-cert-file: "/unbound_server.pem" + server-cert-file: "$RC_SERVER_CERT_FILE" # unbound-control key file. - # control-key-file: "/unbound_control.key" + control-key-file: "$RC_CONTROL_KEY_FILE" # unbound-control certificate file. - # control-cert-file: "/unbound_control.pem" + control-cert-file: "$RC_CONTROL_CERT_FILE" # Stub zones. # Create entries like below, to make all queries for 'example.com' and diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote new file mode 100755 index 0000000..5d2bb4a --- /dev/null +++ b/type/__unbound/gencode-remote @@ -0,0 +1,8 @@ +#!/bin/sh + +UNBOUND_CERTS_DIR=/etc/unbound + +if [ -f "$__object/parameter/enable_rc" ]; then + echo "unbound-control-setup -d $UNBOUND_CERTS_DIR" + echo "chown unbound:unbound $UNBOUND_CERTS_DIR/*.pem $UNBOUND_CERTS_DIR/*.key" +fi diff --git a/type/__unbound/man.rst b/type/__unbound/man.rst index 6d113b0..679e601 100644 --- a/type/__unbound/man.rst +++ b/type/__unbound/man.rst @@ -31,6 +31,9 @@ access_control but localhost is refused by default), can be provided multiple times. The format is described in unbound.conf(5). +rc_interface + Address or path to socket used for remote control (see `--enable_control`. Defaults to `127.0.0.1`). + BOOLEAN PARAMETERS ------------------ disable-ip4 @@ -41,6 +44,9 @@ disable-ip6 Do not answer or issue queries over IPv6. Cannot be used alongside the `--disable-ip4` flag. +enable_rc + Enable remote control (see `unbound-control(8)`). + EXAMPLES -------- diff --git a/type/__unbound/manifest b/type/__unbound/manifest index a25cdec..d85ad66 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -49,6 +49,11 @@ if [ -f "$__object/parameter/access_control" ]; then export ACCESS_CONTROLS fi +if [ -f "$__object/parameter/rc_interface" ]; then + RC_INTERFACE=$(cat "$__object/parameter/rc_interface") + export RC_INTERFACE +fi + # Boolean parameters: if [ -f "$__object/parameter/disable_ip4" ] && \ [ -f "$__object/parameter/disable_ip6" ]; then @@ -68,6 +73,18 @@ else export DO_IP6='yes' fi +if [ -f "$__object/parameter/enable_rc" ]; then + export RC_ENABLE='yes' +else + export RC_ENABLE='no' +fi + +# Certs for remote control: +export RC_SERVER_KEY_FILE='/etc/unbound/unbound_server.key' +export RC_SERVER_CERT_FILE='/etc/unbound/unbound_server.pem' +export RC_CONTROL_KEY_FILE='/etc/unbound/unbound_control.key' +export RC_CONTROL_CERT_FILE='/etc/unbound/unbound_control.pem' + # Generate and deploy configuration files. source_file="$__object/files/unbound.conf" target_file="/etc/unbound/unbound.conf" diff --git a/type/__unbound/parameter/boolean b/type/__unbound/parameter/boolean index a2d5910..b6e53a1 100644 --- a/type/__unbound/parameter/boolean +++ b/type/__unbound/parameter/boolean @@ -1,2 +1,3 @@ disable_ip6 disable_ip4 +enable_rc diff --git a/type/__unbound/parameter/default/rc_interface b/type/__unbound/parameter/default/rc_interface new file mode 100644 index 0000000..7b9ad53 --- /dev/null +++ b/type/__unbound/parameter/default/rc_interface @@ -0,0 +1 @@ +127.0.0.1 diff --git a/type/__unbound/parameter/optional b/type/__unbound/parameter/optional new file mode 100644 index 0000000..0826b6e --- /dev/null +++ b/type/__unbound/parameter/optional @@ -0,0 +1 @@ +rc_interface From 1e45d85d4d3d171b211e957e099f0e1c1580fe57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:53:56 +0200 Subject: [PATCH 027/156] Add new __unbound_exporter type --- type/__unbound_exporter/files/openrc-service | 12 ++++ type/__unbound_exporter/gencode-remote | 46 ++++++++++++++ type/__unbound_exporter/man.rst | 63 ++++++++++++++++++++ type/__unbound_exporter/manifest | 44 ++++++++++++++ type/__unbound_exporter/parameter/required | 1 + type/__unbound_exporter/singleton | 0 6 files changed, 166 insertions(+) create mode 100644 type/__unbound_exporter/files/openrc-service create mode 100755 type/__unbound_exporter/gencode-remote create mode 100644 type/__unbound_exporter/man.rst create mode 100755 type/__unbound_exporter/manifest create mode 100644 type/__unbound_exporter/parameter/required create mode 100644 type/__unbound_exporter/singleton diff --git a/type/__unbound_exporter/files/openrc-service b/type/__unbound_exporter/files/openrc-service new file mode 100644 index 0000000..6caed5e --- /dev/null +++ b/type/__unbound_exporter/files/openrc-service @@ -0,0 +1,12 @@ +#!/sbin/openrc-run + +name=$RC_SVCNAME +command="/usr/local/bin/unbound_exporter" +command_args="" +command_user="unbound" +command_background="yes" +pidfile="/var/run/$RC_SVCNAME.pid" + +depend() { + need unbound +} diff --git a/type/__unbound_exporter/gencode-remote b/type/__unbound_exporter/gencode-remote new file mode 100755 index 0000000..ef7ef9b --- /dev/null +++ b/type/__unbound_exporter/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +upstream=https://github.com/wish/unbound_exporter/archive +version=$(cat $__object/parameter/version) +release="unbound_exporter-$version" + +cat << EOF +if command -v unbound_exporter +then + # already installed - ignoring. + echo "Nothing to do -" +else + # Initialize working directory + workdir=\$(mktemp -d) + cd \$workdir + + # Download and extract sources for requested release. + curl -L $upstream/v$version.tar.gz --output $release.tar.gz + tar xf $release.tar.gz + + # Build and install binary. + cd $release + go build + install -m755 unbound_exporter /usr/local/bin/ + + # Clean up! + rm -r \$workdir +fi +EOF diff --git a/type/__unbound_exporter/man.rst b/type/__unbound_exporter/man.rst new file mode 100644 index 0000000..934bdd7 --- /dev/null +++ b/type/__unbound_exporter/man.rst @@ -0,0 +1,63 @@ +cdist-type__unbound_exporter(7) +=============================== + +NAME +---- +cdist-type__unbound_exporter - A prometheus exporter for unbound + + +DESCRIPTION +----------- +Simple Prometheus metrics exporter for the Unbound DNS +resolver. It leverages the unbound remote control endpoint +and exposes metrics on port 9167. + + +REQUIRED PARAMETERS +------------------- +version + unbound_exporter release to be used. + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __unbound \ + --interface '::0' \ + --forward_addr '2a0a:e5c0:2:1::5' \ + --forward_addr '2a0a:e5c0:2:1::6' \ + --access_control '::0/0 deny' \ + --access_control '2a0a:e5c0::/29 allow' \ + --access_control '2a09:2940::/29 allow' \ + --disable_ip4 \ + --enable_rc \ + --rc_interface '::1' + + __unbound_exporter --version 0.1.3 + +SEE ALSO +-------- +:strong:`cdist-type__unbound(7)` + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__unbound_exporter/manifest b/type/__unbound_exporter/manifest new file mode 100755 index 0000000..9f71834 --- /dev/null +++ b/type/__unbound_exporter/manifest @@ -0,0 +1,44 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + alpine) + # Used in gencode-remote. + __package curl + __package tar + __package go + __package libc-dev + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +__file /etc/init.d/unbound_exporter \ + --source "$__type/files/openrc-service" \ + --mode 755 + +require="__file/etc/init.d/unbound_exporter" __service unbound_exporter --action start +require="__file/etc/init.d/unbound_exporter" __start_on_boot unbound_exporter diff --git a/type/__unbound_exporter/parameter/required b/type/__unbound_exporter/parameter/required new file mode 100644 index 0000000..088eda4 --- /dev/null +++ b/type/__unbound_exporter/parameter/required @@ -0,0 +1 @@ +version diff --git a/type/__unbound_exporter/singleton b/type/__unbound_exporter/singleton new file mode 100644 index 0000000..e69de29 From d14011c50b8dbfbddf1adfab9e7823e462ef042c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:55:34 +0200 Subject: [PATCH 028/156] Sync CHANGELOG with __unbound* changes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a61c82d..df26f7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # cdist-contrib changes +* 2020-06-07: New type: __unbound_exporter (Timothée Floure) +* 2020-06-07: Extended type: wire remote control configuration for __unbond (Timothée Floure) * 2020-06-03: New type: __unbound (Timothée Floure) * 2020-04-28: New type: __find_exec (Ander Punnar) From 17f6a4dd471193369e17b0287817fc9cd43895b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:57:51 +0200 Subject: [PATCH 029/156] Only shellcheck .sh files in type/*/files/ --- scripts/run-shellcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-shellcheck.sh b/scripts/run-shellcheck.sh index 769f853..475f426 100755 --- a/scripts/run-shellcheck.sh +++ b/scripts/run-shellcheck.sh @@ -15,7 +15,7 @@ check () { } check -path "*/explorer/*" -check -path "*/files/*" +check -path "*/files/*.sh" check -name manifest check -name gencode-local check -name gencode-remote From bffc38ad52a54d7c80354fac46bf9f556fb4e0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:58:14 +0200 Subject: [PATCH 030/156] __unbound_exporter: make shellcheck happy again --- type/__unbound_exporter/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__unbound_exporter/gencode-remote b/type/__unbound_exporter/gencode-remote index ef7ef9b..fcd85fb 100755 --- a/type/__unbound_exporter/gencode-remote +++ b/type/__unbound_exporter/gencode-remote @@ -18,7 +18,7 @@ # along with cdist. If not, see . upstream=https://github.com/wish/unbound_exporter/archive -version=$(cat $__object/parameter/version) +version=$(cat "$__object/parameter/version") release="unbound_exporter-$version" cat << EOF From 4ff703e6aa74e18b679302e4b02cd2b9fd41fe6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 10:34:35 +0200 Subject: [PATCH 031/156] Fine tuning prometheus exporter for unbound --- type/__unbound/gencode-remote | 8 ++++++++ type/__unbound/manifest | 3 --- type/__unbound_exporter/manifest | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote index 5d2bb4a..d12a395 100755 --- a/type/__unbound/gencode-remote +++ b/type/__unbound/gencode-remote @@ -6,3 +6,11 @@ if [ -f "$__object/parameter/enable_rc" ]; then echo "unbound-control-setup -d $UNBOUND_CERTS_DIR" echo "chown unbound:unbound $UNBOUND_CERTS_DIR/*.pem $UNBOUND_CERTS_DIR/*.key" fi + +cat << EOF +if pgrep unbound; then + service unbound reload +else + service unbounb start +fi +EOF diff --git a/type/__unbound/manifest b/type/__unbound/manifest index d85ad66..346f30c 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -95,6 +95,3 @@ require="__package/unbound" __file "$target_file" \ --source "$source_file" \ --owner root \ --mode 644 - -# Restart unbound server after reconfiguration. -require="__file/$target_file" __service unbound --action restart diff --git a/type/__unbound_exporter/manifest b/type/__unbound_exporter/manifest index 9f71834..3602e47 100755 --- a/type/__unbound_exporter/manifest +++ b/type/__unbound_exporter/manifest @@ -26,6 +26,7 @@ case "$os" in # Used in gencode-remote. __package curl __package tar + __package openssl __package go __package libc-dev ;; From 0cd19b3a5dc71fa3fed263fe14eb05aafa58b0aa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Jun 2020 14:44:54 +0200 Subject: [PATCH 032/156] [type/__dma] Use "smarthost" spelling to be consistent with DMA --- type/__dma/gencode-remote | 8 ++++---- type/__dma/man.rst | 14 ++++++-------- type/__dma/parameter/required | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 1b00f04..e2bb405 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -22,9 +22,9 @@ fi # Generate config conf_should=$( - if test -s "${__object}/parameter/smart-host" + if test -s "${__object}/parameter/smarthost" then - printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smart-host")" + printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smarthost")" fi printf 'MAILNAME %s\n' "${mailname}" @@ -85,8 +85,8 @@ conf_should=$( if test -f "${__object}/parameter/null-client" then - test -s "${__object}/parameter/smart-host" || { - echo '--null-client requires a --smart-host to be defined' >&2 + test -s "${__object}/parameter/smarthost" || { + echo '--null-client requires a --smarthost to be defined' >&2 exit 1 } diff --git a/type/__dma/man.rst b/type/__dma/man.rst index a8ba546..cbc1c0c 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -13,15 +13,15 @@ mails from locally installed Mail User Agents (MUA) and deliver the mails to a remote destination. Remote delivery happens over TLS to one or more mailboxes that are local to the -email server configured in the `smart-host` parameter. +mail server configured in the ``smarthost`` parameter. REQUIRED PARAMETERS ------------------- -smart-host - The email server used to send email. +smarthost + The mail server used to send email. It must be configured to act as a relay for the host being configured by - this type so that mail can be sent to users non-local to the smart-host. + this type so that mail can be sent to users non-local to the smarthost. BOOLEAN PARAMETERS @@ -33,7 +33,7 @@ full-bounce not just the headers. null-client Enable to bypass aliases and local delivery, and instead forward all mails - to the defined `--smart-host`. + to the defined ``--smarthost``. send-test-email If present, after setup this type will send an email to root, to allow you to easily test your setup. @@ -78,9 +78,7 @@ EXAMPLES .. code-block:: sh - __dma \ - --smart-host mx1.domain.tld \ - --send-test-email + __dma --smarthost mx1.domain.tld --send-test-email SEE ALSO diff --git a/type/__dma/parameter/required b/type/__dma/parameter/required index 262568f..0753fb6 100644 --- a/type/__dma/parameter/required +++ b/type/__dma/parameter/required @@ -1 +1 @@ -smart-host +smarthost From 45b10f3e098f3e06dccdc2483c36e56527e0b9a1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Jun 2020 14:51:11 +0200 Subject: [PATCH 033/156] [type/__dma] Update parameters to match config names in DMA --- type/__dma/gencode-remote | 6 +++--- type/__dma/parameter/boolean | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index e2bb405..e4760d8 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -78,15 +78,15 @@ conf_should=$( echo 'DEFER' fi - if test -f "${__object}/parameter/full-bounce" + if test -f "${__object}/parameter/fullbounce" then echo 'FULLBOUNCE' fi - if test -f "${__object}/parameter/null-client" + if test -f "${__object}/parameter/nullclient" then test -s "${__object}/parameter/smarthost" || { - echo '--null-client requires a --smarthost to be defined' >&2 + echo '--nullclient requires a --smarthost to be defined' >&2 exit 1 } diff --git a/type/__dma/parameter/boolean b/type/__dma/parameter/boolean index ede7dda..523bb97 100644 --- a/type/__dma/parameter/boolean +++ b/type/__dma/parameter/boolean @@ -1,4 +1,4 @@ defer -full-bounce -null-client -send-test-email +fullbounce +nullclient +send-test-mail From 67b989a717d1c2c817b99220712bab7e78a679e3 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Jun 2020 20:53:01 +0200 Subject: [PATCH 034/156] [type/__dma_auth] Simplify code and add more comments --- type/__dma_auth/explorer/{authusers => state} | 51 ++++++++--- type/__dma_auth/gencode-remote | 84 ++++++++++++------- 2 files changed, 91 insertions(+), 44 deletions(-) rename type/__dma_auth/explorer/{authusers => state} (65%) diff --git a/type/__dma_auth/explorer/authusers b/type/__dma_auth/explorer/state similarity index 65% rename from type/__dma_auth/explorer/authusers rename to type/__dma_auth/explorer/state index c76667b..18e13ce 100755 --- a/type/__dma_auth/explorer/authusers +++ b/type/__dma_auth/explorer/state @@ -31,27 +31,54 @@ else fi awk -F'\n' -v server="${server}" ' +function getvalue(path) { + getline < path + close(path) + return $0 +} + BEGIN { DP = "[: \t]" # copied from dma/conf.c + + parameter_dir = ENVIRON["__object"] "/parameter/" + + host_param = getvalue(parameter_dir "server") + if (!host_param) host_param = ENVIRON["__object_id"] + login_param = getvalue(parameter_dir "login") + passwd_param = getvalue(parameter_dir "password") + + state = "absent" } -# skip comments and empty lines -/^#/ || /^$/ { next } +/^#/ || /^$/ { + # skip comments and empty lines + next +} { + # parse line + login = substr($0, 1, index($0, "|") - 1) + if (!login) { login = $0 } # if no "|" found + host = substr($0, length(login) + 2) + if (match(host, DP)) { + passwd = substr(host, RSTART) host = substr(host, 1, RSTART - 1) - endpos = length(login) + RSTART - } else endpos = length + } else { + passwd = "" + } } -host == server { print endpos, $0 } -' "${auth_conf}" \ -| while read -r pos line - do - printf '%s:%s\n' \ - "$(printf '%s' "$line" | cut -c $((-pos)))" \ - "$(printf '%s' "$line" | cut -c $((pos+2))- | cksum | cut -d' ' -f1)" - done +host == host_param && login == login_param { + if (passwd == passwd_param) + state = "present" + else + state = "different_password" +} + +END { + print state +} +' "${auth_conf}" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index b6b7f63..262a17a 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,6 +18,7 @@ # along with cdist. If not, see . # +state_is=$(cat "${__object}/explorer/state") state_should=$(cat "${__object}/parameter/state") if test -f "${__object}/parameter/server" @@ -28,33 +29,27 @@ else fi login=$(cat "${__object}/parameter/login") +if test "${state_is}" = "${state_should}" +then + # state is as it should + exit 0 +fi + case $state_should in (present) - line_should=$(printf '%s|%s:%s\n' \ - "${login}" "${server}" \ - "$(cksum "${__object}/parameter/password" | cut -d' ' -f1)") - if grep -qxF "${line_should}" "${__object}/explorer/authusers" - then - # correct line already present -> nothing to do - exit 0 - fi - test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1; } mode=1 - if test -s "${__object}/explorer/authusers" + if test "${state_is}" = 'absent' then - printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" - else printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + else + printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" fi ;; (absent) - # no matching logins present -> nothing to do - test -s "${__object}/explorer/authusers" || exit 0 - mode=0 printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" @@ -67,16 +62,14 @@ esac auth_conf=$(cat "${__object}/explorer/auth_conf") -if test -z "${auth_conf}" -then +test -n "${auth_conf}" || { echo 'Cannot determine path of dma auth.conf' >&2 exit 1 -fi +} + cat < drop all lines for this host + next + } + } } +# leave other lines alone { print } END { if (mode && !written) { - printf "%s|%s:%s\n", ENVIRON["login"], ENVIRON["server"], getpw() + # append line if no match to replace was found + print_should() } } ' <"${auth_conf}" >"${auth_conf}.tmp" \ From 96fcccf5294e1ed381096b731207674807ce6222 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Jun 2020 21:39:28 +0200 Subject: [PATCH 035/156] [type/__mail_alias] Improve documentation --- type/__mail_alias/explorer/aliases | 43 ++++++++++++------ type/__mail_alias/explorer/aliases_file | 2 +- type/__mail_alias/gencode-remote | 59 ++++++++++++++++--------- 3 files changed, 69 insertions(+), 35 deletions(-) diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index 66940d5..0a8da94 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -32,26 +32,41 @@ function print_aliases(aliases, matches) { } /^#/ { - # comment - select = 0; cont = 0; next -} - -{ - cont = ($0 ~ /\\$/) - if (cont) sub(/[ \t]*\\$/, "", $0) -} - -/^[[:blank:]]/ || cont { - # continuation line - if (select) print_aliases($0) + # comment line (ignore) + select = 0; cont = 0 # comments terminate alias lists and continuations next } -$1 == ENVIRON["__object_id"] { +/^[ \t]/ || cont { + # continuation line (either the previous line ended in a backslash or the + # line starts with whitespace) + + if (select) + print_aliases($0) +} + +{ + # detect if the line is a line to be continued (ends with a backslash) + cont = ($0 ~ /\\$/) + + # if it is, we drop the backslash from the line and skip to next line + # (the contents have been printed above if they should) + if (cont) { + sub(/[ \t]*\\$/, "", $0) + next + } +} + +$1 == ENVIRON["__object_id"] && !select { + # "target" user -> print alias list + # (only if !select; because of whitespacecontinuation lines) select = 1 print_aliases($2) next } -{ select = 0 } +{ + # other user + select = 0 +} ' "${aliases_file}" diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file index f7c4596..2710792 100755 --- a/type/__mail_alias/explorer/aliases_file +++ b/type/__mail_alias/explorer/aliases_file @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer tries to find the correct aliases file. +# This explorer finds the aliases file to modify. found() { echo "$*"; exit 0; } diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 7778536..f4cbf46 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -29,7 +29,13 @@ in exit 0 fi - echo "set aliases" >>"$__messages_out" + if test -s "${__object}/explorer/aliases" + then + echo "update aliases" >>"$__messages_out" + else + echo "add aliases" >>"$__messages_out" + fi + mode=1 ;; (absent) @@ -37,6 +43,7 @@ in test -s "${__object}/explorer/aliases" || exit 0 echo "delete aliases" >>"$__messages_out" + mode=0 ;; (*) @@ -46,11 +53,10 @@ esac aliases_file=$(cat "${__object}/explorer/aliases_file") -if test -z "${aliases_file}" -then +test -n "${aliases_file}" || { echo 'Could not determine aliases file path.' >&2 exit 1 -fi +} # "export" variables to remote printf 'mode=%u\n' "${mode}" @@ -58,16 +64,18 @@ printf "aliases_file='%s'\n" "${aliases_file}" cat <<'EOF' test -f "${aliases_file}" || touch "${aliases_file}" -awk -F ':[ \t]*' -v mode="${mode}" ' -function sepafter(f, default, _) { + +awk -F ':[ \t]*' -v mode=$mode ' +function sepafter(f, default, _) { + # finds the separator between field $f and $(f+1) _ = substr($0, length($f) + 1, index(substr($0, length($f)+1), $(f+1)) - 1) - if (_) return _ - else return default + return _ ? _ : default } function write_aliases() { if (aliases_written) return + # print aliases line printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") while ((getline < aliases_should_file) > 0) { if (aliases_written) printf ", " @@ -83,36 +91,45 @@ BEGIN { } /^#/ { - # comment - select = 0; cont = 0 + # comment line (leave alone) + select = 0; cont = 0 # comments terminate alias lists and continuations print next } -{ - cont = ($0 ~ /\\$/) - if (cont) sub(/[ \t]*\\$/, "", $0) -} - /^[ \t]/ || cont { - # continuation line + # continuation line (either the previous line ended in a backslash or the + # line starts with whitespace) + + # if in the alias list of the "target" user, we drop the line as it has been + # rewritten previously if (select) next } +{ + # detect if the line is a line to be continued (ends with a backslash) + cont = ($0 ~ /\\$/) + # if it is, we drop the backslash from the line. + if (cont) sub(/[ \t]*\\$/, "", $0) +} + $1 == ENVIRON["__object_id"] { - in_list = 1 + # "target" user -> rewrite aliases list + select = 1 if (mode) write_aliases() next } { - in_list = 0 + # other user + select = 0 print } END { - # if the last line as an alias definition, the separator will be reused - if (mode && !aliases_written) write_aliases() + # if the last line was an alias, the separator will be reused (looks better) + if (mode && !aliases_written) + write_aliases() } ' <"${aliases_file}" >"${aliases_file}.tmp" || { echo 'Generating new aliases file failed!' >&2 @@ -121,9 +138,11 @@ END { if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" then + # aliases file was modified, replace and run `newaliases` mv "${aliases_file}.tmp" "${aliases_file}" newaliases else + # no modifications were made, delete the temp file. rm "${aliases_file}.tmp" fi EOF From ca9e011d50ff2296fa0e4db74d99ecadbcee680f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 15:09:51 +0200 Subject: [PATCH 036/156] [type/__dma_auth] Fix off-by-one error --- type/__dma_auth/explorer/state | 2 +- type/__dma_auth/gencode-remote | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/type/__dma_auth/explorer/state b/type/__dma_auth/explorer/state index 18e13ce..0e6f7be 100755 --- a/type/__dma_auth/explorer/state +++ b/type/__dma_auth/explorer/state @@ -64,7 +64,7 @@ BEGIN { host = substr($0, length(login) + 2) if (match(host, DP)) { - passwd = substr(host, RSTART) + passwd = substr(host, RSTART + 1) host = substr(host, 1, RSTART - 1) } else { passwd = "" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 262a17a..c49779f 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -99,7 +99,10 @@ BEGIN { } # skip comments and empty lines -/^#/ || /^$/ { print; next } +/^#/ || /^$/ { + print + next +} { # parse line @@ -110,7 +113,7 @@ BEGIN { host = substr($0, length(login) + 2) if (match(host, DP)) { - passwd = substr(host, RSTART) + passwd = substr(host, RSTART + 1) host = substr(host, 1, RSTART - 1) } else { passwd = "" From 5b8ae33b4e0bad18f8b76b3cb326884dc16b9550 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 15:21:35 +0200 Subject: [PATCH 037/156] [type/__dma_auth] Improve documentation and handle duplicate lines better The state explorer gained a new value "multiple" (it is not used anywhere, just informative). The code will only write a "should" line once and drop duplicate lines. --- type/__dma_auth/explorer/state | 22 ++++++++++++++++------ type/__dma_auth/gencode-remote | 10 +++++++--- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/type/__dma_auth/explorer/state b/type/__dma_auth/explorer/state index 0e6f7be..668b50f 100755 --- a/type/__dma_auth/explorer/state +++ b/type/__dma_auth/explorer/state @@ -17,8 +17,13 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer looks for lines matching the server parameter in dma's auth.conf -# and reports the login and server fields (password is cksummed) +# This explorer looks for a line matching the login and server parameters +# in dma's auth.conf and reports: +# present: a line matching login + host + password exists +# absent: no line matching login + host exists +# different_password: a line exists but with a different pasword +# multiple: multiple lines matching login + host exist +# (should never happen) auth_conf=$("${__type_explorer}/auth_conf") test -r "${auth_conf}" || exit 0 @@ -32,6 +37,7 @@ fi awk -F'\n' -v server="${server}" ' function getvalue(path) { + # Reads the first line of the file located at path and returns it. getline < path close(path) return $0 @@ -42,6 +48,7 @@ BEGIN { parameter_dir = ENVIRON["__object"] "/parameter/" + # Read the parameters of this object host_param = getvalue(parameter_dir "server") if (!host_param) host_param = ENVIRON["__object_id"] login_param = getvalue(parameter_dir "login") @@ -72,10 +79,13 @@ BEGIN { } host == host_param && login == login_param { - if (passwd == passwd_param) - state = "present" - else - state = "different_password" + # a match… + if (state == "absent") { + state = ((passwd == passwd_param) ? "present" : "different_password") + } else { + # report "multiple" to that the type can remove the duplicates. + state = "multiple" + } } END { diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index c49779f..46d9f31 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -78,6 +78,7 @@ test -f "${auth_conf}" || touch "${auth_conf}" awk -F '\n' -v mode=$mode ' function getvalue(path) { + # Reads the first line of the file located at path and returns it. getline < path close(path) return $0 @@ -124,11 +125,12 @@ host == host_param { if (mode) { # state_should == present if (login == login_param && !written) { - # replace line if host and login match + # replace line if host and login match (but only if no line has + # been written already -> no duplicates) print_should() written = 1 - next } + next } else { # state_should == absent if (!login_param || login == login_param) { @@ -139,7 +141,9 @@ host == host_param { } # leave other lines alone -{ print } +{ + print +} END { if (mode && !written) { From 193b1780dee58be94010e06bf3f3e296f7b4b283 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 15:38:34 +0200 Subject: [PATCH 038/156] Improve error message when invalid --state is used. --- type/__dma_auth/gencode-remote | 3 ++- type/__mail_alias/gencode-remote | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 46d9f31..c57e5cc 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -55,7 +55,8 @@ in printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" ;; (*) - printf 'Invalid --state: %s' "${state_should}" >&2 + printf 'Invalid --state: %s.\n' "${state_should}" >&2 + printf 'Acceptable values are: present, absent.\n' >&2 exit 1 ;; esac diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index f4cbf46..22ae89b 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -47,7 +47,8 @@ in mode=0 ;; (*) - printf 'Invalid --state given: %s\n' "$state_should" >&2 + printf 'Invalid --state: %s.\n' "$state_should" >&2 + printf 'Acceptable values are: present, absent.\n' >&2 exit 1 esac From 551348509717b4b44394a59bd11034e6363d5fbd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 17:01:22 +0200 Subject: [PATCH 039/156] [type/__dma] Improve documentation --- type/__dma/explorer/conf | 8 +++- type/__dma/gencode-remote | 96 ++++++++++++++++++++++++++------------- type/__dma/man.rst | 7 ++- 3 files changed, 75 insertions(+), 36 deletions(-) diff --git a/type/__dma/explorer/conf b/type/__dma/explorer/conf index 129e3c3..b4d6d26 100755 --- a/type/__dma/explorer/conf +++ b/type/__dma/explorer/conf @@ -17,8 +17,12 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer looks for lines matching the server parameter in dma's auth.conf -# and reports the login and server fields (password is cksummed) +# This explorer returns a sorted list of "active" (= non-commented) lines +# in the dma.conf file. +# "Trailing" line comments are stripped off. +# +# NOTE: This explorer assumes that the sort(1) utility supports the non-POXIX +# -s (stable sort) option. CONF_PATH=/etc/dma # set in Makefile dma_conf="${CONF_PATH:?}/dma.conf" diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index e4760d8..01537bf 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -20,7 +20,7 @@ else fi -# Generate config +# Generate "should" values for config conf_should=$( if test -s "${__object}/parameter/smarthost" then @@ -60,7 +60,7 @@ conf_should=$( if test -s "${__object}/parameter/port" then printf 'PORT %u\n' "$(cat "${__object}/parameter/port")" - elif test "${default_smtp_port}" -ne 25 + elif test "${default_smtp_port}" -ne 25 # DMA uses port 25 by default then printf 'PORT %u\n' "${default_smtp_port}" fi @@ -93,6 +93,7 @@ conf_should=$( echo 'NULLCLIENT' fi ) +# Sort conf_should to compare against "conf_is" conf_should=$(echo "$conf_should" | sort -s -k 1,1) config_updated=false @@ -100,24 +101,55 @@ if ! echo "$conf_should" | cmp -s "${__object}/explorer/conf" - then # config needs to be updated echo "dma_conf='${CONF_PATH:?}/dma.conf'" + + # The following AWK script will output the new config file to be stored on + # disk. To do so it reads the current dma.conf file and the config options + # that should be set (from stdin). + # Note that the path to the current dma.conf is passed to AWK twice, because + # the new file cannot be generated in one pass. + + # The logic tries to place options at a sensible location, that is: + # a) if the option is already used in the config file: + # group all similar options (e.g. MASQUERADE) at one place in the order + # they are listed in stdin. + # b) if it is a new option and a "default comment" (e.g. "#PORT 25") exists: + # place options grouped directly after the comment (the comment is left + # alone) + # c) otherwise: + # options are grouped by word (the first word in the line) and appended + # at the end of the file. + cat <<'EOF' awk -F '\n' ' -function comment_line(line) { return match(line, /^[ \t]*#+[ \t]*/) } -function empty_line(line) { return match(line, /^[ \t]*$/) } -function is_word(s) { return s ~ /^[A-Z_]+$/ } +function comment_line(line) { + # returns the position in line at which the comment's text starts + # (0 if the line is not a comment) + match(line, /^[ \t]*\#+[ \t]*/) + return RSTART ? (RLENGTH + 1) : 0 +} +function empty_line(line) { return line ~ /^[ \t]*$/ } +function is_word(s) { return s ~ /^[A-Z_]+$/ } # "looks like a plausible word" function first(line, sep) { + # returns the part of the line until sep is found + # (or the whole line if sep is not found) if (!sep) sep = SUBSEP return index(line, sep) ? substr(line, 1, index(line, sep) - 1) : line } function rest(line, sep) { + # returns the part of the line after the first occurrence of sep is found. + # (or nothing if sep is not found) if (!sep) sep = SUBSEP if (index(line, sep)) return substr(line, index(line, sep) + 1) } function conf_pop(word, value) { + # returns the next value for the config `word` and delete it from the list. + # if value is set, this function will only return value if it is the first + # option in the list, otherwise it returns 0. + if (!(word in conf)) return 0 if (!value) { if (index(conf[word], SUBSEP)) # more than one element? @@ -137,12 +169,14 @@ function conf_pop(word, value) { } function print_conf(word, value) { + # print a config line with the given parameters printf "%s", word if (value) printf " %s", value printf "\n" } function print_confs(word, value) { + # print config lines for all values stored in conf[word]. if (!(word in conf)) return if (conf[word]) { while (value = conf_pop(word)) @@ -154,6 +188,7 @@ function print_confs(word, value) { } BEGIN { + # read the "should" state into the `conf` array. while (getline < "/dev/stdin") { word = first($0, " ") if ((word in conf)) @@ -163,11 +198,12 @@ BEGIN { } } -# first pass, gather information +# first pass, gather information about where which information is stored in the +# current config file. This information will be used in the second pass. NR == FNR { if (comment_line($0)) { # comment line - word = first(substr($0, RLENGTH + 1), " ") + word = first(substr($0, comment_line($0) + 1), " ") if (is_word(word)) last_occ["#" word] = FNR } else { word = first($0, " ") @@ -175,19 +211,22 @@ NR == FNR { } } +# before second pass prepare hashes containing location information to be used +# in the second pass. NR > FNR && FNR == 1 { - # before second pass prepare hashes - + # First we drop the locations of commented-out options if a non-commented + # option is available. If a non-commented option is available, we will + # append new config options there to have them all at one place. for (k in last_occ) if (k ~ /^\#/ && (substr(k, 2) in last_occ)) delete last_occ[k] - for (k in last_occ) { - line_map[last_occ[k]] = k - } + # Reverse the option => line mapping. The line_map allows for easier lookups + # in the second pass. + for (k in last_occ) line_map[last_occ[k]] = k } -# second pass, output new config +# second pass, generate and output new config NR > FNR { if (comment_line($0) || empty_line($0)) { # comment or empty line @@ -195,21 +234,24 @@ NR > FNR { if ((FNR in line_map)) { if (line_map[FNR] ~ /^\#/) { - # the "matching" comment line is here + # This line contains a commented config option. If the conf hash + # contains options to be set, we output them here because this + # option is not used in the current config. k = substr(line_map[FNR], 2) if ((k in conf)) print_confs(k) } if (("INSECURE" in conf) && line_map[FNR] ~ /^\#?SECURE$/) { - # INSECURE goes where SECURE comment is + # INSECURE goes where SECURE comment is. print_confs("INSECURE") } } } else { - sub(/[ \t]*\#.*$/, "", $0) # ignore comments word = first($0, " ") + value = rest($0, " ") + sub(/[ \t]*\#.*$/, "", value) # ignore comments in value - if ((word in conf) && rest($0, " ") == first(conf[word])) { + if ((word in conf) && value == first(conf[word])) { # keep config options we want conf_pop(word) print @@ -223,12 +265,13 @@ NR > FNR { } END { - # print rest of config options + # print rest of config options ( for (word in conf) print_confs(word) } ' "${dma_conf}" "${dma_conf}" <<'EOF' >"${dma_conf}.tmp" \ && mv "${dma_conf}.tmp" "${dma_conf}" EOF + # Pass in "conf_should" via stdin echo "${conf_should}" echo 'EOF' @@ -239,20 +282,9 @@ fi if test -f "${__object}/parameter/send-test-email" then - modified=false - - if grep -q '^__mail_alias/root:' "${__messages_in}" - then - modified=true - elif grep -q '^__dma_auth/' "${__messages_in}" - then - modified=true - elif $config_updated - then - modified=true - fi - - if $modified + if grep -q '^__mail_alias/root:' "${__messages_in}" \ + || grep -q '^__dma_auth/' "${__messages_in}" \ + || $config_updated then cat <<-EOF sendmail root < Date: Thu, 11 Jun 2020 18:07:28 +0200 Subject: [PATCH 040/156] [type/__dma] Use EQS to split config lines --- type/__dma/gencode-remote | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 01537bf..cb2cdbe 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -130,19 +130,20 @@ function comment_line(line) { function empty_line(line) { return line ~ /^[ \t]*$/ } function is_word(s) { return s ~ /^[A-Z_]+$/ } # "looks like a plausible word" -function first(line, sep) { +function first(line, sep_re) { # returns the part of the line until sep is found # (or the whole line if sep is not found) - if (!sep) sep = SUBSEP - return index(line, sep) ? substr(line, 1, index(line, sep) - 1) : line + if (!sep_re) sep_re = "[" SUBSEP "]" + match(line, sep_re) + return RSTART ? substr(line, 1, RSTART - 1) : line } -function rest(line, sep) { +function rest(line, sep_re) { # returns the part of the line after the first occurrence of sep is found. # (or nothing if sep is not found) - if (!sep) sep = SUBSEP - if (index(line, sep)) - return substr(line, index(line, sep) + 1) + if (!sep_re) sep_re = "[" SUBSEP "]" + if (match(line, sep_re)) + return substr(line, RSTART + RLENGTH + 1) } function conf_pop(word, value) { @@ -188,13 +189,15 @@ function print_confs(word, value) { } BEGIN { + EQS = /[ \t]/ # copied from dma/conf.c + # read the "should" state into the `conf` array. while (getline < "/dev/stdin") { - word = first($0, " ") + word = first($0, EQS) if ((word in conf)) - conf[word] = conf[word] SUBSEP rest($0, " ") + conf[word] = conf[word] SUBSEP rest($0, EQS) else - conf[word] = rest($0, " ") + conf[word] = rest($0, EQS) } } @@ -203,10 +206,10 @@ BEGIN { NR == FNR { if (comment_line($0)) { # comment line - word = first(substr($0, comment_line($0) + 1), " ") + word = first(substr($0, comment_line($0) + 1), /[ ]/) if (is_word(word)) last_occ["#" word] = FNR } else { - word = first($0, " ") + word = first($0, EQS) if (is_word(word)) last_occ[word] = FNR } } @@ -247,8 +250,8 @@ NR > FNR { } } } else { - word = first($0, " ") - value = rest($0, " ") + word = first($0, EQS) + value = rest($0, EQS) sub(/[ \t]*\#.*$/, "", value) # ignore comments in value if ((word in conf) && value == first(conf[word])) { From 27102340de8bfde2fa03ebf8dfca7157dbe8ea9f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 20:50:42 +0200 Subject: [PATCH 041/156] [type/__mail_alias] Add bug notice about commas --- type/__mail_alias/man.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst index d6c7873..3782ffb 100644 --- a/type/__mail_alias/man.rst +++ b/type/__mail_alias/man.rst @@ -44,6 +44,15 @@ EXAMPLES # Disable redirection of mail for joe __mail_alias joe --state absent + +BUGS +---- +- Quoted strings are not parsed by this type. As a result, email addresses + containing ``,`` (commas) are treated incorrectly (they are treated as two + addresses/aliases.) + Make sure that email addresses do not contain commas. + + SEE ALSO -------- :strong:`aliases`\ (5) From c777a2b1c27ca81f5296c3b4395fa22db74478e1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 21:58:47 +0200 Subject: [PATCH 042/156] [type/__mail_alias] Some fixes in continuation line processing --- type/__mail_alias/explorer/aliases | 25 ++++++++++++++----------- type/__mail_alias/gencode-remote | 25 +++++++++++++------------ 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index 0a8da94..4fffd3b 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -17,13 +17,15 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# Find aliases for a given name and print the aliases line separated +# Find aliases for a given user name and print the aliases (each one on a +# separate line) aliases_file=$("${__type_explorer}/aliases_file") test -r "${aliases_file}" || exit 0 awk -F ':[ \t]*' ' function print_aliases(aliases, matches) { + # prints comma-separated aliases (one per line) split(aliases, matches, /,[ \t]*/) for (i in matches) { gsub(/^[ \t]*|[ \t]*$/, "", matches[i]) @@ -37,15 +39,11 @@ function print_aliases(aliases, matches) { next } -/^[ \t]/ || cont { - # continuation line (either the previous line ended in a backslash or the - # line starts with whitespace) - - if (select) - print_aliases($0) -} - { + # is this line a continuation line? + # (the prev. line ended in a backslash or the line starts with whitespace) + is_cont = /^[ \t]/ || cont + # detect if the line is a line to be continued (ends with a backslash) cont = ($0 ~ /\\$/) @@ -57,9 +55,14 @@ function print_aliases(aliases, matches) { } } -$1 == ENVIRON["__object_id"] && !select { +is_cont { + # if in the alias list of the "target" user, we also print these aliases. + if (select) print_aliases($0) + next +} + +$1 == ENVIRON["__object_id"] { # "target" user -> print alias list - # (only if !select; because of whitespacecontinuation lines) select = 1 print_aliases($2) next diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 22ae89b..cc5fc42 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -69,7 +69,7 @@ test -f "${aliases_file}" || touch "${aliases_file}" awk -F ':[ \t]*' -v mode=$mode ' function sepafter(f, default, _) { # finds the separator between field $f and $(f+1) - _ = substr($0, length($f) + 1, index(substr($0, length($f)+1), $(f+1)) - 1) + _ = substr($0, length($f)+1, index(substr($0, length($f)+1), $(f+1))-1) return _ ? _ : default } @@ -91,29 +91,30 @@ BEGIN { aliases_should_file = (ENVIRON["__object"] "/parameter/alias") } -/^#/ { +/^[ \t]*\#/ { # comment line (leave alone) select = 0; cont = 0 # comments terminate alias lists and continuations print next } -/^[ \t]/ || cont { - # continuation line (either the previous line ended in a backslash or the - # line starts with whitespace) - - # if in the alias list of the "target" user, we drop the line as it has been - # rewritten previously - if (select) next -} - { + # is this line a continuation line? + # (the prev. line ended in a backslash or the line starts with whitespace) + is_cont = /^[ \t]/ || cont + # detect if the line is a line to be continued (ends with a backslash) cont = ($0 ~ /\\$/) # if it is, we drop the backslash from the line. if (cont) sub(/[ \t]*\\$/, "", $0) } +is_cont { + # we ignore the line as it has been rewritten previously or is not + # interesting + next +} + $1 == ENVIRON["__object_id"] { # "target" user -> rewrite aliases list select = 1 @@ -139,7 +140,7 @@ END { if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" then - # aliases file was modified, replace and run `newaliases` + # aliases file was modified, replace and run `newaliases`. mv "${aliases_file}.tmp" "${aliases_file}" newaliases else From 0f81b89f709e26a98a369d6e8b0797eb3ca85909 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 22 Jun 2020 13:29:28 +0200 Subject: [PATCH 043/156] [type/__dma] Make --smarthost optional --- type/__dma/parameter/optional | 1 + type/__dma/parameter/required | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 type/__dma/parameter/required diff --git a/type/__dma/parameter/optional b/type/__dma/parameter/optional index 3f6cb5d..615c189 100644 --- a/type/__dma/parameter/optional +++ b/type/__dma/parameter/optional @@ -1,3 +1,4 @@ mailname port security +smarthost diff --git a/type/__dma/parameter/required b/type/__dma/parameter/required deleted file mode 100644 index 0753fb6..0000000 --- a/type/__dma/parameter/required +++ /dev/null @@ -1 +0,0 @@ -smarthost From 27b832f2127fc0b475a6390cacb9219b9516328f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 22 Jun 2020 14:02:13 +0200 Subject: [PATCH 044/156] [type/__dma] Add support for Alpine Linux requires the testing repository, currently. --- type/__dma/gencode-remote | 8 ++++---- type/__dma/manifest | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index cb2cdbe..a6aca0d 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -122,7 +122,7 @@ then cat <<'EOF' awk -F '\n' ' function comment_line(line) { - # returns the position in line at which the comment's text starts + # returns the position in line at which the comment'\''s text starts # (0 if the line is not a comment) match(line, /^[ \t]*\#+[ \t]*/) return RSTART ? (RLENGTH + 1) : 0 @@ -143,7 +143,7 @@ function rest(line, sep_re) { # (or nothing if sep is not found) if (!sep_re) sep_re = "[" SUBSEP "]" if (match(line, sep_re)) - return substr(line, RSTART + RLENGTH + 1) + return substr(line, RSTART + RLENGTH) } function conf_pop(word, value) { @@ -189,7 +189,7 @@ function print_confs(word, value) { } BEGIN { - EQS = /[ \t]/ # copied from dma/conf.c + EQS = "[ \t]" # copied from dma/conf.c # read the "should" state into the `conf` array. while (getline < "/dev/stdin") { @@ -206,7 +206,7 @@ BEGIN { NR == FNR { if (comment_line($0)) { # comment line - word = first(substr($0, comment_line($0) + 1), /[ ]/) + word = first(substr($0, comment_line($0)), " ") if (is_word(word)) last_occ["#" word] = FNR } else { word = first($0, EQS) diff --git a/type/__dma/manifest b/type/__dma/manifest index 7abd7c8..75e42d7 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -5,6 +5,10 @@ os=$(cat "${__global}/explorer/os") # Install DMA case $os in + (alpine) + __package dma --state present + export require='__package/dma' + ;; (debian|devuan|ubuntu) __package dma --state present export require='__package/dma' From aa605cada45d5a91da5c4153fa822d95a8214d80 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 22 Jun 2020 14:02:42 +0200 Subject: [PATCH 045/156] [type/__mail_aliases] Add support for Alpine Linux Alpine's DMA package has a typo and installs "newailases" instead of "newaliases". I adjusted the code-remote to only run newaliases if it is available. Otherwise, tough luck, user gotta either fix his system or run manually. --- type/__mail_alias/explorer/aliases_file | 4 ++-- type/__mail_alias/gencode-remote | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file index 2710792..90bccde 100755 --- a/type/__mail_alias/explorer/aliases_file +++ b/type/__mail_alias/explorer/aliases_file @@ -28,7 +28,7 @@ check_file() { fi } -case $("$__explorer/os") +case $("${__explorer}/os") in (freebsd|openbsd|solaris) check_file /etc/mail/aliases @@ -36,7 +36,7 @@ in # default found /etc/mail/aliases ;; - (debian|devuan|ubuntu) + (alpine|debian|devuan|ubuntu) check_file /etc/aliases # default diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index cc5fc42..3eaad75 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -142,7 +142,9 @@ if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" then # aliases file was modified, replace and run `newaliases`. mv "${aliases_file}.tmp" "${aliases_file}" - newaliases + + # run newaliases if present + command -v newaliases >/dev/null 2>&1 && newaliases || true else # no modifications were made, delete the temp file. rm "${aliases_file}.tmp" From f6a36a60c0f92ae212c2e1274cba64ffdc0c856c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 25 Jun 2020 11:15:23 +0200 Subject: [PATCH 046/156] Import __matrix_riot from ungleich's dot-cdist --- type/__matrix_riot/files/config.json.sh | 90 +++++++++++++++ type/__matrix_riot/gencode-remote | 73 +++++++++++++ type/__matrix_riot/man.rst | 87 +++++++++++++++ type/__matrix_riot/manifest | 103 ++++++++++++++++++ type/__matrix_riot/parameter/boolean | 1 + type/__matrix_riot/parameter/default/brand | 1 + .../parameter/default/cookie_policy_url | 1 + .../parameter/default/default_country_code | 1 + .../parameter/default/default_server_name | 1 + .../parameter/default/default_server_url | 1 + type/__matrix_riot/parameter/default/owner | 1 + .../parameter/default/privacy_policy_url | 1 + .../parameter/default/room_directory_servers | 1 + type/__matrix_riot/parameter/optional | 13 +++ .../__matrix_riot/parameter/optional_multiple | 1 + type/__matrix_riot/parameter/required | 2 + 16 files changed, 378 insertions(+) create mode 100755 type/__matrix_riot/files/config.json.sh create mode 100755 type/__matrix_riot/gencode-remote create mode 100644 type/__matrix_riot/man.rst create mode 100755 type/__matrix_riot/manifest create mode 100644 type/__matrix_riot/parameter/boolean create mode 100644 type/__matrix_riot/parameter/default/brand create mode 100644 type/__matrix_riot/parameter/default/cookie_policy_url create mode 100644 type/__matrix_riot/parameter/default/default_country_code create mode 100644 type/__matrix_riot/parameter/default/default_server_name create mode 100644 type/__matrix_riot/parameter/default/default_server_url create mode 100644 type/__matrix_riot/parameter/default/owner create mode 100644 type/__matrix_riot/parameter/default/privacy_policy_url create mode 100644 type/__matrix_riot/parameter/default/room_directory_servers create mode 100644 type/__matrix_riot/parameter/optional create mode 100644 type/__matrix_riot/parameter/optional_multiple create mode 100644 type/__matrix_riot/parameter/required diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_riot/files/config.json.sh new file mode 100755 index 0000000..e7fca72 --- /dev/null +++ b/type/__matrix_riot/files/config.json.sh @@ -0,0 +1,90 @@ +#!/bin/sh +# +# Upstream configuration guide/documentation: +# https://github.com/vector-im/riot-web/blob/develop/docs/config.md + +generate_embedded_pages () { + if [ $EMBED_HOMEPAGE ]; then + cat << EOF + "embeddedPages": { + "homeUrl": "home.html" + }, +EOF + fi +} + +generate_jitsi_config () { + if [ "$JITSI_DOMAIN" != "" ]; then + cat << EOF + "jitsi": { + "preferredDomain": "$JITSI_DOMAIN" + }, +EOF + fi +} + +generate_branding () { + echo '"branding": {' + + if [ "$BRANDING_AUTH_HEADER_LOGO_URL" != "" ]; then + cat << EOF + "authHeaderLogoUrl": "$BRANDING_AUTH_HEADER_LOGO_URL", +EOF + fi + + if [ "$BRANDING_AUTH_FOOTER_LINKS" != "" ]; then + cat << EOF + "authFooterLinks": "$BRANDING_AUTH_FOOTER_LINKS", +EOF + fi + + cat << EOF + "welcomeBackgroundUrl": "themes/riot/img/backgrounds/valley.jpg" +EOF + echo '},' +} + +cat << EOF +{ + "default_server_config": { + "m.homeserver": { + "base_url": "$DEFAULT_SERVER_URL", + "server_name": "$DEFAULT_SERVER_NAME" + }, + "m.identity_server": { + "base_url": "https://vector.im" + } + }, + "brand": "$BRAND", + $(generate_branding) + "defaultCountryCode": "$DEFAULT_COUNTRY_CODE", + "integrations_ui_url": "https://scalar.vector.im/", + "integrations_rest_url": "https://scalar.vector.im/api", + "integrations_widgets_urls": [ + "https://scalar.vector.im/_matrix/integrations/v1", + "https://scalar.vector.im/api", + "https://scalar-staging.vector.im/_matrix/integrations/v1", + "https://scalar-staging.vector.im/api", + "https://scalar-staging.riot.im/scalar/api" + ], + "bug_report_endpoint_url": "https://riot.im/bugreports/submit", + "roomDirectory": { + "servers": [ + $ROOM_DIRECTORY_SERVERS + ] + }, + "disable_custom_urls": "$DISABLE_CUSTOM_URLS", + $(generate_embedded_pages) + $(generate_jitsi_config) + "terms_and_conditions_links": [ + { + "url": "$PRIVACY_POLICY_URL", + "text": "Privacy Policy" + }, + { + "url": "$COOKIE_POLICY_URL", + "text": "Cookie Policy" + } + ] +} +EOF diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_riot/gencode-remote new file mode 100755 index 0000000..97c4777 --- /dev/null +++ b/type/__matrix_riot/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh -e +# +# 2019 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +VERSION=$(cat "$__object/parameter/version") +INSTALL_DIR=$(cat "$__object/parameter/install_dir") +OWNER=$(cat "$__object/parameter/owner") + +src="riot-v$VERSION" +archive="$src.tar.gz" +config='config.json' +homepage='home.html' +welcomepage='welcome.html' +welcomelogo='welcome-logo.svg' +url="https://github.com/vector-im/riot-web/releases/download/v$VERSION/$archive" + +# tar and curl are installed by the __matrix-riot manifest. mktemp is usually +# provided by coreutils and assumed installed. +cat << EOF +set -e + +# Ensure that coreutils is installed. +if [ ! -x \$(which mktemp) ]; then + echo "mktemp is not available on the remote host." >&2 + exit 1 +fi + +# Create temporary working directory. +tmpdir=\$(mktemp -d) +custom_files_dir="\$tmpdir/custom_files" +cd \$tmpdir + +# Download and extract sources. +curl -L '$url' > $archive +tar xf $archive + +# Backup files deployed by __matrix_riot. +mkdir -p \$custom_files_dir +for file in $INSTALL_DIR/cdist/*; do + cp "\$file" "\$custom_files_dir" +done + +# Deploy sources and restore configuration. +rm -r '$INSTALL_DIR' +mv '$src' '$INSTALL_DIR' + +for file in \$custom_files_dir/*; do + cp "\$file" '$INSTALL_DIR' +done + +# Chown deployed files to requested owner. +chown -R '$OWNER' '$INSTALL_DIR' + +# Remove temporary working directory. +cd / +rm -r \$tmpdir +EOF diff --git a/type/__matrix_riot/man.rst b/type/__matrix_riot/man.rst new file mode 100644 index 0000000..2fe8eae --- /dev/null +++ b/type/__matrix_riot/man.rst @@ -0,0 +1,87 @@ +cdist-type__matrix_riot(7) +====================== + +NAME +---- +cdist-type__matrix_riot - Install and configure Riot, a web Matrix client. + + +DESCRIPTION +----------- +This type install and configure the Riot web client. + + +REQUIRED PARAMETERS +------------------- +install_dir + Root directory of Riot's static files. + +version + Release of Riot to install. + +OPTIONAL PARAMETERS +------------------- +default_server_name + Name of matrix homeserver to connect to, defaults to 'matrix.org'. + +default_server_url + URL of matrix homeserver to connect to, defaults to 'https://matrix-client.matrix.org'. + +owner + Owner of the deployed files, passed to `chown`. Defaults to 'root'. + +brand + Web UI branding, defaults to 'Riot'. + +default_country_code + ISO 3166 alpha2 country code to use when showing country selectors, such as + phone number inputs. Defaults to GB. + +privacy_policy_url + Defaults to 'https://riot.im/privacy'. + +cookie_policy_url + Defaults to 'https://matrix.org/docs/guides/riot_im_cookie_policy'. + +jitsi_domain + Domain name of preferred Jitsi instance (default is jitsi.riot.im). This is + used whenever a user clicks on the voice/video call buttons. + +homepage + Path to custom homepage, displayed once logged in. + +welcomepage + Path to custom welcome (= login) page. + +custom_asset + Serve a file a the top-level directory (e.g. /my-custom-logo.svg). Can be specified multiple times. + +BOOLEAN PARAMETERS +------------------- +disable_custom_urls + Disallow the user to change the default homeserver when signing up or logging in. + +EXAMPLES +-------- + +.. code-block:: sh + + __matrix_riot my-riot --install_dir /var/www/riot-web --version 1.5.6 + + +SEE ALSO +-------- +- `cdist-type__matrix_synapse(7) `_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2019 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__matrix_riot/manifest b/type/__matrix_riot/manifest new file mode 100755 index 0000000..a843f4b --- /dev/null +++ b/type/__matrix_riot/manifest @@ -0,0 +1,103 @@ +#!/bin/sh -e +# +# 2019 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +INSTALL_DIR=$(cat "$__object/parameter/install_dir") + +export DEFAULT_SERVER_NAME=$(cat "$__object/parameter/default_server_name") +export DEFAULT_SERVER_URL=$(cat "$__object/parameter/default_server_url") +export BRAND=$(cat "$__object/parameter/brand") +export DEFAULT_COUNTRY_CODE=$(cat "$__object/parameter/default_country_code") +export ROOM_DIRECTORY_SERVERS=$(cat "$__object/parameter/room_directory_servers") +export PRIVACY_POLICY_URL=$(cat "$__object/parameter/privacy_policy_url") +export COOKIE_POLICY_URL=$(cat "$__object/parameter/cookie_policy_url") + +if [ -f "$__object/parameter/jitsi_domain" ]; then + export JITSI_DOMAIN=$(cat "$__object/parameter/jitsi_domain") +fi + +if [ -f "$__object/parameter/branding_auth_header_logo_url" ]; then + export BRANDING_AUTH_HEADER_LOGO_URL=$(cat "$__object/parameter/branding_auth_header_logo_url") +fi + +if [ -f "$__object/parameter/branding_auth_footer_links" ]; then + export BRANDING_AUTH_FOOTER_LINKS=$(cat "$__object/parameter/branding_auth_footer_links") +fi + +if [ -f "$__object/parameter/homepage" ]; then + export EMBED_HOMEPAGE=1 + homepage=$(cat "$__object/parameter/homepage") +fi + +if [ -f "$__object/parameter/welcomepage" ]; then + export EMBED_WELCOMEPAGE=1 + welcomepage=$(cat "$__object/parameter/welcomepage") +fi + +if [ -f "$__object/parameter/custom_asset" ]; then + for file in $(cat "$__object/parameter/custom_asset"); do + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename $file)" \ + --source "$file" \ + --mode 0664 \ + --state present + done +fi + +if [ -f "$__object/parameter/disable_custom_urls" ]; then + export DISABLE_CUSTOM_URLS='true' +else + export DISABLE_CUSTOM_URLS='false' +fi + +# Owner of the uploaded files. +owner=$(cat "$__object/parameter/owner") + +# Ensure that curl and tar are installed, as they will be required by the +# gencode-remote script. +__package curl --state present +__package tar --state present + +# Generate and deploy configuration file. +mkdir -p "$__object/files" +"$__type/files/config.json.sh" > "$__object/files/config.json" + +# Install the config.json configuration file. The application's sources are +# downloaded and deployed by gencode-remote. +__directory "$INSTALL_DIR/cdist" \ + --owner "$owner" --mode 0755 --parents \ + --state present + +require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/config.json" \ + --source "$__object/files/config.json" \ + --mode 0664 \ + --state present + +if [ $EMBED_HOMEPAGE ]; then + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/home.html" \ + --source "$homepage" \ + --mode 0664 \ + --state present +fi + +if [ $EMBED_WELCOMEPAGE ]; then + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/welcome.html" \ + --source "$welcomepage" \ + --mode 0664 \ + --state present +fi diff --git a/type/__matrix_riot/parameter/boolean b/type/__matrix_riot/parameter/boolean new file mode 100644 index 0000000..4d77768 --- /dev/null +++ b/type/__matrix_riot/parameter/boolean @@ -0,0 +1 @@ +disable_custom_urls diff --git a/type/__matrix_riot/parameter/default/brand b/type/__matrix_riot/parameter/default/brand new file mode 100644 index 0000000..e8095bb --- /dev/null +++ b/type/__matrix_riot/parameter/default/brand @@ -0,0 +1 @@ +Riot diff --git a/type/__matrix_riot/parameter/default/cookie_policy_url b/type/__matrix_riot/parameter/default/cookie_policy_url new file mode 100644 index 0000000..04e9c2b --- /dev/null +++ b/type/__matrix_riot/parameter/default/cookie_policy_url @@ -0,0 +1 @@ +https://matrix.org/docs/guides/riot_im_cookie_policy diff --git a/type/__matrix_riot/parameter/default/default_country_code b/type/__matrix_riot/parameter/default/default_country_code new file mode 100644 index 0000000..30ac4a3 --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_country_code @@ -0,0 +1 @@ +GB diff --git a/type/__matrix_riot/parameter/default/default_server_name b/type/__matrix_riot/parameter/default/default_server_name new file mode 100644 index 0000000..5528ffd --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_server_name @@ -0,0 +1 @@ +matrix.org diff --git a/type/__matrix_riot/parameter/default/default_server_url b/type/__matrix_riot/parameter/default/default_server_url new file mode 100644 index 0000000..2cb9227 --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_server_url @@ -0,0 +1 @@ +https://matrix-client.matrix.org diff --git a/type/__matrix_riot/parameter/default/owner b/type/__matrix_riot/parameter/default/owner new file mode 100644 index 0000000..d8649da --- /dev/null +++ b/type/__matrix_riot/parameter/default/owner @@ -0,0 +1 @@ +root diff --git a/type/__matrix_riot/parameter/default/privacy_policy_url b/type/__matrix_riot/parameter/default/privacy_policy_url new file mode 100644 index 0000000..4cdd12c --- /dev/null +++ b/type/__matrix_riot/parameter/default/privacy_policy_url @@ -0,0 +1 @@ +https://riot.im/privacy diff --git a/type/__matrix_riot/parameter/default/room_directory_servers b/type/__matrix_riot/parameter/default/room_directory_servers new file mode 100644 index 0000000..4ea73ad --- /dev/null +++ b/type/__matrix_riot/parameter/default/room_directory_servers @@ -0,0 +1 @@ +"matrix.org" diff --git a/type/__matrix_riot/parameter/optional b/type/__matrix_riot/parameter/optional new file mode 100644 index 0000000..21a2faf --- /dev/null +++ b/type/__matrix_riot/parameter/optional @@ -0,0 +1,13 @@ +default_server_url +default_server_name +brand +default_country_code +privacy_policy_url +cookie_policy_url +room_directory_servers +owner +homepage +welcomepage +jitsi_domain +branding_auth_header_logo_url +branding_auth_footer_links diff --git a/type/__matrix_riot/parameter/optional_multiple b/type/__matrix_riot/parameter/optional_multiple new file mode 100644 index 0000000..4c2ca54 --- /dev/null +++ b/type/__matrix_riot/parameter/optional_multiple @@ -0,0 +1 @@ +custom_asset diff --git a/type/__matrix_riot/parameter/required b/type/__matrix_riot/parameter/required new file mode 100644 index 0000000..a76477e --- /dev/null +++ b/type/__matrix_riot/parameter/required @@ -0,0 +1,2 @@ +version +install_dir From 43c59985d0aba80ae6b41a56143e3247423c23a5 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 25 Jun 2020 18:07:51 +0200 Subject: [PATCH 047/156] [type/__mail_alias] Fallback to /etc/aliases instead of /etc/mail/aliases --- type/__mail_alias/explorer/aliases_file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file index 90bccde..a59bb99 100755 --- a/type/__mail_alias/explorer/aliases_file +++ b/type/__mail_alias/explorer/aliases_file @@ -47,6 +47,6 @@ in check_file /etc/aliases # default - found /etc/mail/aliases + found /etc/aliases ;; esac From 331461219387b733d7978171aa0947954ecedbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 11 Jul 2020 09:21:37 +0200 Subject: [PATCH 048/156] __matrix_riot: improve friendship with shellcheck --- type/__matrix_riot/files/config.json.sh | 2 +- type/__matrix_riot/gencode-remote | 4 ---- type/__matrix_riot/manifest | 9 ++++++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_riot/files/config.json.sh index e7fca72..6cd1648 100755 --- a/type/__matrix_riot/files/config.json.sh +++ b/type/__matrix_riot/files/config.json.sh @@ -4,7 +4,7 @@ # https://github.com/vector-im/riot-web/blob/develop/docs/config.md generate_embedded_pages () { - if [ $EMBED_HOMEPAGE ]; then + if [ "$EMBED_HOMEPAGE" != "" ]; then cat << EOF "embeddedPages": { "homeUrl": "home.html" diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_riot/gencode-remote index 97c4777..dbc886c 100755 --- a/type/__matrix_riot/gencode-remote +++ b/type/__matrix_riot/gencode-remote @@ -24,10 +24,6 @@ OWNER=$(cat "$__object/parameter/owner") src="riot-v$VERSION" archive="$src.tar.gz" -config='config.json' -homepage='home.html' -welcomepage='welcome.html' -welcomelogo='welcome-logo.svg' url="https://github.com/vector-im/riot-web/releases/download/v$VERSION/$archive" # tar and curl are installed by the __matrix-riot manifest. mktemp is usually diff --git a/type/__matrix_riot/manifest b/type/__matrix_riot/manifest index a843f4b..544bd96 100755 --- a/type/__matrix_riot/manifest +++ b/type/__matrix_riot/manifest @@ -16,7 +16,10 @@ # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -# + +# Ignore "Declare and assign separately to avoid masking return values. [SC2155]" +# => not relevant for the type arguments. +# shellcheck disable=SC2155 INSTALL_DIR=$(cat "$__object/parameter/install_dir") @@ -51,8 +54,8 @@ if [ -f "$__object/parameter/welcomepage" ]; then fi if [ -f "$__object/parameter/custom_asset" ]; then - for file in $(cat "$__object/parameter/custom_asset"); do - require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename $file)" \ + "$__object/parameter/custom_asset" | while IFS= read -r file; do + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename "$file")" \ --source "$file" \ --mode 0664 \ --state present From 70047d10b1878e46a53c26a2bc8c50779c89f0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 20 Jul 2020 17:20:33 +0200 Subject: [PATCH 049/156] Initial __netbox type --- type/__netbox/files/configuration.py.sh | 258 ++++++++++++++++++++++++ type/__netbox/files/gunicorn.py | 16 ++ type/__netbox/files/netbox-rq.service | 21 ++ type/__netbox/files/netbox.service | 22 ++ type/__netbox/gencode-remote | 58 ++++++ type/__netbox/manifest | 47 +++++ type/__netbox/parameter/required | 5 + type/__netbox/singleton | 0 8 files changed, 427 insertions(+) create mode 100755 type/__netbox/files/configuration.py.sh create mode 100644 type/__netbox/files/gunicorn.py create mode 100644 type/__netbox/files/netbox-rq.service create mode 100644 type/__netbox/files/netbox.service create mode 100755 type/__netbox/gencode-remote create mode 100644 type/__netbox/manifest create mode 100644 type/__netbox/parameter/required create mode 100644 type/__netbox/singleton diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh new file mode 100755 index 0000000..b7e1597 --- /dev/null +++ b/type/__netbox/files/configuration.py.sh @@ -0,0 +1,258 @@ +#!/bin/sh + +cat << EOF +######################### +# # +# Required settings # +# # +######################### + +# This is a list of valid fully-qualified domain names (FQDNs) for the NetBox server. NetBox will not permit write +# access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. +# +# Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] +ALLOWED_HOSTS = [ '$ALLOWED_HOST' ] + +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases +DATABASE = { + 'NAME': '$DATABASE_NAME', # Database name + 'USER': '$DATABASE_USER', # PostgreSQL username + 'PASSWORD': '$DATABASE_PASSWORD', # PostgreSQL password + 'HOST': 'localhost', # Database server + 'PORT': '', # Database port (leave blank for default) + 'CONN_MAX_AGE': 300, # Max database connection age +} + +# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended +# to use two separate database IDs. +REDIS = { + 'tasks': { + 'HOST': 'localhost', + 'PORT': 6379, + # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} + +# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. +# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and +# symbols. NetBox will not run without this defined. For more information, see +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = '$SECRET_KEY' + + +######################### +# # +# Optional settings # +# # +######################### + +# Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +# application errors (assuming correct email settings are provided). +ADMINS = [ + # ['John Doe', 'jdoe@example.com'], +] + +# URL schemes that are allowed within links in NetBox +ALLOWED_URL_SCHEMES = ( + 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +) + +# Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same +# content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. +BANNER_TOP = '' +BANNER_BOTTOM = '' + +# Text to include on the login page above the login form. HTML is allowed. +BANNER_LOGIN = '' + +# Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: +# BASE_PATH = 'netbox/' +BASE_PATH = '' + +# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) +CACHE_TIMEOUT = 900 + +# Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) +CHANGELOG_RETENTION = 90 + +# API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be +# allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or +# CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers +CORS_ORIGIN_ALLOW_ALL = False +CORS_ORIGIN_WHITELIST = [ + # 'https://hostname.example.com', +] +CORS_ORIGIN_REGEX_WHITELIST = [ + # r'^(https?://)?(\w+\.)?example\.com$', +] + +# Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal +# sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging +# on a production system. +DEBUG = False + +# Email settings +EMAIL = { + 'SERVER': 'localhost', + 'PORT': 25, + 'USERNAME': '', + 'PASSWORD': '', + 'USE_SSL': False, + 'USE_TLS': False, + 'TIMEOUT': 10, # seconds + 'FROM_EMAIL': '', +} + +# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table +# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True. +ENFORCE_GLOBAL_UNIQUE = False + +# Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and +# by anonymous users. List models in the form \`.\`. Add '*' to this list to exempt all models. +EXEMPT_VIEW_PERMISSIONS = [ + # 'dcim.site', + # 'dcim.region', + # 'ipam.prefix', +] + +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +# HTTP_PROXIES = { +# 'http': 'http://10.10.1.10:3128', +# 'https': 'http://10.10.1.10:1080', +# } + +# IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing +# NetBox from an internal IP. +INTERNAL_IPS = ('127.0.0.1', '::1') + +# Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: +# https://docs.djangoproject.com/en/stable/topics/logging/ +LOGGING = {} + +# Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users +# are permitted to access most data in NetBox (excluding secrets) but not make any changes. +LOGIN_REQUIRED = False + +# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to +# re-authenticate. (Default: 1209600 [14 days]) +LOGIN_TIMEOUT = None + +# Setting this to True will display a "maintenance mode" banner at the top of every page. +MAINTENANCE_MODE = False + +# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. +# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request +# all objects by specifying "?limit=0". +MAX_PAGE_SIZE = 1000 + +# The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that +# the default value of this setting is derived from the installed location. +# MEDIA_ROOT = '/opt/netbox/netbox/media' + +# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } + +# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' +METRICS_ENABLED = False + +# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM. +NAPALM_USERNAME = '' +NAPALM_PASSWORD = '' + +# NAPALM timeout (in seconds). (Default: 30) +NAPALM_TIMEOUT = 30 + +# NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +# be provided as a dictionary. +NAPALM_ARGS = {} + +# Determine how many objects to display per page within a list. (Default: 50) +PAGINATE_COUNT = 50 + +# Enable installed plugins. Add the name of each plugin to the list. +PLUGINS = [] + +# Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# PLUGINS_CONFIG = { +# 'my_plugin': { +# 'foo': 'bar', +# 'buzz': 'bazz' +# } +# } + +# When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to +# prefer IPv4 instead. +PREFER_IPV4 = False + +# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1. +RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = 22 +RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 + +# Remote authentication support +REMOTE_AUTH_ENABLED = False +REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' +REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' +REMOTE_AUTH_AUTO_CREATE_USER = True +REMOTE_AUTH_DEFAULT_GROUPS = [] +REMOTE_AUTH_DEFAULT_PERMISSIONS = [] + +# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. +RELEASE_CHECK_TIMEOUT = 24 * 3600 + +# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the +# version check or use the URL below to check for release in the official NetBox repository. +RELEASE_CHECK_URL = None +# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +# The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +# REPORTS_ROOT = '/opt/netbox/netbox/reports' + +# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +# SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' + +# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use +# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only +# database access.) Note that the user as which NetBox runs must have read and write permissions to this path. +SESSION_FILE_PATH = None + +# Time zone (default: UTC) +TIME_ZONE = 'UTC' + +# Date/time formatting. See the following link for supported formats: +# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date +DATE_FORMAT = 'N j, Y' +SHORT_DATE_FORMAT = 'Y-m-d' +TIME_FORMAT = 'g:i a' +SHORT_TIME_FORMAT = 'H:i:s' +DATETIME_FORMAT = 'N j, Y g:i a' +SHORT_DATETIME_FORMAT = 'Y-m-d H:i' +EOF diff --git a/type/__netbox/files/gunicorn.py b/type/__netbox/files/gunicorn.py new file mode 100644 index 0000000..c582677 --- /dev/null +++ b/type/__netbox/files/gunicorn.py @@ -0,0 +1,16 @@ +# The IP address (typically localhost) and port that the Netbox WSGI process should listen on +bind = '127.0.0.1:8001' + +# Number of gunicorn workers to spawn. This should typically be 2n+1, where +# n is the number of CPU cores present. +workers = 3 + +# Number of threads per worker process +threads = 3 + +# Timeout (in seconds) for a request to complete +timeout = 120 + +# The maximum number of requests a worker can handle before being respawned +max_requests = 5000 +max_requests_jitter = 500 diff --git a/type/__netbox/files/netbox-rq.service b/type/__netbox/files/netbox-rq.service new file mode 100644 index 0000000..77d7091 --- /dev/null +++ b/type/__netbox/files/netbox-rq.service @@ -0,0 +1,21 @@ +[Unit] +Description=NetBox Request Queue Worker +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service new file mode 100644 index 0000000..18eb045 --- /dev/null +++ b/type/__netbox/files/netbox.service @@ -0,0 +1,22 @@ +[Unit] +Description=NetBox WSGI Service +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +PIDFile=/var/tmp/netbox.pid +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote new file mode 100755 index 0000000..a52b110 --- /dev/null +++ b/type/__netbox/gencode-remote @@ -0,0 +1,58 @@ +#!/bin/sh + +VERSION=$(cat "$__object/parameter/version") + +src="netbox-$VERSION" +archive="v$VERSION.tar.gz" +url="https://github.com/netbox-community/netbox/archive/$archive" +install_dir=/opt/netbox/netbox + +cat << EOF +set -e + +# Ensure that coreutils is installed. +if [ ! -x \$(which mktemp) ]; then + echo "mktemp is not available on the remote host." >&2 + exit 1 +fi + +# Create temporary working directory. +tmpdir=\$(mktemp -d) +cd "\$tmpdir" + +# Download and extract sources. +curl -L '$url' > '$archive' +tar xf '$archive' + +# Save cdist-upload configuration file. +mkdir -p "\$tmpdir" +cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" + +# Deploy sources and restore configuration. +rm -r '$install_dir' +cp -r '$src/netbox' '$install_dir' +cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' + +# Setup & enter python virtualenv. +virtualenv /opt/netbox/venv + +# Install python dependencies. +/opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" + +# Set final permissions. +chown -R netbox /opt/netbox + +# Run database migrations. +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate + +# Generate static assets. +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collectstatic --no-input + +# Remove temporary working directory. +cd / +rm -r "\$tmpdir" + +# Restart services. +service netbox restart +service netbox-rq restart +EOF diff --git a/type/__netbox/manifest b/type/__netbox/manifest new file mode 100644 index 0000000..64a0a55 --- /dev/null +++ b/type/__netbox/manifest @@ -0,0 +1,47 @@ +#!/bin/sh + +os=$(cat "$__global/explorer/os") + +case "$os" in + debian|ubuntu) + # Install netbox dependencies. + for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv; do + __package $pkg + done + + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +export DATABASE_NAME=$(cat "$__object/parameter/database") +export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") +export ALLOWED_HOST=$(cat "$__object/parameter/host") +export SECRET_KEY=$(cat "$__object/parameter/secret-key") + +# Create system user used to run netbox. +__user netbox --system --home /opt/netbox --create-home + +# Generate and upload netbox configuration. +mkdir -p "$__object/files" +"$__type/files/configuration.py.sh" > "$__object/files/configuration.py" + +require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents +require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ + --source "$__object/files/configuration.py" + +# Upload systemd units and gunicorn configuration. +for unit in netbox netbox-rq; do + __file /etc/systemd/system/$unit.service \ + --mode 644 --source "$__type/files/$unit.service" \ + --onchange 'systemctl daemon-reload' +done + +# Python worker configuration. +require="__user/netbox" __file /opt/netbox/gunicorn.py \ + --mode 644 --source "$__type/files/gunicorn.py" diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required new file mode 100644 index 0000000..388048a --- /dev/null +++ b/type/__netbox/parameter/required @@ -0,0 +1,5 @@ +version +database +database-password +secret-key +host diff --git a/type/__netbox/singleton b/type/__netbox/singleton new file mode 100644 index 0000000..e69de29 From ab2c826cc3347cc3ff2842480a20dce185cc89d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jul 2020 07:46:45 +0200 Subject: [PATCH 050/156] __netbox: add minimal LDAP support --- type/__netbox/files/ldap_config.py.sh | 27 +++++++++++++++++++++++++ type/__netbox/gencode-remote | 8 ++++++++ type/__netbox/manifest | 29 +++++++++++++++++++++++++++ type/__netbox/parameter/optional | 4 ++++ 4 files changed, 68 insertions(+) create mode 100755 type/__netbox/files/ldap_config.py.sh create mode 100644 type/__netbox/parameter/optional diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh new file mode 100755 index 0000000..88af4b9 --- /dev/null +++ b/type/__netbox/files/ldap_config.py.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +cat << EOF +############################## +# LDAP-backed authentication # +############################## + +import ldap +from django_auth_ldap.config import LDAPSearch + +# Server URI +AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" + +# Set the DN and password for the NetBox service account. +AUTH_LDAP_BIND_DN = "$LDAP_BIND_DN" +AUTH_LDAP_BIND_PASSWORD = "$LDAP_BIND_PASSWORD" + +# If a user's DN is producible from their username, we don't need to search. +AUTH_LDAP_USER_DN_TEMPLATE = "$LDAP_USER_DN_TEMPLATE" + +# You can map user attributes to Django attributes as so. +AUTH_LDAP_USER_ATTR_MAP = { + "first_name": "givenName", + "last_name": "sn", + "email": "mail" +} +EOF diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index a52b110..e3157a9 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -27,18 +27,26 @@ tar xf '$archive' # Save cdist-upload configuration file. mkdir -p "\$tmpdir" cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" +cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' +cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. virtualenv /opt/netbox/venv # Install python dependencies. /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" +EOF +if [ -f "$__object/parameter/ldap-server" ]; then + echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" +fi + +cat << EOF # Set final permissions. chown -R netbox /opt/netbox diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 64a0a55..2976686 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -10,6 +10,11 @@ case "$os" in __package $pkg done + if [ -f "$__object/parameter/ldap-server" ]; then + for pkg in libldap2-dev libsasl2-dev libssl-dev; do + __package $pkg + done + fi ;; *) printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 @@ -23,18 +28,42 @@ export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") export ALLOWED_HOST=$(cat "$__object/parameter/host") export SECRET_KEY=$(cat "$__object/parameter/secret-key") +if [ -f "$__object/parameter/ldap-server" ]; then + export LDAP_SERVER=$(cat "$__object/parameter/ldap-server") +fi + +if [ -f "$__object/parameter/ldap-bind-dn" ]; then + export LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") +fi + +if [ -f "$__object/parameter/ldap-bind-password" ]; then + export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") +fi + +if [ -f "$__object/parameter/ldap-user-dn-template" ]; then + export LDAP_USER_DN_TEMPLATE=$(cat "$__object/parameter/ldap-user-dn-template") +fi + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home # Generate and upload netbox configuration. mkdir -p "$__object/files" "$__type/files/configuration.py.sh" > "$__object/files/configuration.py" +"$__type/files/ldap_config.py.sh" > "$__object/files/ldap_config.py" require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents require="__directory/opt/netbox/netbox/cdist " __file \ /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ --source "$__object/files/configuration.py" +if [ -f "$__object/parameter/ldap-server" ]; then + require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + --source "$__object/files/ldap_config.py" +fi + + # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do __file /etc/systemd/system/$unit.service \ diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional new file mode 100644 index 0000000..c946d73 --- /dev/null +++ b/type/__netbox/parameter/optional @@ -0,0 +1,4 @@ +ldap-server +ldap-bind-dn +ldap-bind-password +ldap-user-dn-template From ef7a214dae108351e1e2ae66a51966a639ee90d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jul 2020 08:30:09 +0200 Subject: [PATCH 051/156] __netbox: support basic LDAP group mapping --- type/__netbox/files/ldap_config.py.sh | 40 +++++++++++++++++++++++++-- type/__netbox/manifest | 16 +++++++++-- type/__netbox/parameter/optional | 5 +++- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 88af4b9..901d6f5 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -6,7 +6,7 @@ cat << EOF ############################## import ldap -from django_auth_ldap.config import LDAPSearch +from django_auth_ldap.config import LDAPSearch, PosixGroupType # Server URI AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" @@ -15,8 +15,10 @@ AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" AUTH_LDAP_BIND_DN = "$LDAP_BIND_DN" AUTH_LDAP_BIND_PASSWORD = "$LDAP_BIND_PASSWORD" -# If a user's DN is producible from their username, we don't need to search. -AUTH_LDAP_USER_DN_TEMPLATE = "$LDAP_USER_DN_TEMPLATE" +# Search for user entry. +AUTH_LDAP_USER_SEARCH = LDAPSearch("$LDAP_USER_BASE", + ldap.SCOPE_SUBTREE, + "(uid=%(user)s)") # You can map user attributes to Django attributes as so. AUTH_LDAP_USER_ATTR_MAP = { @@ -25,3 +27,35 @@ AUTH_LDAP_USER_ATTR_MAP = { "email": "mail" } EOF + +if [ "$LDAP_GROUP_BASE" != "" ]; then + cat << EOF + +# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group +# hierarchy. +AUTH_LDAP_GROUP_SEARCH = LDAPSearch("$LDAP_GROUP_BASE", ldap.SCOPE_SUBTREE, + "(objectClass=posixGroup)") +AUTH_LDAP_GROUP_TYPE = PosixGroupType() + +# Mirror LDAP group assignments. +AUTH_LDAP_MIRROR_GROUPS = True +EOF + + if [ "$LDAP_REQUIRE_GROUP" != "" ]; then + cat << EOF + +# Define a group required to login. +AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" +EOF + fi + + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + cat << EOF + +# Define special user types using groups. Exercise great caution when assigning superuser status. +AUTH_LDAP_USER_FLAGS_BY_GROUP = { + "is_superuser": "$LDAP_SUPERUSER_GROUP", +} +EOF + fi +fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2976686..32c85e3 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -40,8 +40,20 @@ if [ -f "$__object/parameter/ldap-bind-password" ]; then export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") fi -if [ -f "$__object/parameter/ldap-user-dn-template" ]; then - export LDAP_USER_DN_TEMPLATE=$(cat "$__object/parameter/ldap-user-dn-template") +if [ -f "$__object/parameter/ldap-user-base" ]; then + export LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") +fi + +if [ -f "$__object/parameter/ldap-group-base" ]; then + export LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") +fi + +if [ -f "$__object/parameter/ldap-require-group" ]; then + export LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") +fi + +if [ -f "$__object/parameter/ldap-superuser-group" ]; then + export LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") fi # Create system user used to run netbox. diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index c946d73..290c98b 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,4 +1,7 @@ ldap-server ldap-bind-dn ldap-bind-password -ldap-user-dn-template +ldap-user-base +ldap-group-base +ldap-require-group +ldap-superuser-group From e2f01bbb3b3c297cd17400437eb8bd13f4697a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 22 Jul 2020 17:14:06 +0200 Subject: [PATCH 052/156] __matrix_riot: rename to __matrix_element --- .../files/config.json.sh | 2 +- .../gencode-remote | 2 +- .../man.rst | 20 +++++++++---------- .../manifest | 0 .../parameter/boolean | 0 type/__matrix_element/parameter/default/brand | 1 + .../parameter/default/cookie_policy_url | 0 .../parameter/default/default_country_code | 0 .../parameter/default/default_server_name | 0 .../parameter/default/default_server_url | 0 .../parameter/default/owner | 0 .../parameter/default/privacy_policy_url | 1 + .../parameter/default/room_directory_servers | 0 .../parameter/optional | 0 .../parameter/optional_multiple | 0 .../parameter/required | 0 type/__matrix_riot/parameter/default/brand | 1 - .../parameter/default/privacy_policy_url | 1 - 18 files changed, 14 insertions(+), 14 deletions(-) rename type/{__matrix_riot => __matrix_element}/files/config.json.sh (96%) rename type/{__matrix_riot => __matrix_element}/gencode-remote (97%) rename type/{__matrix_riot => __matrix_element}/man.rst (73%) rename type/{__matrix_riot => __matrix_element}/manifest (100%) rename type/{__matrix_riot => __matrix_element}/parameter/boolean (100%) create mode 100644 type/__matrix_element/parameter/default/brand rename type/{__matrix_riot => __matrix_element}/parameter/default/cookie_policy_url (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_country_code (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_server_name (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_server_url (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/owner (100%) create mode 100644 type/__matrix_element/parameter/default/privacy_policy_url rename type/{__matrix_riot => __matrix_element}/parameter/default/room_directory_servers (100%) rename type/{__matrix_riot => __matrix_element}/parameter/optional (100%) rename type/{__matrix_riot => __matrix_element}/parameter/optional_multiple (100%) rename type/{__matrix_riot => __matrix_element}/parameter/required (100%) delete mode 100644 type/__matrix_riot/parameter/default/brand delete mode 100644 type/__matrix_riot/parameter/default/privacy_policy_url diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_element/files/config.json.sh similarity index 96% rename from type/__matrix_riot/files/config.json.sh rename to type/__matrix_element/files/config.json.sh index 6cd1648..9791f38 100755 --- a/type/__matrix_riot/files/config.json.sh +++ b/type/__matrix_element/files/config.json.sh @@ -39,7 +39,7 @@ EOF fi cat << EOF - "welcomeBackgroundUrl": "themes/riot/img/backgrounds/valley.jpg" + "welcomeBackgroundUrl": "themes/element/img/backgrounds/lake.jpg" EOF echo '},' } diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_element/gencode-remote similarity index 97% rename from type/__matrix_riot/gencode-remote rename to type/__matrix_element/gencode-remote index dbc886c..e643976 100755 --- a/type/__matrix_riot/gencode-remote +++ b/type/__matrix_element/gencode-remote @@ -46,7 +46,7 @@ cd \$tmpdir curl -L '$url' > $archive tar xf $archive -# Backup files deployed by __matrix_riot. +# Backup files deployed by __matrix_element. mkdir -p \$custom_files_dir for file in $INSTALL_DIR/cdist/*; do cp "\$file" "\$custom_files_dir" diff --git a/type/__matrix_riot/man.rst b/type/__matrix_element/man.rst similarity index 73% rename from type/__matrix_riot/man.rst rename to type/__matrix_element/man.rst index 2fe8eae..c3e0d5a 100644 --- a/type/__matrix_riot/man.rst +++ b/type/__matrix_element/man.rst @@ -1,23 +1,23 @@ -cdist-type__matrix_riot(7) +cdist-type__matrix_element(7) ====================== NAME ---- -cdist-type__matrix_riot - Install and configure Riot, a web Matrix client. +cdist-type__matrix_element - Install and configure Element, a web Matrix client. DESCRIPTION ----------- -This type install and configure the Riot web client. +This type install and configure the Element web client. REQUIRED PARAMETERS ------------------- install_dir - Root directory of Riot's static files. + Root directory of Element's static files. version - Release of Riot to install. + Release of Element to install. OPTIONAL PARAMETERS ------------------- @@ -31,20 +31,20 @@ owner Owner of the deployed files, passed to `chown`. Defaults to 'root'. brand - Web UI branding, defaults to 'Riot'. + Web UI branding, defaults to 'Element'. default_country_code ISO 3166 alpha2 country code to use when showing country selectors, such as phone number inputs. Defaults to GB. privacy_policy_url - Defaults to 'https://riot.im/privacy'. + Defaults to 'https://element.io/privacy'. cookie_policy_url - Defaults to 'https://matrix.org/docs/guides/riot_im_cookie_policy'. + Defaults to 'https://matrix.org/docs/guides/element_im_cookie_policy'. jitsi_domain - Domain name of preferred Jitsi instance (default is jitsi.riot.im). This is + Domain name of preferred Jitsi instance (default is jitsi.element.im). This is used whenever a user clicks on the voice/video call buttons. homepage @@ -66,7 +66,7 @@ EXAMPLES .. code-block:: sh - __matrix_riot my-riot --install_dir /var/www/riot-web --version 1.5.6 + __matrix_element my-element --install_dir /var/www/element-web --version 1.5.6 SEE ALSO diff --git a/type/__matrix_riot/manifest b/type/__matrix_element/manifest similarity index 100% rename from type/__matrix_riot/manifest rename to type/__matrix_element/manifest diff --git a/type/__matrix_riot/parameter/boolean b/type/__matrix_element/parameter/boolean similarity index 100% rename from type/__matrix_riot/parameter/boolean rename to type/__matrix_element/parameter/boolean diff --git a/type/__matrix_element/parameter/default/brand b/type/__matrix_element/parameter/default/brand new file mode 100644 index 0000000..907f907 --- /dev/null +++ b/type/__matrix_element/parameter/default/brand @@ -0,0 +1 @@ +Element diff --git a/type/__matrix_riot/parameter/default/cookie_policy_url b/type/__matrix_element/parameter/default/cookie_policy_url similarity index 100% rename from type/__matrix_riot/parameter/default/cookie_policy_url rename to type/__matrix_element/parameter/default/cookie_policy_url diff --git a/type/__matrix_riot/parameter/default/default_country_code b/type/__matrix_element/parameter/default/default_country_code similarity index 100% rename from type/__matrix_riot/parameter/default/default_country_code rename to type/__matrix_element/parameter/default/default_country_code diff --git a/type/__matrix_riot/parameter/default/default_server_name b/type/__matrix_element/parameter/default/default_server_name similarity index 100% rename from type/__matrix_riot/parameter/default/default_server_name rename to type/__matrix_element/parameter/default/default_server_name diff --git a/type/__matrix_riot/parameter/default/default_server_url b/type/__matrix_element/parameter/default/default_server_url similarity index 100% rename from type/__matrix_riot/parameter/default/default_server_url rename to type/__matrix_element/parameter/default/default_server_url diff --git a/type/__matrix_riot/parameter/default/owner b/type/__matrix_element/parameter/default/owner similarity index 100% rename from type/__matrix_riot/parameter/default/owner rename to type/__matrix_element/parameter/default/owner diff --git a/type/__matrix_element/parameter/default/privacy_policy_url b/type/__matrix_element/parameter/default/privacy_policy_url new file mode 100644 index 0000000..37fa4bc --- /dev/null +++ b/type/__matrix_element/parameter/default/privacy_policy_url @@ -0,0 +1 @@ +https://element.io/privacy diff --git a/type/__matrix_riot/parameter/default/room_directory_servers b/type/__matrix_element/parameter/default/room_directory_servers similarity index 100% rename from type/__matrix_riot/parameter/default/room_directory_servers rename to type/__matrix_element/parameter/default/room_directory_servers diff --git a/type/__matrix_riot/parameter/optional b/type/__matrix_element/parameter/optional similarity index 100% rename from type/__matrix_riot/parameter/optional rename to type/__matrix_element/parameter/optional diff --git a/type/__matrix_riot/parameter/optional_multiple b/type/__matrix_element/parameter/optional_multiple similarity index 100% rename from type/__matrix_riot/parameter/optional_multiple rename to type/__matrix_element/parameter/optional_multiple diff --git a/type/__matrix_riot/parameter/required b/type/__matrix_element/parameter/required similarity index 100% rename from type/__matrix_riot/parameter/required rename to type/__matrix_element/parameter/required diff --git a/type/__matrix_riot/parameter/default/brand b/type/__matrix_riot/parameter/default/brand deleted file mode 100644 index e8095bb..0000000 --- a/type/__matrix_riot/parameter/default/brand +++ /dev/null @@ -1 +0,0 @@ -Riot diff --git a/type/__matrix_riot/parameter/default/privacy_policy_url b/type/__matrix_riot/parameter/default/privacy_policy_url deleted file mode 100644 index 4cdd12c..0000000 --- a/type/__matrix_riot/parameter/default/privacy_policy_url +++ /dev/null @@ -1 +0,0 @@ -https://riot.im/privacy From 5960356e7b0e540ed49af17ec05a0ba608dfaae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 10 Jul 2020 10:41:20 +0200 Subject: [PATCH 053/156] __unbound: add --local_data flag --- type/__unbound/files/unbound.conf.sh | 9 +++++++++ type/__unbound/man.rst | 4 ++++ type/__unbound/manifest | 5 +++++ type/__unbound/parameter/optional_multiple | 1 + 4 files changed, 19 insertions(+) diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index 46d2824..2f7ecff 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -18,6 +18,14 @@ generate_forward_addr() { done } +generate_local_data() { + IFS=' +' + for entry in $LOCAL_DATA; do + echo " local-data: \"$entry\"" + done +} + cat << EOF # # THIS FILE HAS BEEN GENERATED BY CDIST, DO NOT EDIT BY HAND. @@ -720,6 +728,7 @@ $(generate_access_control) # (this makes example.com, www.example.com, etc, all go to 192.0.2.3) # local-zone: "example.com" redirect # local-data: "example.com A 192.0.2.3" +$(generate_local_data) # # Shorthand to make PTR records, "IPv4 name" or "IPv6 name". # You can also add PTR records using local-data directly, but then diff --git a/type/__unbound/man.rst b/type/__unbound/man.rst index 679e601..316d011 100644 --- a/type/__unbound/man.rst +++ b/type/__unbound/man.rst @@ -34,6 +34,10 @@ access_control rc_interface Address or path to socket used for remote control (see `--enable_control`. Defaults to `127.0.0.1`). +local_data + Configure local data, which is served in reply to queries for it. Can be + specified multiple times. + BOOLEAN PARAMETERS ------------------ disable-ip4 diff --git a/type/__unbound/manifest b/type/__unbound/manifest index 346f30c..5d6b50f 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -54,6 +54,11 @@ if [ -f "$__object/parameter/rc_interface" ]; then export RC_INTERFACE fi +if [ -f "$__object/parameter/local_data" ]; then + LOCAL_DATA=$(cat "$__object/parameter/local_data") + export LOCAL_DATA +fi + # Boolean parameters: if [ -f "$__object/parameter/disable_ip4" ] && \ [ -f "$__object/parameter/disable_ip6" ]; then diff --git a/type/__unbound/parameter/optional_multiple b/type/__unbound/parameter/optional_multiple index ecacd1d..3fe7eca 100644 --- a/type/__unbound/parameter/optional_multiple +++ b/type/__unbound/parameter/optional_multiple @@ -1,2 +1,3 @@ access_control +local_data interface From 7eecc99c73f2304c5f4837dd9324a5d925256782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 27 Jul 2020 14:53:45 +0200 Subject: [PATCH 054/156] __unbound: fix typo in gencode-remote / service start --- type/__unbound/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote index d12a395..ba6d92b 100755 --- a/type/__unbound/gencode-remote +++ b/type/__unbound/gencode-remote @@ -11,6 +11,6 @@ cat << EOF if pgrep unbound; then service unbound reload else - service unbounb start + service unbound start fi EOF From f73bf49e4b510b426dce4c04d1d63e08455f1ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jul 2020 14:23:30 +0200 Subject: [PATCH 055/156] Remove manual / outdated CHANGELOG --- CHANGELOG.md | 6 ------ README.md | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index df26f7e..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,6 +0,0 @@ -# cdist-contrib changes - -* 2020-06-07: New type: __unbound_exporter (Timothée Floure) -* 2020-06-07: Extended type: wire remote control configuration for __unbond (Timothée Floure) -* 2020-06-03: New type: __unbound (Timothée Floure) -* 2020-04-28: New type: __find_exec (Ander Punnar) diff --git a/README.md b/README.md index ef4b2c3..4231de2 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ tool with community-maitained types which are either too specific to fit/be maintained in cdist itself or were not accepted in code cdist but could still be useful. -This project does not have releases and is continously updated: see -`CHANGELOG.md` for details. +This project does not have releases and is continously updated: see git history +for change log. ## Using cdist-contrib From bf5f85068d8614f97e190e5115e34f5442ba7924 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 7 Aug 2020 10:39:38 +0200 Subject: [PATCH 056/156] Add a type to create a borg repository. --- type/__borg_repo/gencode-remote | 36 ++++++++++++++++ type/__borg_repo/man.rst | 43 +++++++++++++++++++ type/__borg_repo/manifest | 14 ++++++ type/__borg_repo/parameter/boolean | 1 + type/__borg_repo/parameter/default/encryption | 1 + type/__borg_repo/parameter/optional | 1 + type/__borg_repo/parameter/required | 1 + 7 files changed, 97 insertions(+) create mode 100644 type/__borg_repo/gencode-remote create mode 100644 type/__borg_repo/man.rst create mode 100644 type/__borg_repo/manifest create mode 100644 type/__borg_repo/parameter/boolean create mode 100644 type/__borg_repo/parameter/default/encryption create mode 100644 type/__borg_repo/parameter/optional create mode 100644 type/__borg_repo/parameter/required diff --git a/type/__borg_repo/gencode-remote b/type/__borg_repo/gencode-remote new file mode 100644 index 0000000..542bf5f --- /dev/null +++ b/type/__borg_repo/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh + +passphrase= +appendonly= + +case "$(cat "${__object:?}/parameter/encryption")" in + none) + enc=none + ;; + repokey) + enc=repokey + if [ -f "${__object:?}/parameter/passphrase" ]; + then + passphrase="$(cat "${__object:?}/parameter/passphrase")" + else + echo "__borg_repo cannot use repokey encryption with no passphrase. Aborting." >&2; + exit 1; + fi + ;; + *) + echo "$enc is not a known encryption mode for __borg_repo. Aborting." >&2 + exit 1; +esac + +if [ -f "${__object:?}/parameter/append-only" ]; +then + appendonly='--append-only' +fi + +cat <<- EOF + if ! borg check --repository-only 1>&2 2>/dev/null "/${__object_id:?}"; + then + BORG_NEW_PASSPHRASE=$passphrase borg init -e ${enc:?} $appendonly /${__object_id:?} + fi +EOF + diff --git a/type/__borg_repo/man.rst b/type/__borg_repo/man.rst new file mode 100644 index 0000000..38ab0c9 --- /dev/null +++ b/type/__borg_repo/man.rst @@ -0,0 +1,43 @@ +cdist-type__borg_repo(7) +======================== + +NAME +---- +cdist-type__borg_repo - Configure a borg repository on host + + +DESCRIPTION +----------- + +Initializes a borg repository at the location specified in the +`${__object_id}`. Nothing is done if the repository already exists. + +Currently, only `none` and `repokey` are supported as encryption modes; +`repokey` requires the `passphrase` argument to be given. The default is +`none`. + +REQUIRED PARAMETERS +------------------- +encryption + The encryption to use. + +OPTIONAL PARAMETERS +------------------- +passphrase + The passphrase to encrypt the keyfile with. + +BOOLEAN PARAMETERS +------------------ +append-only + If the repository is append-only + +AUTHORS +------- +Joachim Desroches + +COPYING +------- +Copyright \(C) 2020 Joachim Desroches. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__borg_repo/manifest b/type/__borg_repo/manifest new file mode 100644 index 0000000..fe18c9c --- /dev/null +++ b/type/__borg_repo/manifest @@ -0,0 +1,14 @@ +#!/bin/sh + +os="$(cat "${__global:?}"/explorer/os)" + +case "$os" in + "alpine") + borg_package=borgbackup + ;; + *) + echo "__borg_repo is not yet implemented for os $os. Aborting." >&2; + exit 1; +esac + +__package "$borg_package" diff --git a/type/__borg_repo/parameter/boolean b/type/__borg_repo/parameter/boolean new file mode 100644 index 0000000..f8ee7c6 --- /dev/null +++ b/type/__borg_repo/parameter/boolean @@ -0,0 +1 @@ +append-only diff --git a/type/__borg_repo/parameter/default/encryption b/type/__borg_repo/parameter/default/encryption new file mode 100644 index 0000000..621e94f --- /dev/null +++ b/type/__borg_repo/parameter/default/encryption @@ -0,0 +1 @@ +none diff --git a/type/__borg_repo/parameter/optional b/type/__borg_repo/parameter/optional new file mode 100644 index 0000000..f63b25b --- /dev/null +++ b/type/__borg_repo/parameter/optional @@ -0,0 +1 @@ +passphrase diff --git a/type/__borg_repo/parameter/required b/type/__borg_repo/parameter/required new file mode 100644 index 0000000..a5465f8 --- /dev/null +++ b/type/__borg_repo/parameter/required @@ -0,0 +1 @@ +encryption From 47584d6da64b86a266d950a61879ccb82268d2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 08:58:29 +0200 Subject: [PATCH 057/156] __netbox: use __systemd_unit, do not recreate existing directory --- type/__netbox/gencode-remote | 1 - type/__netbox/manifest | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index e3157a9..122df96 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -25,7 +25,6 @@ curl -L '$url' > '$archive' tar xf '$archive' # Save cdist-upload configuration file. -mkdir -p "\$tmpdir" cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 32c85e3..2503bb4 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -78,9 +78,9 @@ fi # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do - __file /etc/systemd/system/$unit.service \ - --mode 644 --source "$__type/files/$unit.service" \ - --onchange 'systemctl daemon-reload' + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled done # Python worker configuration. From 240b8f1615d292ddca081304ce930792e330fc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 09:01:05 +0200 Subject: [PATCH 058/156] __netbox: make shellcheck happy --- type/__netbox/manifest | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2503bb4..fc9192c 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -23,37 +23,48 @@ case "$os" in ;; esac -export DATABASE_NAME=$(cat "$__object/parameter/database") -export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") -export ALLOWED_HOST=$(cat "$__object/parameter/host") -export SECRET_KEY=$(cat "$__object/parameter/secret-key") +DATABASE_NAME=$(cat "$__object/parameter/database") +export DATABASE_NAME +DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") +export DATABASE_PASSWORD +ALLOWED_HOST=$(cat "$__object/parameter/host") +export ALLOWED_HOST +SECRET_KEY=$(cat "$__object/parameter/secret-key") +export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then - export LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + export LDAP_SERVER fi if [ -f "$__object/parameter/ldap-bind-dn" ]; then - export LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + export LDAP_BIND_DN fi if [ -f "$__object/parameter/ldap-bind-password" ]; then - export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + export LDAP_BIND_PASSWORD fi if [ -f "$__object/parameter/ldap-user-base" ]; then - export LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + export LDAP_USER_BASE fi if [ -f "$__object/parameter/ldap-group-base" ]; then - export LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + export LDAP_GROUP_BASE fi if [ -f "$__object/parameter/ldap-require-group" ]; then - export LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + export LDAP_REQUIRE_GROUP fi if [ -f "$__object/parameter/ldap-superuser-group" ]; then - export LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + export LDAP_SUPERUSER_GROUP fi # Create system user used to run netbox. From dfd9f108b1ba11b91c3e198fde60f86e5bf34006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 09:15:18 +0200 Subject: [PATCH 059/156] __netbox: add minimal manpage --- type/__netbox/man.rst | 94 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 type/__netbox/man.rst diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst new file mode 100644 index 0000000..25aeb26 --- /dev/null +++ b/type/__netbox/man.rst @@ -0,0 +1,94 @@ +cdist-type__netbox(7) +===================== + +NAME +---- +cdist-type__netbox - Install and configure NetBox + + +DESCRIPTION +----------- +This (singleton) type installs and configures a NetBox instance, a web +application to help manage and document computer networks + + +REQUIRED PARAMETERS +------------------- +version + NetBox version to be installed. + +database + PostgreSQL database name. + +database-password + PostgreSQL database password. + +secret-key + Random secret key of at least 50 alphanumeric characters. This key must be + unique to this installation and must not be shared outside the local + system. + +host + Hostname (domain or IP address) on which the application is served. + +OPTIONAL PARAMETERS +------------------- +ldap-server + LDAP server URI. Enables LDAP-backed authentication if specified. + +ldap-bind-dn + DN for the NetBox service account. Required for LDAP authentication. + +ldap-bind-password + Password for the NetBox service account. Required for LDAP authentication. + +ldap-user-base + Base used for searching user entries. Required for LDAP authentication. + +ldap-group-base + Base used for searching group entries. + +ldap-require-group + Group required to login. + +ldap-superuser-group + Make members of this groups superusers. + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __netbox --version 2.8.7 --database netbox \ + --database-password "secretsecretsecret" \ + --secret-key "secretsecretsecret" \ + --host "${__target_host:?}" \ + --ldap-server "ldaps://ldap.domain.tld" \ + --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ + --ldap-bind-password "secretsecretsecret" \ + --ldap-user-base "ou=users,dc=domain,dc=tld" \ + --ldap-group-base "ou=groups,dc=domain,dc=tld" \ + --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ + --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + + +SEE ALSO +-------- +- `NetBox documentation `_ + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. From efab6ac23ec52e8d5dad8cda9df9b050d66a23f1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 16 Aug 2020 15:18:47 +0200 Subject: [PATCH 060/156] __netbox: add version memorisation This saves the installed version and checks if a new version must be installed. If no update required, it updates the configuration files if required. --- type/__netbox/explorer/version | 5 ++++ type/__netbox/gencode-remote | 42 ++++++++++++++++++++++++++++------ type/__netbox/man.rst | 10 ++++++++ type/__netbox/manifest | 0 4 files changed, 50 insertions(+), 7 deletions(-) create mode 100755 type/__netbox/explorer/version mode change 100644 => 100755 type/__netbox/manifest diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version new file mode 100755 index 0000000..a9ed590 --- /dev/null +++ b/type/__netbox/explorer/version @@ -0,0 +1,5 @@ +#!/bin/sh + +# output version if exist +version_path="/opt/netbox/netbox/cdist/version" +if [ -f "$version_path" ]; then cat "$version_path"; fi diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 122df96..48463f7 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -1,5 +1,8 @@ #!/bin/sh +echo "set -e" + +old_version="$(cat "$__object/explorer/version")" VERSION=$(cat "$__object/parameter/version") src="netbox-$VERSION" @@ -7,9 +10,8 @@ archive="v$VERSION.tar.gz" url="https://github.com/netbox-community/netbox/archive/$archive" install_dir=/opt/netbox/netbox -cat << EOF -set -e - +if [ "$VERSION" != "$old_version" ]; then + cat << EOF # Ensure that coreutils is installed. if [ ! -x \$(which mktemp) ]; then echo "mktemp is not available on the remote host." >&2 @@ -41,11 +43,11 @@ virtualenv /opt/netbox/venv /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" EOF -if [ -f "$__object/parameter/ldap-server" ]; then - echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" -fi + if [ -f "$__object/parameter/ldap-server" ]; then + echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + fi -cat << EOF + cat << EOF # Set final permissions. chown -R netbox /opt/netbox @@ -59,7 +61,33 @@ sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collect cd / rm -r "\$tmpdir" +# Save version after successful installation +printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' +EOF + + # meta + printf "%s %s\n" "installed" "$VERSION" >> "$__messages_out" + changes=yes + +# check if configuration changed +elif grep -q "^__file/opt/netbox/" "$__messages_in"; then + cat <> "$__messages_out" + changes=yes +fi + + +# check for changes +if [ "$changes" = "yes" ]; then + cat < Date: Sun, 16 Aug 2020 15:31:38 +0200 Subject: [PATCH 061/156] __netbox: add manpage explanation of the type --- type/__netbox/man.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 84f1681..e41b099 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -9,7 +9,11 @@ cdist-type__netbox - Install and configure NetBox DESCRIPTION ----------- This (singleton) type installs and configures a NetBox instance, a web -application to help manage and document computer networks +application to help manage and document computer networks. + +It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. +Netbox will be run via `gnuicorn` as WSGI service. It setup systemd unit files +for the services `netbox` and `netbox-rq`. REQUIRED PARAMETERS From 0281b2c8046832d067dfa3be0da8702ef590faf4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 20 Aug 2020 17:53:25 +0200 Subject: [PATCH 062/156] __netbox: no ldap config if no ldap parameters The ldap config file will be empty (only comments) if no parameter name set matches `^ldap-*`. It still generate a ldap config if one or more ldap parameters are set, but not all basic parameters. --- type/__netbox/files/ldap_config.py.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 901d6f5..3e7aaa6 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,5 +1,19 @@ #!/bin/sh +# no configuration if there are no ldap parameters +if ls -1qA "$__object/parameter"/ldap-* | grep -q .; then + # skip + cat << EOF +############################## +# LDAP-backed authentication # +############################## + +# no options set +EOF + exit 0 +fi + + cat << EOF ############################## # LDAP-backed authentication # From a05cb214e17303068649aab3749099e4a06ceef1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 20 Aug 2020 19:37:09 +0200 Subject: [PATCH 063/156] __netbox: fix shellcheck by changing parameter checks The working `ls | grep .` like solution was dropped based on the shellcheck warning. Rather a solution with `find(1)` is used. As alternative, it could be simply checked if the glob succeeds or not by test all given arguments if there are existing files. It was dropped in question if it's really better than the `find(1)` solution. --- type/__netbox/files/ldap_config.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 3e7aaa6..bc494fb 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if ls -1qA "$__object/parameter"/ldap-* | grep -q .; then +if [ "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then # skip cat << EOF ############################## From e6e6dbcd0029639539ce7fe2d8a6fb7d753ad727 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 21 Aug 2020 23:50:24 +0200 Subject: [PATCH 064/156] __netbox: added some netbox configuration options Those who might be useful are added .. --- type/__netbox/files/configuration.py.sh | 64 ++++++++++++------ type/__netbox/man.rst | 65 ++++++++++++++++++- type/__netbox/manifest | 65 +++++++++++++++++++ type/__netbox/parameter/boolean | 4 ++ type/__netbox/parameter/default/basepath | 1 + .../parameter/default/redis-dbid-offset | 1 + type/__netbox/parameter/default/redis-host | 1 + .../__netbox/parameter/default/redis-password | 1 + type/__netbox/parameter/default/redis-port | 1 + .../parameter/default/smtp-from-email | 1 + type/__netbox/parameter/default/smtp-host | 1 + type/__netbox/parameter/default/smtp-password | 1 + type/__netbox/parameter/default/smtp-port | 1 + type/__netbox/parameter/default/smtp-user | 1 + type/__netbox/parameter/optional | 12 ++++ 15 files changed, 200 insertions(+), 20 deletions(-) create mode 100644 type/__netbox/parameter/boolean create mode 100644 type/__netbox/parameter/default/basepath create mode 100644 type/__netbox/parameter/default/redis-dbid-offset create mode 100644 type/__netbox/parameter/default/redis-host create mode 100644 type/__netbox/parameter/default/redis-password create mode 100644 type/__netbox/parameter/default/redis-port create mode 100644 type/__netbox/parameter/default/smtp-from-email create mode 100644 type/__netbox/parameter/default/smtp-host create mode 100644 type/__netbox/parameter/default/smtp-password create mode 100644 type/__netbox/parameter/default/smtp-port create mode 100644 type/__netbox/parameter/default/smtp-user diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index b7e1597..e621845 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -29,26 +29,26 @@ DATABASE = { # to use two separate database IDs. REDIS = { 'tasks': { - 'HOST': 'localhost', - 'PORT': 6379, + 'HOST': '$REDIS_HOST', + 'PORT': $REDIS_PORT, # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], # 'SENTINEL_SERVICE': 'netbox', - 'PASSWORD': '', - 'DATABASE': 0, + 'PASSWORD': '$REDIS_PASSWORD', + 'DATABASE': $((REDIS_DBID_OFFSET + 0)), 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'SSL': $REDIS_SSL, }, 'caching': { - 'HOST': 'localhost', - 'PORT': 6379, + 'HOST': '$REDIS_HOST', + 'PORT': $REDIS_PORT, # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], # 'SENTINEL_SERVICE': 'netbox', - 'PASSWORD': '', - 'DATABASE': 1, + 'PASSWORD': '$REDIS_PASSWORD', + 'DATABASE': $((REDIS_DBID_OFFSET + 1)), 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'SSL': $REDIS_SSL, } } @@ -86,7 +86,7 @@ BANNER_LOGIN = '' # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: # BASE_PATH = 'netbox/' -BASE_PATH = '' +BASE_PATH = '$BASEPATH' # Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) CACHE_TIMEOUT = 900 @@ -112,14 +112,14 @@ DEBUG = False # Email settings EMAIL = { - 'SERVER': 'localhost', - 'PORT': 25, - 'USERNAME': '', - 'PASSWORD': '', - 'USE_SSL': False, - 'USE_TLS': False, + 'SERVER': '$SMTP_HOST', + 'PORT': $SMTP_PORT, + 'USERNAME': '$SMTP_USER', + 'PASSWORD': '$SMTP_PASSWORD', + 'USE_SSL': $SMTP_USE_SSL, + 'USE_TLS': $SMTP_USE_TLS, 'TIMEOUT': 10, # seconds - 'FROM_EMAIL': '', + 'FROM_EMAIL': '$SMTP_FROM_EMAIL', } # Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table @@ -134,12 +134,38 @@ EXEMPT_VIEW_PERMISSIONS = [ # 'ipam.prefix', ] +EOF + +if [ "$HTTP_PROXY" != "" ] || [ "$HTTPS_PROXY" != "" ]; then + cat << EOF +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +HTTP_PROXIES = { +EOF + if [ "$HTTP_PROXY" != "" ]; then + cat << EOF + 'http': '$HTTP_PROXY', +EOF + fi + if [ "$HTTPS_PROXY" != "" ]; then + cat << EOF + 'https': '$HTTPS_PROXY', +EOF + fi + cat << EOF +} +EOF + +else + cat << EOF # HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). # HTTP_PROXIES = { # 'http': 'http://10.10.1.10:3128', # 'https': 'http://10.10.1.10:1080', # } +EOF +fi +cat << EOF # IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing # NetBox from an internal IP. INTERNAL_IPS = ('127.0.0.1', '::1') @@ -150,7 +176,7 @@ LOGGING = {} # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users # are permitted to access most data in NetBox (excluding secrets) but not make any changes. -LOGIN_REQUIRED = False +LOGIN_REQUIRED = $LOGIN_REQUIRED # The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to # re-authenticate. (Default: 1209600 [14 days]) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index e41b099..3e607b5 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -58,9 +58,62 @@ ldap-require-group ldap-superuser-group Make members of this groups superusers. +redis-host + Redis database hostname. Defaults to ``localhost``. + +redis-port + Redis database port. Defaults to ``6379``. + +redis-password + Redis password. Defaults to empty password. + +redis-dbid-offset + Offset to set the redis database id's. The `tasks` database id is `offset + 0` + and `caching` is `offset + 1`. The offset defaults to ``0``. + +smtp-host + Host of the SMTP email server. Defaults to ``localhost``. + +smtp-port + Port of the SMTP email server. Defaults to ``25``. + +smtp-user + Username to access the SMTP email server. Defaults to empty. + +smtp-password + Password to access the SMTP email server. Defaults to empty. + +smtp-from-email + Email from which NetBox will be sent of. Defaults to empty. + +basepath + Base URL path if accessing netbox within a directory instead of directly the + webroot ``/``. For example, if installed at https://example.com/netbox/, set + the value ``netbox/``. + +http-proxy +https-proxy + Proxy which will be used with any HTTP request like webhooks. + BOOLEAN PARAMETERS ------------------ -None. +redis-ssl + Enables a secure TLS/SSL connection to the redis database. By default, ssl + is disabled. + +smtp-use-tls + Uses TLS to connect to the SMTP email server. `See documentation + ` + SEE ALSO -------- - `NetBox documentation `_ diff --git a/type/__netbox/manifest b/type/__netbox/manifest index fc9192c..1f7f8ed 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -23,6 +23,7 @@ case "$os" in ;; esac + DATABASE_NAME=$(cat "$__object/parameter/database") export DATABASE_NAME DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") @@ -67,6 +68,70 @@ if [ -f "$__object/parameter/ldap-superuser-group" ]; then export LDAP_SUPERUSER_GROUP fi +# have default values +REDIS_HOST="$(cat "$__object/parameter/redis-host")" +export REDIS_HOST +REDIS_PORT="$(cat "$__object/parameter/redis-port")" +export REDIS_PORT +REDIS_PASSWORD="$(cat "$__object/parameter/redis-password")" +export REDIS_PASSWORD +REDIS_DBID_OFFSET="$(cat "$__object/parameter/redis-dbid-offset")" +export REDIS_DBID_OFFSET +if [ -f "$__object/parameter/redis-ssl" ]; then + REDIS_SSL="True" +else + REDIS_SSL="False" +fi +export REDIS_SSL + +SMTP_HOST="$(cat "$__object/parameter/smtp-host")" +export SMTP_HOST +SMTP_PORT="$(cat "$__object/parameter/smtp-port")" +export SMTP_PORT +SMTP_USER="$(cat "$__object/parameter/smtp-user")" +export SMTP_USER +SMTP_PASSWORD="$(cat "$__object/parameter/smtp-password")" +export SMTP_PASSWORD +SMTP_FROM_EMAIL="$(cat "$__object/parameter/smtp-from-email")" +export SMTP_FROM_EMAIL + +if [ -f "$__object/parameter/smtp-use-ssl" ]; then + SMTP_USE_SSL="True" +else + SMTP_USE_SSL="False" +fi +export SMTP_USE_SSL +if [ -f "$__object/parameter/smtp-use-tls" ]; then + if [ "$SMTP_USE_SSL" = "True" ]; then + echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" + exit 2 + fi + SMTP_USE_TLS="True" +else + SMTP_USE_TLS="False" +fi +export SMTP_USE_TLS + +BASEPATH="$(cat "$__object/parameter/basepath")" +export BASEPATH + +if [ -f "$__object/parameter/http-proxy" ]; then + HTTP_PROXY=$(cat "$__object/parameter/http-proxy") + export HTTP_PROXY +fi +if [ -f "$__object/parameter/https-proxy" ]; then + HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") + export HTTPS_PROXY +fi + +if [ -f "$__object/parameter/login-required" ]; then + LOGIN_REQUIRED="True" +else + LOGIN_REQUIRED="False" +fi +export LOGIN_REQUIRED + + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/boolean b/type/__netbox/parameter/boolean new file mode 100644 index 0000000..a94808c --- /dev/null +++ b/type/__netbox/parameter/boolean @@ -0,0 +1,4 @@ +redis-ssl +smtp-use-ssl +smtp-use-tls +login-required diff --git a/type/__netbox/parameter/default/basepath b/type/__netbox/parameter/default/basepath new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/basepath @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/redis-dbid-offset b/type/__netbox/parameter/default/redis-dbid-offset new file mode 100644 index 0000000..573541a --- /dev/null +++ b/type/__netbox/parameter/default/redis-dbid-offset @@ -0,0 +1 @@ +0 diff --git a/type/__netbox/parameter/default/redis-host b/type/__netbox/parameter/default/redis-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/redis-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/redis-password b/type/__netbox/parameter/default/redis-password new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/redis-password @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/redis-port b/type/__netbox/parameter/default/redis-port new file mode 100644 index 0000000..a8c4b8e --- /dev/null +++ b/type/__netbox/parameter/default/redis-port @@ -0,0 +1 @@ +6379 diff --git a/type/__netbox/parameter/default/smtp-from-email b/type/__netbox/parameter/default/smtp-from-email new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-from-email @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/smtp-host b/type/__netbox/parameter/default/smtp-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/smtp-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/smtp-password b/type/__netbox/parameter/default/smtp-password new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-password @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/smtp-port b/type/__netbox/parameter/default/smtp-port new file mode 100644 index 0000000..7273c0f --- /dev/null +++ b/type/__netbox/parameter/default/smtp-port @@ -0,0 +1 @@ +25 diff --git a/type/__netbox/parameter/default/smtp-user b/type/__netbox/parameter/default/smtp-user new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-user @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 290c98b..159c813 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -5,3 +5,15 @@ ldap-user-base ldap-group-base ldap-require-group ldap-superuser-group +redis-host +redis-port +redis-password +redis-dbid-offset +smtp-host +smtp-port +smtp-user +smtp-password +smtp-from-email +basepath +http-proxy +https-proxy From 855843ea34ff896ccc23e08f5e84b6f0bdfb49b5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 09:19:12 +0200 Subject: [PATCH 065/156] __netbox: small code and config handling improvements fixed codestyle and detection if config files need to be copied. --- type/__netbox/gencode-remote | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 48463f7..8533f4a 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -66,26 +66,29 @@ printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' EOF # meta - printf "%s %s\n" "installed" "$VERSION" >> "$__messages_out" + printf "installed %s\n" "$VERSION" >> "$__messages_out" changes=yes # check if configuration changed elif grep -q "^__file/opt/netbox/" "$__messages_in"; then - cat <> "$__messages_out" + printf "configuration\n" >> "$__messages_out" changes=yes fi # check for changes if [ "$changes" = "yes" ]; then - cat < Date: Sat, 22 Aug 2020 14:40:07 +0200 Subject: [PATCH 066/156] __netbox: note + link to find NetBox versions --- type/__netbox/man.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 3e607b5..d91c6e8 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -19,7 +19,9 @@ for the services `netbox` and `netbox-rq`. REQUIRED PARAMETERS ------------------- version - NetBox version to be installed. + NetBox version to be installed. You can find the correct and newest version + on GitHub at the NetBox project page under + "`Releases `_". database PostgreSQL database name. From 987393942b9acc10e743c60b264f08dc203877bb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 15:53:24 +0200 Subject: [PATCH 067/156] __netbox: add more database parameters --- type/__netbox/files/configuration.py.sh | 4 ++-- type/__netbox/man.rst | 9 +++++++++ type/__netbox/manifest | 7 +++++++ type/__netbox/parameter/default/database-host | 1 + type/__netbox/parameter/default/database-port | 1 + type/__netbox/parameter/optional | 2 ++ type/__netbox/parameter/required | 1 + 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 type/__netbox/parameter/default/database-host create mode 100644 type/__netbox/parameter/default/database-port diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index e621845..65b00f5 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -19,8 +19,8 @@ DATABASE = { 'NAME': '$DATABASE_NAME', # Database name 'USER': '$DATABASE_USER', # PostgreSQL username 'PASSWORD': '$DATABASE_PASSWORD', # PostgreSQL password - 'HOST': 'localhost', # Database server - 'PORT': '', # Database port (leave blank for default) + 'HOST': '$DATABASE_HOST', # Database server + 'PORT': '$DATABASE_PORT', # Database port (leave blank for default) 'CONN_MAX_AGE': 300, # Max database connection age } diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index d91c6e8..a09b13a 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -26,6 +26,9 @@ version database PostgreSQL database name. +database-user + PostgreSQL database user. + database-password PostgreSQL database password. @@ -39,6 +42,12 @@ host OPTIONAL PARAMETERS ------------------- +database-host + PostgreSQL database hostname. Defaults to ``localhost``. + +database-port + PostgreSQL database port. Defaults to empty (uses the default port). + ldap-server LDAP server URI. Enables LDAP-backed authentication if specified. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 1f7f8ed..45c8afc 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -26,8 +26,15 @@ esac DATABASE_NAME=$(cat "$__object/parameter/database") export DATABASE_NAME +DATABASE_USER="$(cat "$__object/parameter/database-user")" +export DATABASE_USER DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") export DATABASE_PASSWORD +DATABASE_HOST="$(cat "$__object/parameter/database-host")" +export DATABASE_HOST +DATABASE_PORT="$(cat "$__object/parameter/database-port")" +export DATABASE_PORT + ALLOWED_HOST=$(cat "$__object/parameter/host") export ALLOWED_HOST SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox/parameter/default/database-host b/type/__netbox/parameter/default/database-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/database-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/database-port b/type/__netbox/parameter/default/database-port new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/database-port @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 159c813..d97f961 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,3 +1,5 @@ +database-host +database-port ldap-server ldap-bind-dn ldap-bind-password diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 388048a..84e8e4b 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -1,5 +1,6 @@ version database +database-user database-password secret-key host From 9fb3c7f52ef2a27e7d60d850d1918083c90d84c5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 16:00:15 +0200 Subject: [PATCH 068/156] __netbox: fix manpage indents --- type/__netbox/man.rst | 67 ++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index a09b13a..2e43bbd 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -43,88 +43,89 @@ host OPTIONAL PARAMETERS ------------------- database-host - PostgreSQL database hostname. Defaults to ``localhost``. + PostgreSQL database hostname. Defaults to ``localhost``. database-port - PostgreSQL database port. Defaults to empty (uses the default port). + PostgreSQL database port. Defaults to empty (uses the default port). ldap-server - LDAP server URI. Enables LDAP-backed authentication if specified. + LDAP server URI. Enables LDAP-backed authentication if specified. ldap-bind-dn - DN for the NetBox service account. Required for LDAP authentication. + DN for the NetBox service account. Required for LDAP authentication. ldap-bind-password - Password for the NetBox service account. Required for LDAP authentication. + Password for the NetBox service account. Required for LDAP authentication. ldap-user-base - Base used for searching user entries. Required for LDAP authentication. + Base used for searching user entries. Required for LDAP authentication. ldap-group-base - Base used for searching group entries. + Base used for searching group entries. ldap-require-group - Group required to login. + Group required to login. ldap-superuser-group - Make members of this groups superusers. + Make members of this groups superusers. redis-host - Redis database hostname. Defaults to ``localhost``. + Redis database hostname. Defaults to ``localhost``. redis-port - Redis database port. Defaults to ``6379``. + Redis database port. Defaults to ``6379``. redis-password - Redis password. Defaults to empty password. + Redis password. Defaults to empty password. redis-dbid-offset - Offset to set the redis database id's. The `tasks` database id is `offset + 0` - and `caching` is `offset + 1`. The offset defaults to ``0``. + Offset to set the redis database id's. The `tasks` database id is + `offset + 0` and `caching` is `offset + 1`. The offset defaults + to ``0``. smtp-host - Host of the SMTP email server. Defaults to ``localhost``. + Host of the SMTP email server. Defaults to ``localhost``. smtp-port - Port of the SMTP email server. Defaults to ``25``. + Port of the SMTP email server. Defaults to ``25``. smtp-user - Username to access the SMTP email server. Defaults to empty. + Username to access the SMTP email server. Defaults to empty. smtp-password - Password to access the SMTP email server. Defaults to empty. + Password to access the SMTP email server. Defaults to empty. smtp-from-email - Email from which NetBox will be sent of. Defaults to empty. + Email from which NetBox will be sent of. Defaults to empty. basepath - Base URL path if accessing netbox within a directory instead of directly the - webroot ``/``. For example, if installed at https://example.com/netbox/, set - the value ``netbox/``. + Base URL path if accessing netbox within a directory instead of directly the + webroot ``/``. For example, if installed at https://example.com/netbox/, set + the value ``netbox/``. http-proxy https-proxy - Proxy which will be used with any HTTP request like webhooks. + Proxy which will be used with any HTTP request like webhooks. BOOLEAN PARAMETERS ------------------ redis-ssl - Enables a secure TLS/SSL connection to the redis database. By default, ssl - is disabled. + Enables a secure TLS/SSL connection to the redis database. By default, ssl + is disabled. smtp-use-tls - Uses TLS to connect to the SMTP email server. `See documentation - Date: Sat, 22 Aug 2020 20:55:35 +0200 Subject: [PATCH 069/156] __netbox: auto-install sudo as dependency There are may be systems does not have `sudo` preinstalled like debian-minimal. Else, `su` could be used as alternative. --- type/__netbox/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 45c8afc..3033c4d 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -6,7 +6,7 @@ case "$os" in debian|ubuntu) # Install netbox dependencies. for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv; do + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do __package $pkg done From ae7ffac6bbf17e64b6ed79768f6db944395672dd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:29:30 +0200 Subject: [PATCH 070/156] __netbox: force virtualenv to be python3 Set a flag to be sure it is python3. Else, you may get an ugly error if python2 is not installed (and netbox requires python3.6 at minimum). --- type/__netbox/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 8533f4a..9359252 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -37,7 +37,8 @@ cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. -virtualenv /opt/netbox/venv +# forcing python3 to be sure (till python4 gets released ..) +virtualenv -p python3 /opt/netbox/venv # Install python dependencies. /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" From ce47cec2e7883723d8b11efda1f97090ac95cd8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:50:54 +0200 Subject: [PATCH 071/156] __netbox: silence curl progress bar .. but still showing errors if any. --- type/__netbox/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 9359252..e0eb09d 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -23,7 +23,7 @@ tmpdir=\$(mktemp -d) cd "\$tmpdir" # Download and extract sources. -curl -L '$url' > '$archive' +curl -sS -L '$url' > '$archive' tar xf '$archive' # Save cdist-upload configuration file. From 4a04a2c4728ef0625bc49a039ece3240058159f0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:58:44 +0200 Subject: [PATCH 072/156] __netbox: repalace tabs with whitespaces Doing `s/\t/ /g` to have all times the same indents. --- type/__netbox/files/ldap_config.py.sh | 14 ++--- type/__netbox/man.rst | 20 +++--- type/__netbox/manifest | 88 +++++++++++++-------------- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index bc494fb..cf4ed92 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -43,7 +43,7 @@ AUTH_LDAP_USER_ATTR_MAP = { EOF if [ "$LDAP_GROUP_BASE" != "" ]; then - cat << EOF + cat << EOF # This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group # hierarchy. @@ -55,21 +55,21 @@ AUTH_LDAP_GROUP_TYPE = PosixGroupType() AUTH_LDAP_MIRROR_GROUPS = True EOF - if [ "$LDAP_REQUIRE_GROUP" != "" ]; then - cat << EOF + if [ "$LDAP_REQUIRE_GROUP" != "" ]; then + cat << EOF # Define a group required to login. AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" EOF - fi + fi - if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then - cat << EOF + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + cat << EOF # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { "is_superuser": "$LDAP_SUPERUSER_GROUP", } EOF - fi + fi fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 2e43bbd..4e63ccc 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -144,16 +144,16 @@ EXAMPLES .. code-block:: sh __netbox --version 2.8.7 --database netbox \ - --database-password "secretsecretsecret" \ - --secret-key "secretsecretsecret" \ - --host "${__target_host:?}" \ - --ldap-server "ldaps://ldap.domain.tld" \ - --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ - --ldap-bind-password "secretsecretsecret" \ - --ldap-user-base "ou=users,dc=domain,dc=tld" \ - --ldap-group-base "ou=groups,dc=domain,dc=tld" \ - --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ - --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + --database-password "secretsecretsecret" \ + --secret-key "secretsecretsecret" \ + --host "${__target_host:?}" \ + --ldap-server "ldaps://ldap.domain.tld" \ + --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ + --ldap-bind-password "secretsecretsecret" \ + --ldap-user-base "ou=users,dc=domain,dc=tld" \ + --ldap-group-base "ou=groups,dc=domain,dc=tld" \ + --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ + --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" NOTES diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 3033c4d..21a9bc6 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -3,24 +3,24 @@ os=$(cat "$__global/explorer/os") case "$os" in - debian|ubuntu) - # Install netbox dependencies. - for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do - __package $pkg - done + debian|ubuntu) + # Install netbox dependencies. + for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do + __package $pkg + done - if [ -f "$__object/parameter/ldap-server" ]; then - for pkg in libldap2-dev libsasl2-dev libssl-dev; do - __package $pkg - done - fi - ;; - *) - printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; + if [ -f "$__object/parameter/ldap-server" ]; then + for pkg in libldap2-dev libsasl2-dev libssl-dev; do + __package $pkg + done + fi + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; esac @@ -41,38 +41,38 @@ SECRET_KEY=$(cat "$__object/parameter/secret-key") export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then - LDAP_SERVER=$(cat "$__object/parameter/ldap-server") - export LDAP_SERVER + LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + export LDAP_SERVER fi if [ -f "$__object/parameter/ldap-bind-dn" ]; then - LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") - export LDAP_BIND_DN + LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + export LDAP_BIND_DN fi if [ -f "$__object/parameter/ldap-bind-password" ]; then - LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") - export LDAP_BIND_PASSWORD + LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + export LDAP_BIND_PASSWORD fi if [ -f "$__object/parameter/ldap-user-base" ]; then - LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") - export LDAP_USER_BASE + LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + export LDAP_USER_BASE fi if [ -f "$__object/parameter/ldap-group-base" ]; then - LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") - export LDAP_GROUP_BASE + LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + export LDAP_GROUP_BASE fi if [ -f "$__object/parameter/ldap-require-group" ]; then - LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") - export LDAP_REQUIRE_GROUP + LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + export LDAP_REQUIRE_GROUP fi if [ -f "$__object/parameter/ldap-superuser-group" ]; then - LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") - export LDAP_SUPERUSER_GROUP + LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + export LDAP_SUPERUSER_GROUP fi # have default values @@ -123,12 +123,12 @@ BASEPATH="$(cat "$__object/parameter/basepath")" export BASEPATH if [ -f "$__object/parameter/http-proxy" ]; then - HTTP_PROXY=$(cat "$__object/parameter/http-proxy") - export HTTP_PROXY + HTTP_PROXY=$(cat "$__object/parameter/http-proxy") + export HTTP_PROXY fi if [ -f "$__object/parameter/https-proxy" ]; then - HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") - export HTTPS_PROXY + HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") + export HTTPS_PROXY fi if [ -f "$__object/parameter/login-required" ]; then @@ -149,23 +149,23 @@ mkdir -p "$__object/files" require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ - --source "$__object/files/configuration.py" + /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ + --source "$__object/files/configuration.py" if [ -f "$__object/parameter/ldap-server" ]; then - require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ - --source "$__object/files/ldap_config.py" + require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + --source "$__object/files/ldap_config.py" fi # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled done # Python worker configuration. require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" + --mode 644 --source "$__type/files/gunicorn.py" From 0b261070d362fed393c9e021654917bb525f8b7a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:04:33 +0200 Subject: [PATCH 073/156] __netbox: man note regards creating a superuser --- type/__netbox/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 4e63ccc..91313be 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -166,6 +166,11 @@ you are welcome to contribute! - `Possible optional settings ` +If you not setup ldap authentification, you may be interested into how to +`setting up a super user +` +directly on the machine to be able to access and use NetBox. + SEE ALSO -------- - `NetBox documentation `_ From 584e0bbda85669de99e907784c17a4d176aae996 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:23:36 +0200 Subject: [PATCH 074/156] __netbox: fix netbox version save The cdist directory needs to be created to write there. This is done directly after the new tarball is installed. --- type/__netbox/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index e0eb09d..d2ec5f9 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -33,6 +33,8 @@ cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' +mkdir '$install_dir/cdist' + cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' From 88ddebd0404a5b0d4adf597ec6b42c61c7240ea5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:47:15 +0200 Subject: [PATCH 075/156] __netbox: fix deprecated configuration option Fixes error message `/opt/netbox/netbox/netbox/settings.py:141: UserWarning: REMOTE_AUTH_DEFAULT_PERMISSIONS should be a dictionary. Backward compatibility will be removed in v2.10.` --- type/__netbox/files/configuration.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 65b00f5..8679437 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -247,7 +247,7 @@ REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] -REMOTE_AUTH_DEFAULT_PERMISSIONS = [] +REMOTE_AUTH_DEFAULT_PERMISSIONS = {} # This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. RELEASE_CHECK_TIMEOUT = 24 * 3600 From f5c8c26db6436777430a31c410c1c442fce184c7 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 11:36:35 +0200 Subject: [PATCH 076/156] __netbox: new config parameters for data paths .. to move them to an other partition. --- type/__netbox/files/configuration.py.sh | 36 ++++++++++++++++--------- type/__netbox/man.rst | 33 +++++++++++++++++++++++ type/__netbox/manifest | 19 +++++++++++++ type/__netbox/parameter/optional | 4 +++ 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 8679437..af19153 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -154,15 +154,6 @@ EOF cat << EOF } EOF - -else - cat << EOF -# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). -# HTTP_PROXIES = { -# 'http': 'http://10.10.1.10:3128', -# 'https': 'http://10.10.1.10:1080', -# } -EOF fi cat << EOF @@ -190,10 +181,18 @@ MAINTENANCE_MODE = False # all objects by specifying "?limit=0". MAX_PAGE_SIZE = 1000 +EOF + +if [ "$MEDIA_ROOT" != "" ]; then + cat << EOF # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that # the default value of this setting is derived from the installed location. -# MEDIA_ROOT = '/opt/netbox/netbox/media' +MEDIA_ROOT = '$MEDIA_ROOT' +EOF +fi + +cat << EOF # By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the # class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: # STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' @@ -257,14 +256,27 @@ RELEASE_CHECK_TIMEOUT = 24 * 3600 RELEASE_CHECK_URL = None # RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' +EOF + +if [ "$REPORTS_ROOT" != "" ]; then + cat << EOF # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -# REPORTS_ROOT = '/opt/netbox/netbox/reports' +REPORTS_ROOT = '$REPORTS_ROOT' +EOF +fi + +if [ "$SCRIPTS_ROOT" != "" ]; then + cat << EOF # The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -# SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' +SCRIPTS_ROOT = '$SCRIPTS_ROOT' +EOF +fi + +cat << EOF # By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use # local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 91313be..c910d5c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -107,6 +107,39 @@ http-proxy https-proxy Proxy which will be used with any HTTP request like webhooks. +data-root + This parameter set's the media, reports and scripts root to subdirectories + of the given directory. Values can be overwritten by special parameters like + `--media-root` for example. Use this option if you want to store persistant + data of netbox on an other partition. A trailing shlash is not needed. + + The data directories have following predefined sub-directory names: + + media root: + ``$data_root/media`` + reports root: + ``$data_root/reports`` + scripts root: + ``$data_root/scripts`` + +media-root + The file path to where media files (like image attachments) are stored. + Change this path if you require to store data on an other partiotion. + A trailing slash is not needed. By default, it will be stored into the + installation directory (``/opt/netbox/netbox/netbox/media``). + +reports-root + The file path of where custom reports are kept. Change this path if you + require to store data on an other partition. A trailing slash is not + needed. By default, it will be stored into the installation directory + (``/opt/netbox/netbox/netbox/reports``). + +scripts-root + The file path of where custom scripts are kept. Change this path if you + require to store data on an other partition. A trailing slash is not + needed. By default, it will be stored into the installation directory + (``/opt/netbox/netbox/netbox/scripts``). + BOOLEAN PARAMETERS ------------------ redis-ssl diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 21a9bc6..133b527 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -138,6 +138,25 @@ else fi export LOGIN_REQUIRED +if [ -f "$__object/parameter/data-root" ]; then + data_root="$(cat "$__object/parameter/data-root")" + MEDIA_ROOT="$data_root/media" + REPORTS_ROOT="$data_root/reports" + SCRIPTS_ROOT="$data_root/scripts" +fi +if [ -f "$__object/parameter/media-root" ]; then + MEDIA_ROOT="$(cat "$__object/parameter/media-root")" +fi +export MEDIA_ROOT +if [ -f "$__object/parameter/reports-root" ]; then + REPORTS_ROOT="$(cat "$__object/parameter/reports-root")" +fi +export REPORTS_ROOT +if [ -f "$__object/parameter/scripts-root" ]; then + SCRIPTS_ROOT="$(cat "$__object/parameter/scripts-root")" +fi +export SCRIPTS_ROOT + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index d97f961..21fea5f 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -19,3 +19,7 @@ smtp-from-email basepath http-proxy https-proxy +data-root +media-root +reports-root +scripts-root From 49bb527dea8752fe14155516581bd7ee9049143a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 16:52:02 +0200 Subject: [PATCH 077/156] __netbox: autogen secretkey and store at remote The secret key is generated if it is not set via parameter and the explorer does not return any. It will be saved in the netbox home directory to easily read the key for the config generation. --- type/__netbox/explorer/secretkey | 8 ++++++++ type/__netbox/explorer/version | 2 +- type/__netbox/man.rst | 16 +++++++++++----- type/__netbox/manifest | 19 ++++++++++++++++++- type/__netbox/parameter/optional | 1 + type/__netbox/parameter/required | 1 - 6 files changed, 39 insertions(+), 8 deletions(-) create mode 100755 type/__netbox/explorer/secretkey diff --git a/type/__netbox/explorer/secretkey b/type/__netbox/explorer/secretkey new file mode 100755 index 0000000..9f0331c --- /dev/null +++ b/type/__netbox/explorer/secretkey @@ -0,0 +1,8 @@ +#!/bin/sh -e + +# Explorer will output the key if he exists. + +secretkey="/opt/netbox/.secretkey" +if [ -f "$secretkey" ]; then + cat "$secretkey" +fi diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version index a9ed590..5231e16 100755 --- a/type/__netbox/explorer/version +++ b/type/__netbox/explorer/version @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # output version if exist version_path="/opt/netbox/netbox/cdist/version" diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index c910d5c..77b47d0 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -32,16 +32,22 @@ database-user database-password PostgreSQL database password. -secret-key - Random secret key of at least 50 alphanumeric characters. This key must be - unique to this installation and must not be shared outside the local - system. - host Hostname (domain or IP address) on which the application is served. OPTIONAL PARAMETERS ------------------- +secret-key + Random secret key of at least 50 alphanumeric characters and symbols. This + key must be unique to this installation and must not be shared outside the + local system. If no secret key is given, the type generates an own 50 chars + long key and saves it on the remote host to remember it for the next run. + + The secret, random string is used to assist in the creation new + cryptographic hashes for passwords and HTTP cookies. It is not directly + used for hasing user passwords or for encrpted storage. It can be changed + at any time, but will invalidate all existing sessions. + database-host PostgreSQL database hostname. Defaults to ``localhost``. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 133b527..cd47d38 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -37,7 +37,18 @@ export DATABASE_PORT ALLOWED_HOST=$(cat "$__object/parameter/host") export ALLOWED_HOST -SECRET_KEY=$(cat "$__object/parameter/secret-key") + +if [ -f "$__object/parameter/secret-key" ]; then + SECRET_KEY=$(cat "$__object/parameter/secret-key") +elif [ -s "$__object/explorer/secretkey" ]; then + # Can be done over netbox/generate_secret_key.py too, but it's to + # complicated with the variable setup (can't generated right now!). + # Generates a 50-character long key (without ' cause of python quotes) + SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" +else + # take the key that is already used + SECRET_KEY="$(cat "$__object/explorer/secretkey")" +fi export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then @@ -177,6 +188,12 @@ if [ -f "$__object/parameter/ldap-server" ]; then --source "$__object/files/ldap_config.py" fi +# save secret +require="__user/netbox" __file /opt/netbox/.secretkey --mode 400 \ + --owner netbox --source - << SECRET +$SECRET_KEY +SECRET + # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 21fea5f..47722e6 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,3 +1,4 @@ +secret-key database-host database-port ldap-server diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 84e8e4b..3d03e5a 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -2,5 +2,4 @@ version database database-user database-password -secret-key host From 4e90c93eef881a228272167f2e328de9a16bf892 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 17:15:32 +0200 Subject: [PATCH 078/156] __netbox: updated copyright Added me to the copyright after there are some more changes :-) --- type/__netbox/man.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 77b47d0..10485a1 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -217,11 +217,13 @@ SEE ALSO AUTHORS ------- Timothée Floure +Matthias Stecher COPYING ------- -Copyright \(C) 2020 Timothée Floure. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2020 Timothée Floure. +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. From 863c95059e7d5b10607b97b88aa49b402c0561d3 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 19:48:56 +0200 Subject: [PATCH 079/156] __netbox: fix REMOTE_AUTH_BACKEND module not found Updated with the backend from the current upstream configuration. --- type/__netbox/files/configuration.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index af19153..0b8304d 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -242,7 +242,7 @@ RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 # Remote authentication support REMOTE_AUTH_ENABLED = False -REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' +REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] From 95fd7d872ed1c649742008604d4becc88be010a4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 24 Aug 2020 18:57:27 +0200 Subject: [PATCH 080/156] __netbox: fix ldap config short-circuit Forgot to negotate if statement .. works now :/ --- type/__netbox/files/ldap_config.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index cf4ed92..c31f46a 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if [ "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then +if [ -z "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then # skip cat << EOF ############################## From 7d694342fffe1dd2bd850ac38c5fa7580da8fedb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 24 Aug 2020 19:31:49 +0200 Subject: [PATCH 081/156] __netbox: new parameter for automatic release checks --- type/__netbox/files/configuration.py.sh | 14 ++++++++++++-- type/__netbox/man.rst | 8 ++++++-- type/__netbox/manifest | 5 +++++ type/__netbox/parameter/boolean | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 0b8304d..21104b4 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -253,11 +253,21 @@ RELEASE_CHECK_TIMEOUT = 24 * 3600 # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the # version check or use the URL below to check for release in the official NetBox repository. -RELEASE_CHECK_URL = None -# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' EOF +if [ "$UPDATE_CHECK" != "" ]; then + cat << EOF +RELEASE_CHECK_URL = None + +EOF +else + cat << EOF +RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +EOF +fi + if [ "$REPORTS_ROOT" != "" ]; then cat << EOF # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 10485a1..761494a 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -163,8 +163,12 @@ smtp-use-ssl for more information. login-required - Sets if a login is required to access all sites. By default, anounymous users - can see most data (excluding secrets) but not make any changes. + Sets if a login is required to access all sites. By default, anonymous + users can see most data (excluding secrets) but not make any changes. + +update-notify + Enables the NetBox version check for new upstream updates. It checks every + 24 hours for new releases and notify the admin users in the gui if any. MESSAGES -------- diff --git a/type/__netbox/manifest b/type/__netbox/manifest index cd47d38..62da0a2 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -168,6 +168,11 @@ if [ -f "$__object/parameter/scripts-root" ]; then fi export SCRIPTS_ROOT +if [ -f "$__object/parameter/update-notify" ]; then + UPDATE_CHECK="yes" + export UPDATE_CHECK +fi + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/boolean b/type/__netbox/parameter/boolean index a94808c..d568037 100644 --- a/type/__netbox/parameter/boolean +++ b/type/__netbox/parameter/boolean @@ -2,3 +2,4 @@ redis-ssl smtp-use-ssl smtp-use-tls login-required +update-notify From 8ff7a4616dcedac333b5e93d994cbff2887ffbe9 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 25 Aug 2020 19:30:32 +0200 Subject: [PATCH 082/156] __netbox: allow multiple hostnames --- type/__netbox/files/configuration.py.sh | 2 +- type/__netbox/man.rst | 2 ++ type/__netbox/manifest | 8 ++++++-- type/__netbox/parameter/required | 1 - type/__netbox/parameter/required_multiple | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 type/__netbox/parameter/required_multiple diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 21104b4..4b7be11 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -11,7 +11,7 @@ cat << EOF # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. # # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] -ALLOWED_HOSTS = [ '$ALLOWED_HOST' ] +ALLOWED_HOSTS = [$ALLOWED_HOSTS ] # PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: # https://docs.djangoproject.com/en/stable/ref/settings/#databases diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 761494a..2af2bef 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -34,6 +34,7 @@ database-password host Hostname (domain or IP address) on which the application is served. + Multiple hostnames are possible; given as multiple arguments. OPTIONAL PARAMETERS ------------------- @@ -190,6 +191,7 @@ EXAMPLES --database-password "secretsecretsecret" \ --secret-key "secretsecretsecret" \ --host "${__target_host:?}" \ + --host "cool-netbox.xyz" \ --ldap-server "ldaps://ldap.domain.tld" \ --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ --ldap-bind-password "secretsecretsecret" \ diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 62da0a2..2f5cf4e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -35,8 +35,12 @@ export DATABASE_HOST DATABASE_PORT="$(cat "$__object/parameter/database-port")" export DATABASE_PORT -ALLOWED_HOST=$(cat "$__object/parameter/host") -export ALLOWED_HOST +# list of hosts +ALLOWED_HOSTS="" +while read -r hostname; do + ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," +done < "$__object/parameter/host" +export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 3d03e5a..02fca9f 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -2,4 +2,3 @@ version database database-user database-password -host diff --git a/type/__netbox/parameter/required_multiple b/type/__netbox/parameter/required_multiple new file mode 100644 index 0000000..c70dc2d --- /dev/null +++ b/type/__netbox/parameter/required_multiple @@ -0,0 +1 @@ +host From eaf639790b00bdb5d7d9197939d0e6fa7963f3a2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 17:39:19 +0200 Subject: [PATCH 083/156] __netbox: remove ldap pip3 package if not required --- type/__netbox/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d2ec5f9..abe7ffa 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -48,6 +48,8 @@ EOF if [ -f "$__object/parameter/ldap-server" ]; then echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + else + echo "/opt/netbox/venv/bin/pip3 uninstall django-auth-ldap" fi cat << EOF From 49d39eaee50bc848840cb4fda91e2ed0cb4a14f1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 26 Aug 2020 18:01:52 +0200 Subject: [PATCH 084/156] [type/__mail_alias] Fix error with some AWK implementations Some AWK implementations seem to have a problem with parameters named default. awk: cmd. line:2: function sepafter(f, default, _) { awk: cmd. line:2: ^ syntax error awk: cmd. line:5: return _ ? _ : default awk: cmd. line:5: ^ syntax error In addition the temp file is removed if an error occurs. --- type/__mail_alias/gencode-remote | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 3eaad75..9f4af1b 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -67,10 +67,10 @@ cat <<'EOF' test -f "${aliases_file}" || touch "${aliases_file}" awk -F ':[ \t]*' -v mode=$mode ' -function sepafter(f, default, _) { +function sepafter(f, def, _) { # finds the separator between field $f and $(f+1) _ = substr($0, length($f)+1, index(substr($0, length($f)+1), $(f+1))-1) - return _ ? _ : default + return _ ? _ : def } function write_aliases() { @@ -134,6 +134,7 @@ END { write_aliases() } ' <"${aliases_file}" >"${aliases_file}.tmp" || { + rm -f "${aliases_file}.tmp" echo 'Generating new aliases file failed!' >&2 exit 1 } From e0a1b4f66343415b002c6d1c86114da3c9d402f8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 20:01:23 +0200 Subject: [PATCH 085/156] __netbox_gunicorn: extract gunicorn setup from __netbox Because someone *want* to use something other than just gunicorn, it was extracted to a own type. Because gunicorn is a bit deep in the netbox installation process, it's a bit harder to isolate it. `__netbox_uwsgi` will come, too. --- type/__netbox/gencode-remote | 8 +- type/__netbox/manifest | 14 +--- type/__netbox_gunicorn/explorer/installed | 4 + .../explorer/should_installed | 3 + .../files/gunicorn.py.sh} | 19 ++++- .../files/netbox.service | 3 +- type/__netbox_gunicorn/gencode-remote | 36 +++++++++ type/__netbox_gunicorn/man.rst | 75 +++++++++++++++++++ type/__netbox_gunicorn/manifest | 20 +++++ .../parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + type/__netbox_gunicorn/singleton | 0 12 files changed, 168 insertions(+), 16 deletions(-) create mode 100755 type/__netbox_gunicorn/explorer/installed create mode 100755 type/__netbox_gunicorn/explorer/should_installed rename type/{__netbox/files/gunicorn.py => __netbox_gunicorn/files/gunicorn.py.sh} (56%) mode change 100644 => 100755 rename type/{__netbox => __netbox_gunicorn}/files/netbox.service (88%) create mode 100755 type/__netbox_gunicorn/gencode-remote create mode 100644 type/__netbox_gunicorn/man.rst create mode 100755 type/__netbox_gunicorn/manifest create mode 100644 type/__netbox_gunicorn/parameter/default/bind-to create mode 100644 type/__netbox_gunicorn/parameter/optional_multiple create mode 100644 type/__netbox_gunicorn/singleton diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index abe7ffa..af4d8c8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -35,6 +35,7 @@ rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' mkdir '$install_dir/cdist' +cp '$src/requirements.txt' /opt/netbox/ # backup dependency info cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' @@ -43,7 +44,9 @@ cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Install python dependencies. -/opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" +# avoid gunicorn, because it will be done in an other type +grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ + | xargs /opt/netbox/venv/bin/pip3 install EOF if [ -f "$__object/parameter/ldap-server" ]; then @@ -94,8 +97,7 @@ fi # check for changes if [ "$changes" = "yes" ]; then cat << EOF -# Restart services. -service netbox restart +# Restart service. service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2f5cf4e..f0da2a3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,13 +204,7 @@ $SECRET_KEY SECRET -# Upload systemd units and gunicorn configuration. -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done - -# Python worker configuration. -require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" +# Upload systemd unit for worker service +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled diff --git a/type/__netbox_gunicorn/explorer/installed b/type/__netbox_gunicorn/explorer/installed new file mode 100755 index 0000000..c6f5d87 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show gunicorn | awk '/Version:/{print $2}' diff --git a/type/__netbox_gunicorn/explorer/should_installed b/type/__netbox_gunicorn/explorer/should_installed new file mode 100755 index 0000000..073be92 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/should_installed @@ -0,0 +1,3 @@ +#!/bin/sh -e + +awk -v FS="==" '$1 ~ /gunicorn/{print $2}' /opt/netbox/requirements.txt diff --git a/type/__netbox/files/gunicorn.py b/type/__netbox_gunicorn/files/gunicorn.py.sh old mode 100644 new mode 100755 similarity index 56% rename from type/__netbox/files/gunicorn.py rename to type/__netbox_gunicorn/files/gunicorn.py.sh index c582677..862ba06 --- a/type/__netbox/files/gunicorn.py +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -1,9 +1,23 @@ +#!/bin/sh -e + +# Generates gunicorn config +# see https://docs.gunicorn.org/en/stable/settings.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = '127.0.0.1:8001' +bind = [$HOST ] # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. -workers = 3 +workers = $(( 2*cores + 1 )) # Number of threads per worker process threads = 3 @@ -14,3 +28,4 @@ timeout = 120 # The maximum number of requests a worker can handle before being respawned max_requests = 5000 max_requests_jitter = 500 +EOF diff --git a/type/__netbox/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service similarity index 88% rename from type/__netbox/files/netbox.service rename to type/__netbox_gunicorn/files/netbox.service index 18eb045..475ddc4 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,5 +1,5 @@ [Unit] -Description=NetBox WSGI Service +Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ After=network-online.target Wants=network-online.target @@ -19,4 +19,5 @@ RestartSec=30 PrivateTmp=true [Install] +Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote new file mode 100755 index 0000000..6958976 --- /dev/null +++ b/type/__netbox_gunicorn/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh -e + +curr_installed="$(cat "$__object/explorer/installed")" +should_installed="$(cat "$__object/explorer/should_installed")" + +# gunicorn version change +if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" + + do_restart=yes + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" +fi + +# configuration changes +if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configuration\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no message cause it should be obvious +fi + + +# restart gunicorn +if [ "$do_restart"]; then + cat << EOF +# Restart service +service gunicorn-netbox restart +EOF + echo restarted >> "$__messages_out" +fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst new file mode 100644 index 0000000..38ac4a5 --- /dev/null +++ b/type/__netbox_gunicorn/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_gunicorn - run netbox with gunicorn + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The hosts the gunicorn socket should be bind to. Formats are `IP`, + `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple + times. Defaults to ``127.0.0.1:8001``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +updated $old to $new + The version of the gunicorn software was updated from `$old` to `$new`. + +configuration + Configuration for gunicorn changed. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_gunicorn + + # with arguments + __netbox $args + require="__netbox" __netbox_gunicorn \ + --bind-to 0.0.0.0:8001 \ + --bind-to 1.2.3.4:5678 + + +SEE ALSO +-------- +:strong:`__netbox`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest new file mode 100755 index 0000000..dab8181 --- /dev/null +++ b/type/__netbox_gunicorn/manifest @@ -0,0 +1,20 @@ +#!/bin/sh -e + +HOST="" +while read -r host; do + HOST="$HOST '$host'," +done < "$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + +# gunicorn config file +__file /opt/netbox/gunicorn.py \ + --mode 644 --source "$__object/files/gunicorn.py" + +# install service file +__systemd_unit gunicorn-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_gunicorn/parameter/default/bind-to b/type/__netbox_gunicorn/parameter/default/bind-to new file mode 100644 index 0000000..f4c980e --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:8001 diff --git a/type/__netbox_gunicorn/parameter/optional_multiple b/type/__netbox_gunicorn/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to diff --git a/type/__netbox_gunicorn/singleton b/type/__netbox_gunicorn/singleton new file mode 100644 index 0000000..e69de29 From bbce0030aba387232f48a6c4b676c2b82cba28ed Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 21:07:31 +0200 Subject: [PATCH 086/156] __netbox: enable ldap usage via configuration.py The REMOTE_AUTH_BACKEND must be set to use LDAP. It now exports USE_LDAP to generally say if LDAP is being used in the configuration or not. --- type/__netbox/files/configuration.py.sh | 14 ++++++++++++++ type/__netbox/files/ldap_config.py.sh | 2 +- type/__netbox/manifest | 12 ++++++------ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 4b7be11..ea73133 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -240,9 +240,23 @@ PREFER_IPV4 = False RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = 22 RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 +EOF + +if [ "$USE_LDAP" ]; then + cat << EOF +# Remote authentication support with ldap +REMOTE_AUTH_ENABLED = True +REMOTE_AUTH_BACKEND = 'netbox.authentication.LDAPBackend' +EOF +else + cat << EOF # Remote authentication support REMOTE_AUTH_ENABLED = False REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' +EOF +fi + +cat << EOF REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index c31f46a..a49d800 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if [ -z "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then +if [ -z "$USE_LDAP" ]; then # skip cat << EOF ############################## diff --git a/type/__netbox/manifest b/type/__netbox/manifest index f0da2a3..079ea48 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -57,38 +57,38 @@ export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + USE_LDAP=yes export LDAP_SERVER fi - if [ -f "$__object/parameter/ldap-bind-dn" ]; then LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + USE_LDAP=yes export LDAP_BIND_DN fi - if [ -f "$__object/parameter/ldap-bind-password" ]; then LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + USE_LDAP=yes export LDAP_BIND_PASSWORD fi - if [ -f "$__object/parameter/ldap-user-base" ]; then LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + USE_LDAP=yes export LDAP_USER_BASE fi - if [ -f "$__object/parameter/ldap-group-base" ]; then LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") export LDAP_GROUP_BASE fi - if [ -f "$__object/parameter/ldap-require-group" ]; then LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") export LDAP_REQUIRE_GROUP fi - if [ -f "$__object/parameter/ldap-superuser-group" ]; then LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") export LDAP_SUPERUSER_GROUP fi +# export if base ldap parameters are used +export USE_LDAP # have default values REDIS_HOST="$(cat "$__object/parameter/redis-host")" From 1ef4420c537570206780757fa5b78aa6895f5829 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 27 Aug 2020 19:46:09 +0200 Subject: [PATCH 087/156] __netbox_uwsgi: new type to handle uwsgi for netbox --- type/__netbox/gencode-remote | 2 +- type/__netbox/man.rst | 2 +- type/__netbox_gunicorn/gencode-remote | 5 +- type/__netbox_gunicorn/man.rst | 2 +- type/__netbox_uwsgi/explorer/installed | 4 + type/__netbox_uwsgi/explorer/upgradeable | 4 + type/__netbox_uwsgi/files/netbox.service | 23 ++++++ type/__netbox_uwsgi/files/uwsgi.ini.sh | 22 ++++++ type/__netbox_uwsgi/gencode-remote | 35 +++++++++ type/__netbox_uwsgi/man.rst | 75 +++++++++++++++++++ type/__netbox_uwsgi/manifest | 18 +++++ type/__netbox_uwsgi/parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + 13 files changed, 188 insertions(+), 6 deletions(-) create mode 100755 type/__netbox_uwsgi/explorer/installed create mode 100755 type/__netbox_uwsgi/explorer/upgradeable create mode 100644 type/__netbox_uwsgi/files/netbox.service create mode 100755 type/__netbox_uwsgi/files/uwsgi.ini.sh create mode 100755 type/__netbox_uwsgi/gencode-remote create mode 100644 type/__netbox_uwsgi/man.rst create mode 100755 type/__netbox_uwsgi/manifest create mode 100644 type/__netbox_uwsgi/parameter/default/bind-to create mode 100644 type/__netbox_uwsgi/parameter/optional_multiple diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index af4d8c8..0153579 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -89,7 +89,7 @@ EOF fi # meta - printf "configuration\n" >> "$__messages_out" + printf "configured\n" >> "$__messages_out" changes=yes fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 2af2bef..6bb1e36 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -176,7 +176,7 @@ MESSAGES installed $VERSION Netbox was fresh installed or updated. The new version number is appended. -configuration +configured Some configuration files got updated and therefore the service was restarted. This message will not be echoed if configuration got updated due a standard installation. diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 6958976..e424c22 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -16,7 +16,7 @@ fi # configuration changes if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then do_restart=yes - printf "configuration\n" >> "$__messages_out" + printf "configured\n" >> "$__messages_out" fi # application @@ -27,10 +27,9 @@ fi # restart gunicorn -if [ "$do_restart"]; then +if [ "$do_restart" ]; then cat << EOF # Restart service service gunicorn-netbox restart EOF - echo restarted >> "$__messages_out" fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 38ac4a5..43f5864 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -34,7 +34,7 @@ MESSAGES updated $old to $new The version of the gunicorn software was updated from `$old` to `$new`. -configuration +configured Configuration for gunicorn changed. In both cases, and at messages from the `__netbox` type, it restarts the diff --git a/type/__netbox_uwsgi/explorer/installed b/type/__netbox_uwsgi/explorer/installed new file mode 100755 index 0000000..a2393d0 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show uwsgi | awk '/Version:/{print $2}' diff --git a/type/__netbox_uwsgi/explorer/upgradeable b/type/__netbox_uwsgi/explorer/upgradeable new file mode 100755 index 0000000..f4b0a22 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/upgradeable @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print latest version if availble +/opt/netbox/venv/bin/pip3 list --outdated | awk '$1 == "uwsgi" {print $3}' diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service new file mode 100644 index 0000000..cfce752 --- /dev/null +++ b/type/__netbox_uwsgi/files/netbox.service @@ -0,0 +1,23 @@ +[Unit] +Description=Netbox uWSGI WSGI Service +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +PIDFile=/var/tmp/netbox.pid +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/uwsgi --master --wsgi-file netbox/netbox/wsgi.py uwsgi.ini + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +Alias=netbox.service +WantedBy=multi-user.target diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh new file mode 100755 index 0000000..557a5fa --- /dev/null +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# Generates uwsgi config +# see https://uwsgi-docs.readthedocs.io/en/latest/Options.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF +[uwsgi] +; socket to bind +socket = $HOST + +; processes and threads +processes = $(( 2*cores + 1 )) +threads = 2 +EOF diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote new file mode 100755 index 0000000..5cc82d1 --- /dev/null +++ b/type/__netbox_uwsgi/gencode-remote @@ -0,0 +1,35 @@ +#!/bin/sh -e + +# not installed +if ! [ -s "$__object/explorer/installed" ]; then + echo "/opt/netbox/venv/bin/pip3 install uwsgi" + do_restart=yes + printf "installed\n" >> "$__messages_out" + +# updates available +elif [ -s "$__object/explorer/upgradeable" ]; then + echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + do_restart=yes + printf "upgraded\n" >> "$__messages_out" +fi + +# changed configuration +if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no messages cause this is obvious +fi + + +# restart uwsgi +if [ "$do_restart" ]; then + cat << EOF +# Restart service +service uwsgi-netbox restart +EOF +fi diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst new file mode 100644 index 0000000..5bf3160 --- /dev/null +++ b/type/__netbox_uwsgi/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_uwsgi - run netbox with uwsgi + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. + Defaults to ``127.0.0.1:3031``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +installed + The uwsgi service was installed. + +upgraded + The uwsgi service was upgraded. + +configured + The uwsgi configuration got updated. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_uwsgi + + # with special bind + require="__netbox" __netbox_uwsgi --bind-to 0.0.0.0:3032 \ + --bind-to 0.0.0.0:3033 + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest new file mode 100755 index 0000000..779f93b --- /dev/null +++ b/type/__netbox_uwsgi/manifest @@ -0,0 +1,18 @@ +#!/bin/sh -e + +HOST="$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" + +# uwsgi config file +# TODO maybe patching with __key_value cause of .ini ? +__file /opt/netbox/uwsgi.ini \ + --mode 644 --source "$__object/files/uwsgi.ini" + +# install service file +__systemd_unit uwsgi-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_uwsgi/parameter/default/bind-to b/type/__netbox_uwsgi/parameter/default/bind-to new file mode 100644 index 0000000..c696456 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:3031 diff --git a/type/__netbox_uwsgi/parameter/optional_multiple b/type/__netbox_uwsgi/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to From 9d8b3ebe743685daac13d54ac3bf2449517c3318 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 1 Sep 2020 10:02:08 +0200 Subject: [PATCH 088/156] __netbox: add wrapper service to manage the wsgi services The wrapper service will "control" the services added from the __netbox_* types to provide a general interface. This is more dynamic than the alias approach used previously. Through this, it is possible to handle multiple wsgi services for netbox - if this works .. See as a reference: http://alesnosek.com/blog/2016/12/04/controlling-a-multi-service-application-with-systemd/ --- type/__netbox/files/netbox.service | 12 ++++++++++++ type/__netbox/gencode-remote | 1 + type/__netbox/manifest | 10 ++++++---- type/__netbox_gunicorn/files/netbox.service | 3 ++- type/__netbox_gunicorn/gencode-remote | 6 ------ type/__netbox_gunicorn/man.rst | 3 +-- type/__netbox_uwsgi/files/netbox.service | 3 ++- type/__netbox_uwsgi/gencode-remote | 6 ------ type/__netbox_uwsgi/man.rst | 3 +-- 9 files changed, 25 insertions(+), 22 deletions(-) create mode 100644 type/__netbox/files/netbox.service diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service new file mode 100644 index 0000000..88baaeb --- /dev/null +++ b/type/__netbox/files/netbox.service @@ -0,0 +1,12 @@ +[Unit] +Description=NetBox Service Wrapper +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 0153579..d598aa8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -98,6 +98,7 @@ fi if [ "$changes" = "yes" ]; then cat << EOF # Restart service. +service netbox restart service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 079ea48..95688d3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,7 +204,9 @@ $SECRET_KEY SECRET -# Upload systemd unit for worker service -__systemd_unit netbox-rq.service \ - --source "$__type/files/netbox-rq.service" \ - --enablement-state enabled +# Upload systemd unit for worker and wsgi service +for unit in netbox netbox-rq; do + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled +done diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 475ddc4..2154d27 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,6 +1,8 @@ [Unit] Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ +PartOf=netbox.service +After=netbox.service After=network-online.target Wants=network-online.target @@ -19,5 +21,4 @@ RestartSec=30 PrivateTmp=true [Install] -Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index e424c22..163bb1a 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -19,12 +19,6 @@ if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then printf "configured\n" >> "$__messages_out" fi -# application -if grep -q "^__netbox:" "$__messages_in"; then - do_restart=yes - # no message cause it should be obvious -fi - # restart gunicorn if [ "$do_restart" ]; then diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 43f5864..ac86261 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -37,8 +37,7 @@ updated $old to $new configured Configuration for gunicorn changed. -In both cases, and at messages from the `__netbox` type, it restarts the -service to using the up-to-date version. +In both cases, it restarts the service to use the up-to-date version. EXAMPLES diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index cfce752..bc10571 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -1,6 +1,8 @@ [Unit] Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ +PartOf=netbox.service +After=netbox.service After=network-online.target Wants=network-online.target @@ -19,5 +21,4 @@ RestartSec=30 PrivateTmp=true [Install] -Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 5cc82d1..3127b8b 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -19,12 +19,6 @@ if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then printf "configured\n" >> "$__messages_out" fi -# application -if grep -q "^__netbox:" "$__messages_in"; then - do_restart=yes - # no messages cause this is obvious -fi - # restart uwsgi if [ "$do_restart" ]; then diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 5bf3160..f328496 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -39,8 +39,7 @@ upgraded configured The uwsgi configuration got updated. -In both cases, and at messages from the `__netbox` type, it restarts the -service to using the up-to-date version. +In all cases, it restarts the service to use the up-to-date version. EXAMPLES From facb5a64d3824df095c6374a39f3a78482987046 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 1 Sep 2020 12:40:05 +0200 Subject: [PATCH 089/156] __netbox{,_gunicorn}: disabled shellcheck warnings Shellcheck warned about creating content for a python array. As the string will be printed literally into the config, the warning does not match to the current case. --- type/__netbox/manifest | 3 ++- type/__netbox_gunicorn/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 95688d3..aef50c1 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -38,9 +38,10 @@ export DATABASE_PORT # list of hosts ALLOWED_HOSTS="" while read -r hostname; do + # shellcheck disable=SC2089 ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," done < "$__object/parameter/host" -export ALLOWED_HOSTS +export ALLOWED_HOSTS # shellcheck disable=SC2090 if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index dab8181..d61cbe6 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -2,9 +2,10 @@ HOST="" while read -r host; do + # shellcheck disable=SC2089 HOST="$HOST '$host'," done < "$__object/parameter/bind-to" -export HOST +export HOST # shellcheck disable=SC2090 # process template mkdir "$__object/files" From dbc91cb33914e0b3c899dd169d61613f7a766c2a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Sep 2020 07:50:12 +0200 Subject: [PATCH 090/156] __netbox*: updated man pages --- type/__netbox/man.rst | 20 +++++++++++++++++--- type/__netbox_gunicorn/man.rst | 13 ++++++++++--- type/__netbox_uwsgi/man.rst | 13 ++++++++++--- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 6bb1e36..a3f283c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -12,8 +12,12 @@ This (singleton) type installs and configures a NetBox instance, a web application to help manage and document computer networks. It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. -Netbox will be run via `gnuicorn` as WSGI service. It setup systemd unit files -for the services `netbox` and `netbox-rq`. +It setup systemd unit files for the services `netbox` and `netbox-rq`. To +access the application through WSGI, uWSGI or Gunicorn can be used. The setup +can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. + +The Gunicorn setup is recommended from the NetBox documentation. Consult each +manual page to decide. The types must be called after the `__netbox` type. REQUIRED PARAMETERS @@ -36,6 +40,7 @@ host Hostname (domain or IP address) on which the application is served. Multiple hostnames are possible; given as multiple arguments. + OPTIONAL PARAMETERS ------------------- secret-key @@ -147,6 +152,7 @@ scripts-root needed. By default, it will be stored into the installation directory (``/opt/netbox/netbox/netbox/scripts``). + BOOLEAN PARAMETERS ------------------ redis-ssl @@ -171,6 +177,7 @@ update-notify Enables the NetBox version check for new upstream updates. It checks every 24 hours for new releases and notify the admin users in the gui if any. + MESSAGES -------- installed $VERSION @@ -199,6 +206,8 @@ EXAMPLES --ldap-group-base "ou=groups,dc=domain,dc=tld" \ --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + # using recommended gunicorn setup + require="__netbox" __netbox_gunicorn NOTES @@ -216,9 +225,14 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. + SEE ALSO -------- -- `NetBox documentation `_ +`NetBox documentation `_ + +:strong:`cdist-type__netbox_gunicorn`\ (7) +:strong:`cdist-type__netbox_uwsgi`\ (7) + AUTHORS ------- diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index ac86261..c51be0c 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -3,12 +3,16 @@ cdist-type__netbox_uwsgi(7) NAME ---- -cdist-type__netbox_gunicorn - run netbox with gunicorn +cdist-type__netbox_gunicorn - Run NetBox with Gunicorn DESCRIPTION ----------- -This space intentionally left blank. +This (singleton) type installs Gunicorn into the NetBox `python-venv` to host +the NetBox WSGI application. It provides the application as HTTP over the given +sockets. Static content must be served independent of Gunicorn. The Gunicorn +daemon is available as the `gunicorn-netbox` systemd service, but also +available via the `netbox` wrapper service. REQUIRED PARAMETERS @@ -58,7 +62,10 @@ EXAMPLES SEE ALSO -------- -:strong:`__netbox`\ (7) +`Gunicorn Documentation `_ + +:strong:`cdist-type__netbox`\ (7) +:strong:`cdist-type__netbox_uwsgi`\ (7) AUTHORS diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index f328496..9d5ced7 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -3,12 +3,16 @@ cdist-type__netbox_uwsgi(7) NAME ---- -cdist-type__netbox_uwsgi - run netbox with uwsgi +cdist-type__netbox_uwsgi - Run NetBox with uWSGI DESCRIPTION ----------- -This space intentionally left blank. +This (singleton) type installs uWSGI into the NetBox `python-venv`. It hosts +the NetBox WSGI application over the WSGI protocol. A further server must be +installed to provide it as HTTP. This application is available via the +`uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper +service, too. REQUIRED PARAMETERS @@ -58,7 +62,10 @@ EXAMPLES SEE ALSO -------- -:strong:`TODO`\ (7) +`uWSGI Documentation `_ + +:strong:`cdist-type__netbox`\ (7) +:strong:`cdist-type__netbox_gunicorn`\ (7) AUTHORS From 2805b6beffc61c380ebffe26a487bbdf587c992b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Sep 2020 09:15:45 +0200 Subject: [PATCH 091/156] __netbox{,_gunicorn}: fix shellcheck directives --- type/__netbox/manifest | 3 ++- type/__netbox_gunicorn/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index aef50c1..273423e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -41,7 +41,8 @@ while read -r hostname; do # shellcheck disable=SC2089 ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," done < "$__object/parameter/host" -export ALLOWED_HOSTS # shellcheck disable=SC2090 +# shellcheck disable=SC2090 +export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index d61cbe6..25ea452 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -5,7 +5,8 @@ while read -r host; do # shellcheck disable=SC2089 HOST="$HOST '$host'," done < "$__object/parameter/bind-to" -export HOST # shellcheck disable=SC2090 +# shellcheck disable=SC2090 +export HOST # process template mkdir "$__object/files" From 243e34f0a5e6d7e012c3eee3865cc995432f6f07 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 11:20:20 +0200 Subject: [PATCH 092/156] __netbox_uwsgi: enable further protocols to bind to Enables multiple protocols like fastcgi or HTTP to bind to. This makes it more flexible to use. Also, a little fix for __netbox was done: correctly output a error msg. --- type/__netbox/manifest | 2 +- type/__netbox_uwsgi/files/uwsgi.ini.sh | 29 +++++++++++++++++-- type/__netbox_uwsgi/man.rst | 29 +++++++++++++++---- .../parameter/optional_multiple | 4 +++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 273423e..cb6062e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -127,7 +127,7 @@ fi export SMTP_USE_SSL if [ -f "$__object/parameter/smtp-use-tls" ]; then if [ "$SMTP_USE_SSL" = "True" ]; then - echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" + echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" >&2 exit 2 fi SMTP_USE_TLS="True" diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 557a5fa..324b150 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -2,6 +2,18 @@ # Generates uwsgi config # see https://uwsgi-docs.readthedocs.io/en/latest/Options.html +# or https://uwsgi-docs-additions.readthedocs.io/en/latest/Options.html + +# params: +# 1: parameter name +# 2: parameter value file +# +# output: the lines for the configuration option +multi_options() { + while read -r line; do + printf "%s = %s\n" "$1" "$line" + done < "$2" +} # fix missing $__explorer # see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 @@ -13,8 +25,21 @@ cores="$(cat "$__explorer/cpu_cores")" cat << EOF [uwsgi] -; socket to bind -socket = $HOST +; socket(s) to bind +EOF + +# special protocol to bind +for param in $(find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print); do + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" + socket_changes=yes +done +# else, default bind to +if [ -z "$socket_changes" ]; then + multi_options "socket" "$__object/parameter/bind-to" +fi + + +cat << EOF ; processes and threads processes = $(( 2*cores + 1 )) diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 9d5ced7..7501573 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -9,10 +9,11 @@ cdist-type__netbox_uwsgi - Run NetBox with uWSGI DESCRIPTION ----------- This (singleton) type installs uWSGI into the NetBox `python-venv`. It hosts -the NetBox WSGI application over the WSGI protocol. A further server must be -installed to provide it as HTTP. This application is available via the -`uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper -service, too. +the NetBox WSGI application via the WSGI protocol. A further server must be +installed to provide it as HTTP and serve static content. It supports multiple +protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This +application is available via the `uwsgi-netbox` systemd service. It is +controllable via the `netbox` wrapper service, too. REQUIRED PARAMETERS @@ -24,7 +25,15 @@ OPTIONAL PARAMETERS ------------------- bind-to The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. - Defaults to ``127.0.0.1:3031``. + Defaults to ``127.0.0.1:3031``. Can be set multiple times. + +uwsgi-bind +http-bind +fastcgi-bind +scgi-bind + Bind the application to a specific protocol instead of implicit uwsgi via + ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must + be a UNIX/TCP socket. Can be set multiple times. BOOLEAN PARAMETERS @@ -55,10 +64,18 @@ EXAMPLES __netbox $args require="__netbox" __netbox_uwsgi - # with special bind + # with multiple binds + __netbox $args require="__netbox" __netbox_uwsgi --bind-to 0.0.0.0:3032 \ --bind-to 0.0.0.0:3033 + # with multiple protocols + # parameter `--bind-to` will be ignored + __netbox $args + require="__netbox" __netbox_uwsgi --uwsgi-bind 0.0.0.0:3031 \ + --http-bind 0.0.0.0:8080 \ + --fastcgi-bind 1.2.3.4:5678 + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/parameter/optional_multiple b/type/__netbox_uwsgi/parameter/optional_multiple index 331c077..3f3e7d4 100644 --- a/type/__netbox_uwsgi/parameter/optional_multiple +++ b/type/__netbox_uwsgi/parameter/optional_multiple @@ -1 +1,5 @@ bind-to +uwsgi-bind +http-bind +fastcgi-bind +scgi-bind From af91b6e3a1ad15f6579983e25570a0bda8796a9b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 11:26:15 +0200 Subject: [PATCH 093/156] __netbox_uwsgi: mark as singleton This should have been done much earlier .. :/ --- type/__netbox_uwsgi/singleton | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 type/__netbox_uwsgi/singleton diff --git a/type/__netbox_uwsgi/singleton b/type/__netbox_uwsgi/singleton new file mode 100644 index 0000000..e69de29 From 314a0d4d8ef6235c70fbf3d7b1421f1b34cb18c2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 13:53:05 +0200 Subject: [PATCH 094/156] __netbox: kill all running venv processes To avoid aborts because of the python venv could not be updated by killing all processes that uses the venv. It will be done all times to prevent any error, because it could not be reliably detected if the type installs or updates NetBox. --- type/__netbox/gencode-remote | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d598aa8..f200408 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -30,6 +30,17 @@ tar xf '$archive' cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" +EOF + + # Stop everything in the pyenv to update + cat << EOF +# Try to kill everything in the venv +systemctl stop netbox gunicorn-netbox uwsgi-netbox || true +ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true + +EOF + + cat << EOF # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' From 755bd9098ede2dbad6c557f7c4983037fe3aab22 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 15:18:49 +0200 Subject: [PATCH 095/156] __netbox*: update systemd service files The `uwsgi-netbox` service now works, also the `netbox` wrapper service. The PID file was removed from the Gunicorn service as it is not required and a bit more efford to move it to `/run/` due to permissions. Generally, all depend on `network.target` instead of `network-online.target` now, and signals for reload, stop and kill were added (especially required the uwsgi service). --- type/__netbox/files/netbox.service | 5 +++-- type/__netbox_gunicorn/files/netbox.service | 14 ++++++++------ type/__netbox_uwsgi/files/netbox.service | 16 +++++++++------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service index 88baaeb..738e178 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox/files/netbox.service @@ -1,12 +1,13 @@ [Unit] Description=NetBox Service Wrapper Documentation=https://netbox.readthedocs.io/en/stable/ -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=oneshot RemainAfterExit=yes +ExecStart=/bin/true [Install] WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 2154d27..55f3549 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -3,22 +3,24 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service After=netbox.service -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=simple User=netbox Group=netbox -PIDFile=/var/tmp/netbox.pid WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi +ExecStart=/opt/netbox/venv/bin/gunicorn --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi +# signals: https://docs.gunicorn.org/en/stable/signals.html +ExecReload=kill -HUP $MAINPID +ExecStop=kill -TERM $MAINPID +KillSignal=SIGQUIT Restart=on-failure RestartSec=30 -PrivateTmp=true [Install] -WantedBy=multi-user.target +WantedBy=netbox.service diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index bc10571..10ea734 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -3,22 +3,24 @@ Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service After=netbox.service -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=simple User=netbox Group=netbox -PIDFile=/var/tmp/netbox.pid -WorkingDirectory=/opt/netbox +WorkingDirectory=/opt/netbox/netbox/ -ExecStart=/opt/netbox/venv/bin/uwsgi --master --wsgi-file netbox/netbox/wsgi.py uwsgi.ini +ExecStart=/opt/netbox/venv/bin/uwsgi --master --module netbox.wsgi uwsgi.ini +# signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi +ExecReload=kill -HUP $MAINPID +ExecStop=kill -INT $MAINPID +KillSignal=SIGQUIT Restart=on-failure RestartSec=30 -PrivateTmp=true [Install] -WantedBy=multi-user.target +WantedBy=netbox.service From 5a403de0571b57c0da07824c91d703a21231e303 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 19:09:34 +0200 Subject: [PATCH 096/156] __netbox_uwsgi: fix shellcheck Fixed shellcheck cause of the find command. This required a bigger change due to variables and subshell. Now, input is used through a here-document. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 10 +++++++--- type/__netbox_uwsgi/manifest | 3 --- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 324b150..0f85804 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -29,10 +29,14 @@ cat << EOF EOF # special protocol to bind -for param in $(find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print); do +while read -r param; do multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" - socket_changes=yes -done + socket_changes="yes" + +done << INPUT # here-doc cause of SC2031 +$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print ) +INPUT + # else, default bind to if [ -z "$socket_changes" ]; then multi_options "socket" "$__object/parameter/bind-to" diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index 779f93b..3d9c1b4 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,8 +1,5 @@ #!/bin/sh -e -HOST="$__object/parameter/bind-to" -export HOST - # process template mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" From ffba3ae776ac4e4bd67d4812b8073c2e81a69cdd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 21:48:04 +0200 Subject: [PATCH 097/156] __netbox_uwsgi: add --serve-static parameter New option to directly serve static content via uWSGI. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 11 +++++++++++ type/__netbox_uwsgi/man.rst | 11 ++++++++++- type/__netbox_uwsgi/manifest | 6 ++++++ type/__netbox_uwsgi/parameter/boolean | 1 + 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 type/__netbox_uwsgi/parameter/boolean diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 0f85804..4abc59a 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -43,9 +43,20 @@ if [ -z "$socket_changes" ]; then fi +# multi-process settings cat << EOF ; processes and threads processes = $(( 2*cores + 1 )) threads = 2 EOF + + +# optional mapping of static content +if [ "$STATIC_MAP" != "" ]; then + cat << EOF + +; map static content +static-map = /static=/opt/netbox/netbox/static +EOF +fi diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 7501573..f5eff0b 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -38,7 +38,12 @@ scgi-bind BOOLEAN PARAMETERS ------------------ -None. +serve-static + Setup uWSGI to serve the static content, too. This is generally not + recommended for real production setups, as it is the job of the reverse + proxy server, who will thread it as static cachable content. This option + is only recommended for small setups or direct usage of the uWSGI socket + like using it as standalone HTTP server for NetBox. MESSAGES @@ -76,6 +81,10 @@ EXAMPLES --http-bind 0.0.0.0:8080 \ --fastcgi-bind 1.2.3.4:5678 + # as standalone server + __netbox $args + require="__netbox" __netbox_uwsgi --serve-static --http-bind 0.0.0.0:80 + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index 3d9c1b4..bb4f23f 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,5 +1,11 @@ #!/bin/sh -e +# *bind* parameters are directly processed in the gen script +if [ -f "$__object/parameter/serve-static" ]; then + STATIC_MAP="yes" + export STATIC_MAP +fi + # process template mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" diff --git a/type/__netbox_uwsgi/parameter/boolean b/type/__netbox_uwsgi/parameter/boolean new file mode 100644 index 0000000..aa08196 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/boolean @@ -0,0 +1 @@ +serve-static From 090a8f015e86f8dd0385b0b37a234a2538c254da Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:08:37 +0200 Subject: [PATCH 098/156] __netbox_uwsgi: fix uwsgi netbox service file Revert working directory changes as the configuration file still needs to be accessable. An absolute path would work, too, but it is not the preferred way. To still work with the python wsgi application, `--chdir` is used. --- type/__netbox_uwsgi/files/netbox.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index 10ea734..8d41cd0 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -11,9 +11,9 @@ Type=simple User=netbox Group=netbox -WorkingDirectory=/opt/netbox/netbox/ +WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/uwsgi --master --module netbox.wsgi uwsgi.ini +ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi ExecReload=kill -HUP $MAINPID ExecStop=kill -INT $MAINPID From 9d330a91bc5bdbdbe69a4db98ddad0eeaed344cb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:27:42 +0200 Subject: [PATCH 099/156] __netbox_{gunicorn,uwsgi}: add netbox ownership to config file --- type/__netbox_gunicorn/manifest | 3 ++- type/__netbox_uwsgi/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index 25ea452..b72b8fd 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -14,7 +14,8 @@ mkdir "$__object/files" # gunicorn config file __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__object/files/gunicorn.py" + --mode 644 --owner netbox \ + --source "$__object/files/gunicorn.py" # install service file __systemd_unit gunicorn-netbox.service \ diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index bb4f23f..b6c0308 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -13,7 +13,8 @@ mkdir "$__object/files" # uwsgi config file # TODO maybe patching with __key_value cause of .ini ? __file /opt/netbox/uwsgi.ini \ - --mode 644 --source "$__object/files/uwsgi.ini" + --mode 644 --owner netbox \ + --source "$__object/files/uwsgi.ini" # install service file __systemd_unit uwsgi-netbox.service \ From b10f6b71d051377f023fd6e96abbc00ff06e5d5d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:28:48 +0200 Subject: [PATCH 100/156] __netbox: avoid useless generated code-remote Because `set -e` got printed all the time, the type __netbox always had some generated code for the remote side. This line was removed because this is already done by cdist when executing the code-remote script. Rather, the exit-on-error option was set to some scirpts (two ..). --- type/__netbox/gencode-remote | 4 +--- type/__netbox/manifest | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index f200408..9ae6b5b 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -1,6 +1,4 @@ -#!/bin/sh - -echo "set -e" +#!/bin/sh -e old_version="$(cat "$__object/explorer/version")" VERSION=$(cat "$__object/parameter/version") diff --git a/type/__netbox/manifest b/type/__netbox/manifest index cb6062e..5f1762a 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e os=$(cat "$__global/explorer/os") From 3b07a660b3a7dd442e3184600fad25ed27769a5f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:55:43 +0200 Subject: [PATCH 101/156] __netbox_uwsgi: add hint to the --serve-static parameter --- type/__netbox_uwsgi/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index f5eff0b..6826cca 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -45,6 +45,11 @@ serve-static is only recommended for small setups or direct usage of the uWSGI socket like using it as standalone HTTP server for NetBox. + **Hint**: This parameter does not work in junction with the `__netbox` + parameter ``--basepath``. It is because this type does not know the + parameter value and this case is very unlikly to happen; although an + implementation is not difficult. + MESSAGES -------- From c9e4e8d7dce88c3f82a4bbf053b174d88a70a4ce Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 13:44:03 +0200 Subject: [PATCH 102/156] __netbox_{gunicorn,uwsgi}: add state parameter Adds the --state parameter to both types. With it, the transition between both types can be done smothly. --- type/__netbox_gunicorn/gencode-remote | 59 +++++++++++------ type/__netbox_gunicorn/man.rst | 40 ++++++++++-- type/__netbox_gunicorn/manifest | 64 ++++++++++++++----- .../__netbox_gunicorn/parameter/default/state | 1 + type/__netbox_gunicorn/parameter/optional | 1 + type/__netbox_uwsgi/gencode-remote | 58 +++++++++++------ type/__netbox_uwsgi/man.rst | 32 +++++++++- type/__netbox_uwsgi/manifest | 62 +++++++++++++----- type/__netbox_uwsgi/parameter/default/state | 1 + type/__netbox_uwsgi/parameter/optional | 1 + 10 files changed, 244 insertions(+), 75 deletions(-) create mode 100644 type/__netbox_gunicorn/parameter/default/state create mode 100644 type/__netbox_gunicorn/parameter/optional create mode 100644 type/__netbox_uwsgi/parameter/default/state create mode 100644 type/__netbox_uwsgi/parameter/optional diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 163bb1a..0da92f0 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -1,29 +1,50 @@ #!/bin/sh -e -curr_installed="$(cat "$__object/explorer/installed")" -should_installed="$(cat "$__object/explorer/should_installed")" +# control state +state="$(cat "$__object/parameter/state")" -# gunicorn version change -if [ "$curr_installed" != "$should_installed" ]; then - # (re)installing gunicorn - echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" +case "$state" in + # install gunicorn + enabled|disabled) + curr_installed="$(cat "$__object/explorer/installed")" + should_installed="$(cat "$__object/explorer/should_installed")" - do_restart=yes - printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ - >> "$__messages_out" -fi + # gunicorn version change + if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" -# configuration changes -if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then - do_restart=yes - printf "configured\n" >> "$__messages_out" -fi + if [ "$curr_installed" != "" ]; then + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" + else + printf "installed\n" >> "$__messages_out" + fi + do_restart=yes + fi + + # configuration changes + if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" + fi -# restart gunicorn -if [ "$do_restart" ]; then - cat << EOF + # restart gunicorn + if [ "$do_restart" ] && [ "$state" != "disabled" ]; then + cat << EOF # Restart service service gunicorn-netbox restart EOF -fi + fi + ;; + + # uninstall + absent) + # check if installed + if [ -s "$__object/explorer/installed" ]; then + # service already disabled + echo "/opt/netbox/venv/bin/pip3 uninstall -y gunicorn" + printf "uninstalled\n" >> "$__messages_out" + fi +esac diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index c51be0c..1c09d78 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -1,5 +1,5 @@ -cdist-type__netbox_uwsgi(7) -=========================== +cdist-type__netbox_gunicorn(7) +============================== NAME ---- @@ -22,6 +22,20 @@ None. OPTIONAL PARAMETERS ------------------- +state + Represents the state of the Gunciron application. Defaults to ``enabled``. + + enabled + The Gunicorn service is enabled and running. + disabled + The Gunicorn service is installed, but disabled. + absent + The uWSGI service is not installed and all configuration removed. + + This type does not guarantee anything about the running state of the + service. To be sure about the service is stopped or not, use the type + :strong:`cdist-type__systemd_service`\ (7) after this execution. + bind-to The hosts the gunicorn socket should be bind to. Formats are `IP`, `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple @@ -35,13 +49,20 @@ None. MESSAGES -------- -updated $old to $new +installed + The software was installed. + +upgraded $old to $new The version of the gunicorn software was updated from `$old` to `$new`. configured Configuration for gunicorn changed. -In both cases, it restarts the service to use the up-to-date version. +uninstalled + The Gunicorn application was removed. + +In all cases where the application is still present, it restarts the service to +use the up-to-date version. EXAMPLES @@ -59,6 +80,17 @@ EXAMPLES --bind-to 0.0.0.0:8001 \ --bind-to 1.2.3.4:5678 + # replace uwsgi with gunicorn + __netbox $args + require="__netbox" __netbox_uwsgi --state absent + # it should depend on __netbox_uwsgi if they use the same socket + require="__netbox_uwsgi" __netbox_gunicorn --state enabled + + # be sure the service is disabled + __netbox $args + require="__netbox" __netbox_gunicorn --state disabled + require="__netbox_gunicorn" __systemd_service gunicorn-netbox --state stopped + SEE ALSO -------- diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index b72b8fd..2e5ccce 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -1,23 +1,55 @@ #!/bin/sh -e +# __netbox_gunicorn/manifest -HOST="" -while read -r host; do - # shellcheck disable=SC2089 - HOST="$HOST '$host'," -done < "$__object/parameter/bind-to" -# shellcheck disable=SC2090 -export HOST +# Check states +state="" +unit_state="" +param_state="$(cat "$__object/parameter/state")" -# process template -mkdir "$__object/files" -"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" +case "$param_state" in + enabled|disabled) + state="present" + unit_state="$param_state" + ;; + + absent) + state="absent" + unit_state="disabled" + ;; + + *) + # does not exist + printf "The state '%s' does not exist, can't continue!\n" "$param_state" >&2 + exit 2 + ;; +esac + + +if [ "$state" = "present" ]; then + HOST="" + while read -r host; do + # shellcheck disable=SC2089 + HOST="$HOST '$host'," + done < "$__object/parameter/bind-to" + # shellcheck disable=SC2090 + export HOST + + # process template + mkdir "$__object/files" + "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + + # gunicorn config file + __file /opt/netbox/gunicorn.py \ + --mode 644 --owner netbox \ + --source "$__object/files/gunicorn.py" + +else + # absent config file + __file /opt/netbox/gunicorn.py --state absent +fi -# gunicorn config file -__file /opt/netbox/gunicorn.py \ - --mode 644 --owner netbox \ - --source "$__object/files/gunicorn.py" # install service file __systemd_unit gunicorn-netbox.service \ - --source "$__type/files/netbox.service" \ - --enablement-state enabled --restart + --state "$state" --enablement-state "$unit_state" \ + --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_gunicorn/parameter/default/state b/type/__netbox_gunicorn/parameter/default/state new file mode 100644 index 0000000..86981e6 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/state @@ -0,0 +1 @@ +enabled diff --git a/type/__netbox_gunicorn/parameter/optional b/type/__netbox_gunicorn/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional @@ -0,0 +1 @@ +state diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 3127b8b..9b6a603 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -1,29 +1,47 @@ #!/bin/sh -e -# not installed -if ! [ -s "$__object/explorer/installed" ]; then - echo "/opt/netbox/venv/bin/pip3 install uwsgi" - do_restart=yes - printf "installed\n" >> "$__messages_out" +# control state +state="$(cat "$__object/parameter/state")" -# updates available -elif [ -s "$__object/explorer/upgradeable" ]; then - echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" - do_restart=yes - printf "upgraded\n" >> "$__messages_out" -fi +case "$state" in + # install uwsgi + enabled|disabled) + # not installed + if ! [ -s "$__object/explorer/installed" ]; then + echo "/opt/netbox/venv/bin/pip3 install uwsgi" + do_restart=yes + printf "installed\n" >> "$__messages_out" -# changed configuration -if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then - do_restart=yes - printf "configured\n" >> "$__messages_out" -fi + # updates available + elif [ -s "$__object/explorer/upgradeable" ]; then + echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + do_restart=yes + printf "upgraded\n" >> "$__messages_out" + fi + + # changed configuration + if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" + fi -# restart uwsgi -if [ "$do_restart" ]; then - cat << EOF + # restart uwsgi + if [ "$do_restart" ] && [ "$state" != "disabled" ]; then + cat << EOF # Restart service service uwsgi-netbox restart EOF -fi + fi + ;; + + # uninstall + absent) + # check if installed + if [ -s "$__object/explorer/installed" ]; then + # service already disabled + echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi" + printf "uninstalled\n" >> "$__messages_out" + fi + ;; +esac diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 6826cca..3fb8515 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -23,6 +23,21 @@ None. OPTIONAL PARAMETERS ------------------- +state + Represents the state of the uWSGI application. Defaults to ``enabled``. + + enabled + The uWSGI service is enabled and running. + disabled + The uWSGI service is installed, but disabled. + absent + The uWSGI service is not installed and all configuration removed. + + This type does not guarantee anything about the running state of the + service. To be sure about the service is stopped or not, use the type + :strong:`cdist-type__systemd_service`\ (7) after this execution. + + bind-to The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. Defaults to ``127.0.0.1:3031``. Can be set multiple times. @@ -62,7 +77,11 @@ upgraded configured The uwsgi configuration got updated. -In all cases, it restarts the service to use the up-to-date version. +uninstalled + The uWSGI application was removed. + +In all cases where the application is still present, it restarts the service to +use the up-to-date version. EXAMPLES @@ -90,6 +109,17 @@ EXAMPLES __netbox $args require="__netbox" __netbox_uwsgi --serve-static --http-bind 0.0.0.0:80 + # replace gunicorn with uwsgi + __netbox $args + require="__netbox" __netbox_gunicorn --state absent + # it should depend on __netbox_gunicorn if they use the same socket + require="__netbox_gunicorn" __netbox_uwsgi --state enabled + + # be sure the service is disabled + __netbox $args + require="__netbox" __netbox_uwsgi --state disabled + require="__netbox_uwsgi" __systemd_service uwsgi-netbox --state stopped + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index b6c0308..c5885c9 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,22 +1,54 @@ #!/bin/sh -e +# __netbox_uwsgi/manifest -# *bind* parameters are directly processed in the gen script -if [ -f "$__object/parameter/serve-static" ]; then - STATIC_MAP="yes" - export STATIC_MAP +# Check states +state="" +unit_state="" +param_state="$(cat "$__object/parameter/state")" + +case "$param_state" in + enabled|disabled) + state="present" + unit_state="$param_state" + ;; + + absent) + state="absent" + unit_state="disabled" + ;; + + *) + # does not exist + printf "The state '%s' does not exist, can't continue!\n" "$param_state" >&2 + exit 2 + ;; +esac + + +if [ "$state" = "present" ]; then + # *bind* parameters are directly processed in the gen script + if [ -f "$__object/parameter/serve-static" ]; then + STATIC_MAP="yes" + export STATIC_MAP + fi + + # process template + mkdir "$__object/files" + "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" + + # uwsgi config file + # TODO maybe patching with __key_value cause of .ini ? + __file /opt/netbox/uwsgi.ini \ + --mode 644 --owner netbox \ + --source "$__object/files/uwsgi.ini" + +else + # absent config file + __file /opt/netbox/uwsgi.ini --state absent fi -# process template -mkdir "$__object/files" -"$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" - -# uwsgi config file -# TODO maybe patching with __key_value cause of .ini ? -__file /opt/netbox/uwsgi.ini \ - --mode 644 --owner netbox \ - --source "$__object/files/uwsgi.ini" # install service file __systemd_unit uwsgi-netbox.service \ - --source "$__type/files/netbox.service" \ - --enablement-state enabled --restart + --state "$state" --enablement-state "$unit_state" \ + --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/parameter/default/state b/type/__netbox_uwsgi/parameter/default/state new file mode 100644 index 0000000..86981e6 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/state @@ -0,0 +1 @@ +enabled diff --git a/type/__netbox_uwsgi/parameter/optional b/type/__netbox_uwsgi/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__netbox_uwsgi/parameter/optional @@ -0,0 +1 @@ +state From 0afc7136f8e4f304f179965f229ed368ce73abb6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 18:39:07 +0200 Subject: [PATCH 103/156] __netbox_uwsgi: fix if only --bind-to given If the here-document is empty because no file could be found, an empty line will be read. The new line of code will skip in those cases. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 4abc59a..7835a3e 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -30,6 +30,8 @@ EOF # special protocol to bind while read -r param; do + if [ -z "$param" ]; then continue; fi # ignore empty lines from the here-doc + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" socket_changes="yes" From eed3515424668b8684058b0d02368746ee5f5c06 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Sep 2020 17:10:59 +0200 Subject: [PATCH 104/156] __netbox: add --ldap-staff-group parameter Required to get "admin access". --- type/__netbox/files/ldap_config.py.sh | 15 +++++++++++---- type/__netbox/man.rst | 4 ++++ type/__netbox/manifest | 4 ++++ type/__netbox/parameter/optional | 1 + 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index a49d800..4e6b099 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -53,6 +53,8 @@ AUTH_LDAP_GROUP_TYPE = PosixGroupType() # Mirror LDAP group assignments. AUTH_LDAP_MIRROR_GROUPS = True +# For more granular permissions, map LDAP groups to Django groups. +AUTH_LDAP_FIND_GROUP_PERMS = True EOF if [ "$LDAP_REQUIRE_GROUP" != "" ]; then @@ -63,13 +65,18 @@ AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" EOF fi - if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then - cat << EOF + cat << EOF # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { - "is_superuser": "$LDAP_SUPERUSER_GROUP", -} EOF + # superuser + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + echo " \"is_superuser\": \"$LDAP_SUPERUSER_GROUP\"," fi + # staff user + if [ "$LDAP_STAFF_GROUP" != "" ]; then + echo " \"is_staff\": \"$LDAP_STAFF_GROUP\"," + fi + echo "}" fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index a3f283c..bc4bbc9 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -78,6 +78,10 @@ ldap-group-base ldap-require-group Group required to login. +ldap-staff-group + Make members of this group to "staff". This gives the users "Admin Access", + which means access to the "NetBox Administration" site. + ldap-superuser-group Make members of this groups superusers. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 5f1762a..3681ea0 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -89,6 +89,10 @@ if [ -f "$__object/parameter/ldap-superuser-group" ]; then LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") export LDAP_SUPERUSER_GROUP fi +if [ -f "$__object/parameter/ldap-staff-group" ]; then + LDAP_STAFF_GROUP="$(cat "$__object/parameter/ldap-staff-group")" + export LDAP_STAFF_GROUP +fi # export if base ldap parameters are used export USE_LDAP diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 47722e6..9495f7a 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -7,6 +7,7 @@ ldap-bind-password ldap-user-base ldap-group-base ldap-require-group +ldap-staff-group ldap-superuser-group redis-host redis-port From 3389752decfb48975f539b1702d37fe07d4e7755 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Sep 2020 17:23:22 +0200 Subject: [PATCH 105/156] __netbox: fix detection of the secret-key to use Mixed up a parameter. Should be correct now. --- type/__netbox/manifest | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 3681ea0..06d0c77 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -47,13 +47,13 @@ export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") elif [ -s "$__object/explorer/secretkey" ]; then + # take the key that is already used + SECRET_KEY="$(cat "$__object/explorer/secretkey")" +else # Can be done over netbox/generate_secret_key.py too, but it's to # complicated with the variable setup (can't generated right now!). # Generates a 50-character long key (without ' cause of python quotes) SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" -else - # take the key that is already used - SECRET_KEY="$(cat "$__object/explorer/secretkey")" fi export SECRET_KEY From 398a3da10eeaf67caa042b2a99a1a3e652b31e0f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 18:10:15 +0200 Subject: [PATCH 106/156] __netbox: fix gerneration random generated secret key Because `/dev/random` was used, the `cdist config` could hang a long time to get real random values. The pseudo-generated values through `/dev/urandom` are fully enought for the secret key. --- type/__netbox/manifest | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 06d0c77..4ad7901 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -50,10 +50,11 @@ elif [ -s "$__object/explorer/secretkey" ]; then # take the key that is already used SECRET_KEY="$(cat "$__object/explorer/secretkey")" else - # Can be done over netbox/generate_secret_key.py too, but it's to - # complicated with the variable setup (can't generated right now!). + # Can be done over netbox/generate_secret_key.py too, but it can't be + # generated right now where it's required (only if it's preloaded for + # this type to execute it now). # Generates a 50-character long key (without ' cause of python quotes) - SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" + SECRET_KEY="$(tr -cd '[:graph:]' < /dev/urandom | tr -d \' | head -c50)" fi export SECRET_KEY From 549feb87f96518a38ab7d3368bd3f2a2ed552d0d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 19:20:22 +0200 Subject: [PATCH 107/156] __netbox: fix --update-notify Template script for the `configuration.py` interpreted the boolean flag incorrectly and did the thing the user do not wanted. --- type/__netbox/files/configuration.py.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index ea73133..06a1c86 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -272,12 +272,12 @@ EOF if [ "$UPDATE_CHECK" != "" ]; then cat << EOF -RELEASE_CHECK_URL = None +RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' EOF else cat << EOF -RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' +RELEASE_CHECK_URL = None EOF fi From 5d437839f68da0c29787172ea3daa3170926f8d2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 19:52:03 +0200 Subject: [PATCH 108/156] __netbox: replaced secret key character set The used character set should now only contain characters used by the `netbox/generate_secret_key.py` program. Finially got correct escaping. --- type/__netbox/manifest | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 4ad7901..d754ae6 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -53,8 +53,9 @@ else # Can be done over netbox/generate_secret_key.py too, but it can't be # generated right now where it's required (only if it's preloaded for # this type to execute it now). - # Generates a 50-character long key (without ' cause of python quotes) - SECRET_KEY="$(tr -cd '[:graph:]' < /dev/urandom | tr -d \' | head -c50)" + # Generates a 50-character long key with the same character set like + # the helper script. Must escape the '-' to be no character range. + SECRET_KEY="$(tr -cd '!@#$%^&*(\-_=+)[:alnum:]' < /dev/urandom | head -c50)" fi export SECRET_KEY From dd167f075d25ecec4f1aca461658d2b93ff1ae8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 9 Sep 2020 19:08:46 +0200 Subject: [PATCH 109/156] __netbox*: fix service restart order Cause of corrupt databases if the services are restarted incorrectly, the order and dependencies are adjusted. Now, the `netbox-rq` service will be included in restarts of `netbox` and required for the WSGI servers that it must running. For these changes, the restart command of `__netbox` was adjusted. The other ones where edited too, to use the same command. All services now require redis and postgresql to be started before them to prevent any start order issues. If someone asked for what the RQ worker is required, see here: https://netbox.readthedocs.io/en/stable/additional-features/webhooks/#webhook-processing --- type/__netbox/files/netbox-rq.service | 7 ++++-- type/__netbox/files/netbox.service | 2 +- type/__netbox/gencode-remote | 7 +++--- type/__netbox/man.rst | 24 ++++++++++++++++++++- type/__netbox_gunicorn/files/netbox.service | 4 +++- type/__netbox_gunicorn/gencode-remote | 2 +- type/__netbox_uwsgi/files/netbox.service | 4 +++- type/__netbox_uwsgi/gencode-remote | 2 +- 8 files changed, 41 insertions(+), 11 deletions(-) diff --git a/type/__netbox/files/netbox-rq.service b/type/__netbox/files/netbox-rq.service index 77d7091..330e675 100644 --- a/type/__netbox/files/netbox-rq.service +++ b/type/__netbox/files/netbox-rq.service @@ -1,8 +1,11 @@ [Unit] Description=NetBox Request Queue Worker Documentation=https://netbox.readthedocs.io/en/stable/ -After=network-online.target -Wants=network-online.target +PartOf=netbox.service +Wants=network.target +After=netbox.service +After=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service index 738e178..68010e9 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox/files/netbox.service @@ -1,8 +1,8 @@ [Unit] Description=NetBox Service Wrapper Documentation=https://netbox.readthedocs.io/en/stable/ -After=network.target Wants=network.target +After=network.target [Service] Type=oneshot diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 9ae6b5b..0370949 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -105,9 +105,10 @@ fi # check for changes if [ "$changes" = "yes" ]; then + # to avoid database corruption at config changes, both services must be + # stopped at the same time (noted in the manual, too). cat << EOF -# Restart service. -service netbox restart -service netbox-rq restart +# Restart service. All required services are included with netbox.service. +systemctl restart netbox EOF fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index bc4bbc9..3330b1b 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -12,7 +12,11 @@ This (singleton) type installs and configures a NetBox instance, a web application to help manage and document computer networks. It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. -It setup systemd unit files for the services `netbox` and `netbox-rq`. To +It setup systemd unit files for the services `netbox` and `netbox-rq`. The +`netbox` service only wrap all netbox related services, e.g. restarting and +so one will be delegated to all related services. + +The application is still not accessable because WSGI server is required. To access the application through WSGI, uWSGI or Gunicorn can be used. The setup can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. @@ -229,6 +233,24 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. +If you change a configuration, the database may go corrupt if two instances of +the application are running with different configurations at the same time. +This most commonly happens when the WSGI server and RQ-worker restarts after a +configuration change. This occours in the following case for example: + +.. code-block:: sh + + systemctl restart gunicorn-netbox # WSGI-server already online with new + # configuration after this command. + systemctl restart netbox-rq # RQ-Worker still worked with the old + # configuration till here. + +This type handles the restart of both services correctly to avoid such database +corruptions. To safely manual restart the whole netbox instance manual, simply +restart all services in one ``systemctl restart netbox`` command, as it ensures +that first all services are shut down before starting one of them. The service +``netbox`` wraps all required services that are available. + SEE ALSO -------- diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 55f3549..dcf1a66 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -2,9 +2,11 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service +Requires=netbox-rq.service +Wants=network.target After=netbox.service After=network.target -Wants=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 0da92f0..4fae788 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -34,7 +34,7 @@ case "$state" in if [ "$do_restart" ] && [ "$state" != "disabled" ]; then cat << EOF # Restart service -service gunicorn-netbox restart +systemctl restart gunicorn-netbox EOF fi ;; diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index 8d41cd0..cd06e82 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -2,9 +2,11 @@ Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service +Requires=netbox-rq.service +Wants=network.target After=netbox.service After=network.target -Wants=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 9b6a603..c219643 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -30,7 +30,7 @@ case "$state" in if [ "$do_restart" ] && [ "$state" != "disabled" ]; then cat << EOF # Restart service -service uwsgi-netbox restart +systemctl restart uwsgi-netbox EOF fi ;; From e800f42a6d95b9378516252d53944383f673653d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 9 Sep 2020 20:33:20 +0200 Subject: [PATCH 110/156] __netbox: consistency with __systemd_unit --restart This commit brings consistency into the --restart parameter for systemd units. All units except the netbox wrapper service will be restarted on unit change. --- type/__netbox/manifest | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index d754ae6..f8ed5fc 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -213,8 +213,10 @@ SECRET # Upload systemd unit for worker and wsgi service -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done +# does not restart netbox on change cause it only restart all other services +__systemd_unit netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled --restart From 445bc75deba18b8fa978e7d5d8ed8189908457d6 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 10:59:13 +0200 Subject: [PATCH 111/156] [type/__dma_auth] Drop --server parameter Currently, dma does not differentiate between login users on the SMTP server. It will pick whatever entry it finds first (https://github.com/corecode/dma/blob/v0.13/net.c#L531). As a result, the --server parameter only adds confusion. --- type/__dma_auth/explorer/state | 29 +++++++++++++---------------- type/__dma_auth/gencode-remote | 24 +++++++++--------------- type/__dma_auth/man.rst | 6 ++++-- type/__dma_auth/parameter/optional | 1 - 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/type/__dma_auth/explorer/state b/type/__dma_auth/explorer/state index 668b50f..621e5a2 100755 --- a/type/__dma_auth/explorer/state +++ b/type/__dma_auth/explorer/state @@ -17,25 +17,18 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer looks for a line matching the login and server parameters +# This explorer looks for a line matching the server parameter # in dma's auth.conf and reports: # present: a line matching login + host + password exists # absent: no line matching login + host exists -# different_password: a line exists but with a different pasword -# multiple: multiple lines matching login + host exist -# (should never happen) +# different_login: a line exists but with a different login user +# different_password: a line exists but with a different password +# multiple: multiple lines matching host exist (should not happen) auth_conf=$("${__type_explorer}/auth_conf") test -r "${auth_conf}" || exit 0 -if test -f "${__object}/parameter/server" -then - server=$(cat "${__object}/parameter/server") -else - server=$__object_id -fi - -awk -F'\n' -v server="${server}" ' +awk -F'\n' ' function getvalue(path) { # Reads the first line of the file located at path and returns it. getline < path @@ -49,8 +42,7 @@ BEGIN { parameter_dir = ENVIRON["__object"] "/parameter/" # Read the parameters of this object - host_param = getvalue(parameter_dir "server") - if (!host_param) host_param = ENVIRON["__object_id"] + host_param = ENVIRON["__object_id"] login_param = getvalue(parameter_dir "login") passwd_param = getvalue(parameter_dir "password") @@ -78,10 +70,15 @@ BEGIN { } } -host == host_param && login == login_param { +host == host_param { # a match… if (state == "absent") { - state = ((passwd == passwd_param) ? "present" : "different_password") + if (login != login_param) + state = "different_login" + else if (passwd != passwd_param) + state = "different_password" + else + state = "present" } else { # report "multiple" to that the type can remove the duplicates. state = "multiple" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index c57e5cc..e73c424 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -21,14 +21,16 @@ state_is=$(cat "${__object}/explorer/state") state_should=$(cat "${__object}/parameter/state") -if test -f "${__object}/parameter/server" -then - server=$(cat "${__object}/parameter/server") -else - server=$__object_id -fi +server=$__object_id login=$(cat "${__object}/parameter/login") + +auth_conf=$(cat "${__object}/explorer/auth_conf") +test -n "${auth_conf}" || { + echo 'Cannot determine path of dma auth.conf' >&2 + exit 1 +} + if test "${state_is}" = "${state_should}" then # state is as it should @@ -61,13 +63,6 @@ in ;; esac -auth_conf=$(cat "${__object}/explorer/auth_conf") - -test -n "${auth_conf}" || { - echo 'Cannot determine path of dma auth.conf' >&2 - exit 1 -} - cat < Date: Mon, 28 Sep 2020 16:34:12 +0200 Subject: [PATCH 112/156] Move auth_conf explorer from __dma_auth to __dma --- type/__dma/explorer/auth_conf | 50 +++++++++++++++++++++++++++++- type/__dma_auth/explorer/auth_conf | 50 +----------------------------- 2 files changed, 50 insertions(+), 50 deletions(-) mode change 120000 => 100755 type/__dma/explorer/auth_conf mode change 100755 => 120000 type/__dma_auth/explorer/auth_conf diff --git a/type/__dma/explorer/auth_conf b/type/__dma/explorer/auth_conf deleted file mode 120000 index db038ae..0000000 --- a/type/__dma/explorer/auth_conf +++ /dev/null @@ -1 +0,0 @@ -../../__dma_auth/explorer/auth_conf \ No newline at end of file diff --git a/type/__dma/explorer/auth_conf b/type/__dma/explorer/auth_conf new file mode 100755 index 0000000..cef0aca --- /dev/null +++ b/type/__dma/explorer/auth_conf @@ -0,0 +1,49 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# This explorer determines the path of dma's auth.conf file + +# No dma.conf -> use default +test -f /etc/dma/dma.conf || { + echo /etc/dma/auth.conf + exit 0 +} +test -r /etc/dma/dma.conf || { + echo 'Cannot read /etc/dma/dma.conf' >&2 + exit 1 +} + +# Get AUTHPATH from dma.conf +awk -F'[ \t]' ' +{ + sub(/#.*$/, "", $0) # remove comments + if (!$0) next # ignore empty lines +} +$1 == "AUTHPATH" { + # Store authpath. In dma conf parsing last wins. + if ($2) authpath = substr($0, index($0, " ") + 1) +} +END { + if (authpath) { + print authpath + exit 0 + } else exit 1 +} +' /etc/dma/dma.conf \ +|| echo /etc/dma/auth.conf # default diff --git a/type/__dma_auth/explorer/auth_conf b/type/__dma_auth/explorer/auth_conf deleted file mode 100755 index cef0aca..0000000 --- a/type/__dma_auth/explorer/auth_conf +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -e -# -# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# This explorer determines the path of dma's auth.conf file - -# No dma.conf -> use default -test -f /etc/dma/dma.conf || { - echo /etc/dma/auth.conf - exit 0 -} -test -r /etc/dma/dma.conf || { - echo 'Cannot read /etc/dma/dma.conf' >&2 - exit 1 -} - -# Get AUTHPATH from dma.conf -awk -F'[ \t]' ' -{ - sub(/#.*$/, "", $0) # remove comments - if (!$0) next # ignore empty lines -} -$1 == "AUTHPATH" { - # Store authpath. In dma conf parsing last wins. - if ($2) authpath = substr($0, index($0, " ") + 1) -} -END { - if (authpath) { - print authpath - exit 0 - } else exit 1 -} -' /etc/dma/dma.conf \ -|| echo /etc/dma/auth.conf # default diff --git a/type/__dma_auth/explorer/auth_conf b/type/__dma_auth/explorer/auth_conf new file mode 120000 index 0000000..e89de93 --- /dev/null +++ b/type/__dma_auth/explorer/auth_conf @@ -0,0 +1 @@ +../../__dma/explorer/auth_conf \ No newline at end of file From 3feaea1d96a361bf16893f1a2442ce4338ac6c89 Mon Sep 17 00:00:00 2001 From: Marko Seric Date: Mon, 28 Sep 2020 16:43:31 +0200 Subject: [PATCH 113/156] [type/__dma_auth] Externalise AWK update script to separate file --- type/__dma_auth/files/update_dma_auth.awk | 93 +++++++++++++++++++++++ type/__dma_auth/gencode-remote | 90 ++-------------------- 2 files changed, 98 insertions(+), 85 deletions(-) create mode 100644 type/__dma_auth/files/update_dma_auth.awk diff --git a/type/__dma_auth/files/update_dma_auth.awk b/type/__dma_auth/files/update_dma_auth.awk new file mode 100644 index 0000000..c50198b --- /dev/null +++ b/type/__dma_auth/files/update_dma_auth.awk @@ -0,0 +1,93 @@ +#!/usr/bin/awk -f +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +function getvalue(path) { + # Reads the first line of the file located at path and returns it. + getline < path + close(path) + return $0 +} + +function print_should() { + printf "%s|%s:%s\n", login_param, host_param, passwd_param +} + +BEGIN { + FS = "\n" + DP = "[: \t]" # copied from dma/conf.c + + parameter_dir = ENVIRON["__object"] "/parameter/" + + mode = (getvalue(parameter_dir "state") != "absent") + + host_param = ENVIRON["__object_id"] + login_param = getvalue(parameter_dir "login") + passwd_param = getvalue(parameter_dir "password") +} + +# skip comments and empty lines +/^#/ || /^$/ { + print + next +} + +{ + # parse line (like dma/conf.c would) + + login = substr($0, 1, index($0, "|") - 1) + if (!login) { login = $0 } # if no "|" found + + host = substr($0, length(login) + 2) + + if (match(host, DP)) { + passwd = substr(host, RSTART + 1) + host = substr(host, 1, RSTART - 1) + } else { + passwd = "" + } +} + +host == host_param { + if (mode) { + # state_should == present + if (!written) { + # replace first line if host matches (but only if no line has + # been written already -> no duplicates) + print_should() + written = 1 + } + next + } else { + # state_should == absent + next + } +} + +# leave other lines alone +{ + print +} + +END { + if (mode && !written) { + # append line if no match to replace was found + print_should() + } +} diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index e73c424..d8be7e8 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,6 +18,8 @@ # along with cdist. If not, see . # +drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } + state_is=$(cat "${__object}/explorer/state") state_should=$(cat "${__object}/parameter/state") @@ -42,8 +44,6 @@ in (present) test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1; } - mode=1 - if test "${state_is}" = 'absent' then printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" @@ -52,8 +52,6 @@ in fi ;; (absent) - mode=0 - printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" ;; (*) @@ -65,87 +63,9 @@ esac cat < no duplicates) - print_should() - written = 1 - } - next - } else { - # state_should == absent - if (!login_param || login == login_param) { - # empty --login -> drop all lines for this host - next - } - } -} - -# leave other lines alone -{ - print -} - -END { - if (mode && !written) { - # append line if no match to replace was found - print_should() - } -} -' <"${auth_conf}" >"${auth_conf}.tmp" \ - && mv "${auth_conf}.tmp" "${auth_conf}" +awk '$(drop_awk_comments "${__type}/files/update_dma_auth.awk")' <'${auth_conf}' >'${auth_conf}.tmp' \ +&& cat '${auth_conf}.tmp' >'${auth_conf}' +rm -f '${auth_conf}.tmp' EOF From 6ae08085607f1a73832b80a20ad4671950e5c82e Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 16:54:21 +0200 Subject: [PATCH 114/156] [type/__dma] Externalise AWK update script to separate file --- type/__dma/files/update_dma_conf.awk | 170 +++++++++++++++++++++++++++ type/__dma/gencode-remote | 167 ++------------------------ 2 files changed, 179 insertions(+), 158 deletions(-) create mode 100644 type/__dma/files/update_dma_conf.awk diff --git a/type/__dma/files/update_dma_conf.awk b/type/__dma/files/update_dma_conf.awk new file mode 100644 index 0000000..67661fd --- /dev/null +++ b/type/__dma/files/update_dma_conf.awk @@ -0,0 +1,170 @@ +#!/usr/bin/awk -f +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +function comment_line(line) { + # returns the position in line at which the comment'\''s text starts + # (0 if the line is not a comment) + match(line, /^[ \t]*\#+[ \t]*/) + return RSTART ? (RLENGTH + 1) : 0 +} +function empty_line(line) { return line ~ /^[ \t]*$/ } +function is_word(s) { return s ~ /^[A-Z_]+$/ } # "looks like a plausible word" + +function first(line, sep_re) { + # returns the part of the line until sep is found + # (or the whole line if sep is not found) + if (!sep_re) sep_re = "[" SUBSEP "]" + match(line, sep_re) + return RSTART ? substr(line, 1, RSTART - 1) : line +} + +function rest(line, sep_re) { + # returns the part of the line after the first occurrence of sep is found. + # (or nothing if sep is not found) + if (!sep_re) sep_re = "[" SUBSEP "]" + if (match(line, sep_re)) + return substr(line, RSTART + RLENGTH) +} + +function conf_pop(word, value) { + # returns the next value for the config `word` and delete it from the list. + # if value is set, this function will only return value if it is the first + # option in the list, otherwise it returns 0. + + if (!(word in conf)) return 0 + if (!value) { + if (index(conf[word], SUBSEP)) # more than one element? + value = substr(conf[word], 1, index(conf[word], SUBSEP) - 1) + else + value = conf[word] + } + + if (index(conf[word], SUBSEP)) { + if (index(conf[word], value SUBSEP) != 1) return 0 + conf[word] = substr(conf[word], length(value) + 2) + } else { + if (conf[word] != value) return 0 + delete conf[word] + } + return value +} + +function print_conf(word, value) { + # print a config line with the given parameters + printf "%s", word + if (value) printf " %s", value + printf "\n" +} + +function print_confs(word, value) { + # print config lines for all values stored in conf[word]. + if (!(word in conf)) return + if (conf[word]) { + while (value = conf_pop(word)) + print_conf(word, value) + } else { + print_conf(word) + delete conf[word] + } +} + +BEGIN { + FS = "\n" + EQS = "[ \t]" # copied from dma/conf.c + + # read the "should" state into the `conf` array. + while (getline < "/dev/stdin") { + word = first($0, EQS) + if ((word in conf)) + conf[word] = conf[word] SUBSEP rest($0, EQS) + else + conf[word] = rest($0, EQS) + } +} + +# first pass, gather information about where which information is stored in the +# current config file. This information will be used in the second pass. +NR == FNR { + if (comment_line($0)) { + # comment line + word = first(substr($0, comment_line($0)), " ") + if (is_word(word)) last_occ["#" word] = FNR + } else { + word = first($0, EQS) + if (is_word(word)) last_occ[word] = FNR + } +} + +# before second pass prepare hashes containing location information to be used +# in the second pass. +NR > FNR && FNR == 1 { + # First we drop the locations of commented-out options if a non-commented + # option is available. If a non-commented option is available, we will + # append new config options there to have them all at one place. + for (k in last_occ) + if (k ~ /^\#/ && (substr(k, 2) in last_occ)) + delete last_occ[k] + + # Reverse the option => line mapping. The line_map allows for easier lookups + # in the second pass. + for (k in last_occ) line_map[last_occ[k]] = k +} + +# second pass, generate and output new config +NR > FNR { + if (comment_line($0) || empty_line($0)) { + # comment or empty line + print + + if ((FNR in line_map)) { + if (line_map[FNR] ~ /^\#/) { + # This line contains a commented config option. If the conf hash + # contains options to be set, we output them here because this + # option is not used in the current config. + k = substr(line_map[FNR], 2) + if ((k in conf)) print_confs(k) + } + + if (("INSECURE" in conf) && line_map[FNR] ~ /^\#?SECURE$/) { + # INSECURE goes where SECURE comment is. + print_confs("INSECURE") + } + } + } else { + word = first($0, EQS) + value = rest($0, EQS) + sub(/[ \t]*\#.*$/, "", value) # ignore comments in value + + if ((word in conf) && value == first(conf[word])) { + # keep config options we want + conf_pop(word) + print + } + + if ((FNR in line_map) && line_map[FNR] == word) { + # rest of config options should be here + print_confs(word) + } + } +} + +END { + # print rest of config options ( + for (word in conf) print_confs(word) +} diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index a6aca0d..8177de9 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -1,5 +1,7 @@ #!/bin/sh -e +drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } + CONF_PATH=/etc/dma # set in Makefile # Determine mailname @@ -100,7 +102,7 @@ config_updated=false if ! echo "$conf_should" | cmp -s "${__object}/explorer/conf" - then # config needs to be updated - echo "dma_conf='${CONF_PATH:?}/dma.conf'" + dma_conf="${CONF_PATH:?}/dma.conf" # The following AWK script will output the new config file to be stored on # disk. To do so it reads the current dma.conf file and the config options @@ -119,164 +121,13 @@ then # options are grouped by word (the first word in the line) and appended # at the end of the file. - cat <<'EOF' -awk -F '\n' ' -function comment_line(line) { - # returns the position in line at which the comment'\''s text starts - # (0 if the line is not a comment) - match(line, /^[ \t]*\#+[ \t]*/) - return RSTART ? (RLENGTH + 1) : 0 -} -function empty_line(line) { return line ~ /^[ \t]*$/ } -function is_word(s) { return s ~ /^[A-Z_]+$/ } # "looks like a plausible word" - -function first(line, sep_re) { - # returns the part of the line until sep is found - # (or the whole line if sep is not found) - if (!sep_re) sep_re = "[" SUBSEP "]" - match(line, sep_re) - return RSTART ? substr(line, 1, RSTART - 1) : line -} - -function rest(line, sep_re) { - # returns the part of the line after the first occurrence of sep is found. - # (or nothing if sep is not found) - if (!sep_re) sep_re = "[" SUBSEP "]" - if (match(line, sep_re)) - return substr(line, RSTART + RLENGTH) -} - -function conf_pop(word, value) { - # returns the next value for the config `word` and delete it from the list. - # if value is set, this function will only return value if it is the first - # option in the list, otherwise it returns 0. - - if (!(word in conf)) return 0 - if (!value) { - if (index(conf[word], SUBSEP)) # more than one element? - value = substr(conf[word], 1, index(conf[word], SUBSEP) - 1) - else - value = conf[word] - } - - if (index(conf[word], SUBSEP)) { - if (index(conf[word], value SUBSEP) != 1) return 0 - conf[word] = substr(conf[word], length(value) + 2) - } else { - if (conf[word] != value) return 0 - delete conf[word] - } - return value -} - -function print_conf(word, value) { - # print a config line with the given parameters - printf "%s", word - if (value) printf " %s", value - printf "\n" -} - -function print_confs(word, value) { - # print config lines for all values stored in conf[word]. - if (!(word in conf)) return - if (conf[word]) { - while (value = conf_pop(word)) - print_conf(word, value) - } else { - print_conf(word) - delete conf[word] - } -} - -BEGIN { - EQS = "[ \t]" # copied from dma/conf.c - - # read the "should" state into the `conf` array. - while (getline < "/dev/stdin") { - word = first($0, EQS) - if ((word in conf)) - conf[word] = conf[word] SUBSEP rest($0, EQS) - else - conf[word] = rest($0, EQS) - } -} - -# first pass, gather information about where which information is stored in the -# current config file. This information will be used in the second pass. -NR == FNR { - if (comment_line($0)) { - # comment line - word = first(substr($0, comment_line($0)), " ") - if (is_word(word)) last_occ["#" word] = FNR - } else { - word = first($0, EQS) - if (is_word(word)) last_occ[word] = FNR - } -} - -# before second pass prepare hashes containing location information to be used -# in the second pass. -NR > FNR && FNR == 1 { - # First we drop the locations of commented-out options if a non-commented - # option is available. If a non-commented option is available, we will - # append new config options there to have them all at one place. - for (k in last_occ) - if (k ~ /^\#/ && (substr(k, 2) in last_occ)) - delete last_occ[k] - - # Reverse the option => line mapping. The line_map allows for easier lookups - # in the second pass. - for (k in last_occ) line_map[last_occ[k]] = k -} - -# second pass, generate and output new config -NR > FNR { - if (comment_line($0) || empty_line($0)) { - # comment or empty line - print - - if ((FNR in line_map)) { - if (line_map[FNR] ~ /^\#/) { - # This line contains a commented config option. If the conf hash - # contains options to be set, we output them here because this - # option is not used in the current config. - k = substr(line_map[FNR], 2) - if ((k in conf)) print_confs(k) - } - - if (("INSECURE" in conf) && line_map[FNR] ~ /^\#?SECURE$/) { - # INSECURE goes where SECURE comment is. - print_confs("INSECURE") - } - } - } else { - word = first($0, EQS) - value = rest($0, EQS) - sub(/[ \t]*\#.*$/, "", value) # ignore comments in value - - if ((word in conf) && value == first(conf[word])) { - # keep config options we want - conf_pop(word) - print - } - - if ((FNR in line_map) && line_map[FNR] == word) { - # rest of config options should be here - print_confs(word) - } - } -} - -END { - # print rest of config options ( - for (word in conf) print_confs(word) -} -' "${dma_conf}" "${dma_conf}" <<'EOF' >"${dma_conf}.tmp" \ - && mv "${dma_conf}.tmp" "${dma_conf}" + cat <'${dma_conf}.tmp' \ +&& cat '${dma_conf}.tmp' >'${dma_conf}' +${conf_should} EOF - # Pass in "conf_should" via stdin - echo "${conf_should}" - echo 'EOF' +rm '${dma_conf}.tmp' +CODE config_updated=true echo 'config updated' >>"${__messages_out}" From b48b48e4047d8855e9ee2d635a0c345cad007997 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 17:29:41 +0200 Subject: [PATCH 115/156] [type/__mail_alias] Externalise AWK update script to separate file --- type/__mail_alias/files/update_aliases.awk | 98 ++++++++++++++++++ type/__mail_alias/gencode-remote | 115 ++++----------------- 2 files changed, 119 insertions(+), 94 deletions(-) create mode 100644 type/__mail_alias/files/update_aliases.awk diff --git a/type/__mail_alias/files/update_aliases.awk b/type/__mail_alias/files/update_aliases.awk new file mode 100644 index 0000000..87ea202 --- /dev/null +++ b/type/__mail_alias/files/update_aliases.awk @@ -0,0 +1,98 @@ +#!/usr/bin/awk -f +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +function getvalue(path) { + # Reads the first line of the file located at path and returns it. + getline < path + close(path) + return $0 +} + +function sepafter(f, def, _) { + # finds the separator between field $f and $(f+1) + _ = substr($0, length($f)+1, index(substr($0, length($f)+1), $(f+1))-1) + return _ ? _ : def +} + +function write_aliases() { + if (aliases_written) return + + # print aliases line + printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") + while ((getline < aliases_should_file) > 0) { + if (aliases_written) printf ", " + printf "%s", $0 + aliases_written = 1 + } + printf "\n" + close(aliases_should_file) +} + +BEGIN { + FS = ":[ \t]*" + + parameter_dir = ENVIRON["__object"] "/parameter/" + + mode = (getvalue(parameter_dir "state") != "absent") + aliases_should_file = (parameter_dir "/alias") +} + +/^[ \t]*\#/ { + # comment line (leave alone) + select = 0; cont = 0 # comments terminate alias lists and continuations + print + next +} + +{ + # is this line a continuation line? + # (the prev. line ended in a backslash or the line starts with whitespace) + is_cont = /^[ \t]/ || cont + + # detect if the line is a line to be continued (ends with a backslash) + cont = ($0 ~ /\\$/) + # if it is, we drop the backslash from the line. + if (cont) sub(/[ \t]*\\$/, "", $0) +} + +is_cont { + # we ignore the line as it has been rewritten previously or is not + # interesting + next +} + +$1 == ENVIRON["__object_id"] { + # "target" user -> rewrite aliases list + select = 1 + if (mode) write_aliases() + next +} + +{ + # other user + select = 0 + print +} + +END { + # if the last line was an alias, the separator will be reused (looks better) + if (mode && !aliases_written) + write_aliases() +} diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 9f4af1b..e5bc2b7 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -18,6 +18,16 @@ # along with cdist. If not, see . # +drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } + +aliases_file=$(cat "${__object}/explorer/aliases_file") + +test -n "${aliases_file}" || { + echo 'Could not determine aliases file path.' >&2 + exit 1 +} + + state_should=$(cat "${__object}/parameter/state") case $state_should @@ -35,119 +45,36 @@ in else echo "add aliases" >>"$__messages_out" fi - - mode=1 ;; (absent) # nothing to do if no aliases found. test -s "${__object}/explorer/aliases" || exit 0 echo "delete aliases" >>"$__messages_out" - - mode=0 ;; (*) - printf 'Invalid --state: %s.\n' "$state_should" >&2 + printf 'Invalid --state: %s.\n' "${state_should}" >&2 printf 'Acceptable values are: present, absent.\n' >&2 exit 1 esac -aliases_file=$(cat "${__object}/explorer/aliases_file") +cat <&2 - exit 1 -} - -# "export" variables to remote -printf 'mode=%u\n' "${mode}" -printf "aliases_file='%s'\n" "${aliases_file}" - -cat <<'EOF' -test -f "${aliases_file}" || touch "${aliases_file}" - -awk -F ':[ \t]*' -v mode=$mode ' -function sepafter(f, def, _) { - # finds the separator between field $f and $(f+1) - _ = substr($0, length($f)+1, index(substr($0, length($f)+1), $(f+1))-1) - return _ ? _ : def -} - -function write_aliases() { - if (aliases_written) return - - # print aliases line - printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") - while ((getline < aliases_should_file) > 0) { - if (aliases_written) printf ", " - printf "%s", $0 - aliases_written = 1 - } - printf "\n" - close(aliases_should_file) -} - -BEGIN { - aliases_should_file = (ENVIRON["__object"] "/parameter/alias") -} - -/^[ \t]*\#/ { - # comment line (leave alone) - select = 0; cont = 0 # comments terminate alias lists and continuations - print - next -} - -{ - # is this line a continuation line? - # (the prev. line ended in a backslash or the line starts with whitespace) - is_cont = /^[ \t]/ || cont - - # detect if the line is a line to be continued (ends with a backslash) - cont = ($0 ~ /\\$/) - # if it is, we drop the backslash from the line. - if (cont) sub(/[ \t]*\\$/, "", $0) -} - -is_cont { - # we ignore the line as it has been rewritten previously or is not - # interesting - next -} - -$1 == ENVIRON["__object_id"] { - # "target" user -> rewrite aliases list - select = 1 - if (mode) write_aliases() - next -} - -{ - # other user - select = 0 - print -} - -END { - # if the last line was an alias, the separator will be reused (looks better) - if (mode && !aliases_written) - write_aliases() -} -' <"${aliases_file}" >"${aliases_file}.tmp" || { - rm -f "${aliases_file}.tmp" +awk '$(drop_awk_comments "${__type}/files/update_aliases.awk")' <'${aliases_file}' >'${aliases_file}.tmp' \ +|| { + rm -f '${aliases_file}.tmp' echo 'Generating new aliases file failed!' >&2 exit 1 } -if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" +if ! cmp -s '${aliases_file}' '${aliases_file}.tmp' then - # aliases file was modified, replace and run `newaliases`. - mv "${aliases_file}.tmp" "${aliases_file}" + # aliases file was modified, replace: + cat '${aliases_file}.tmp' >'${aliases_file}' - # run newaliases if present + # then, run newaliases if present ("missing" on Alpine Linux because of typo) command -v newaliases >/dev/null 2>&1 && newaliases || true -else - # no modifications were made, delete the temp file. - rm "${aliases_file}.tmp" fi +rm -f '${aliases_file}.tmp' EOF From 2270c32ddb7d2fa2c9bd904755e20fd0fbae13dc Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 17:30:53 +0200 Subject: [PATCH 116/156] [type/__dma] Add missing license headers --- type/__dma/gencode-remote | 18 ++++++++++++++++++ type/__dma/manifest | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 8177de9..bcd4530 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } diff --git a/type/__dma/manifest b/type/__dma/manifest index 75e42d7..df62308 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# os=$(cat "${__global}/explorer/os") From b55186544f4cfeb9f371c4c0ff429bdc2a63c6cc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 28 Sep 2020 19:49:29 +0200 Subject: [PATCH 117/156] __netbox: adding seperate /opt/netbox/cdist/ directory Adding a place to keep all configuration stuff for cdist seperated from the installation directory. For safety reasons. --- type/__netbox/explorer/secretkey | 2 +- type/__netbox/explorer/version | 2 +- type/__netbox/gencode-remote | 26 +++++++------------------- type/__netbox/manifest | 17 ++++++++++------- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/type/__netbox/explorer/secretkey b/type/__netbox/explorer/secretkey index 9f0331c..7cce279 100755 --- a/type/__netbox/explorer/secretkey +++ b/type/__netbox/explorer/secretkey @@ -2,7 +2,7 @@ # Explorer will output the key if he exists. -secretkey="/opt/netbox/.secretkey" +secretkey="/opt/netbox/cdist/secretkey" if [ -f "$secretkey" ]; then cat "$secretkey" fi diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version index 5231e16..ee3dde8 100755 --- a/type/__netbox/explorer/version +++ b/type/__netbox/explorer/version @@ -1,5 +1,5 @@ #!/bin/sh -e # output version if exist -version_path="/opt/netbox/netbox/cdist/version" +version_path="/opt/netbox/cdist/version" if [ -f "$version_path" ]; then cat "$version_path"; fi diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 0370949..aa3864e 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -24,16 +24,13 @@ cd "\$tmpdir" curl -sS -L '$url' > '$archive' tar xf '$archive' -# Save cdist-upload configuration file. -cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" -cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" - EOF # Stop everything in the pyenv to update cat << EOF # Try to kill everything in the venv -systemctl stop netbox gunicorn-netbox uwsgi-netbox || true +systemctl -q --wait stop netbox gunicorn-netbox uwsgi-netbox || true +# don't know if this is required since using --wait ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true EOF @@ -42,11 +39,10 @@ EOF # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' -mkdir '$install_dir/cdist' cp '$src/requirements.txt' /opt/netbox/ # backup dependency info -cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' -cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' +ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. # forcing python3 to be sure (till python4 gets released ..) @@ -79,24 +75,16 @@ cd / rm -r "\$tmpdir" # Save version after successful installation -printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' +printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version EOF # meta printf "installed %s\n" "$VERSION" >> "$__messages_out" changes=yes +fi # check if configuration changed -elif grep -q "^__file/opt/netbox/" "$__messages_in"; then - # check if coping is required - if grep -q "^__file/opt/netbox/netbox/cdist/" "$__messages_in"; then - cat << EOF -# Copy configuration -cp '$install_dir/cdist/configuration.py' '$install_dir/netbox/configuration.py' -cp '$install_dir/cdist/ldap_config.py' '$install_dir/netbox/ldap_config.py' -EOF - fi - +if grep -q "^__file/opt/netbox/" "$__messages_in"; then # meta printf "configured\n" >> "$__messages_out" changes=yes diff --git a/type/__netbox/manifest b/type/__netbox/manifest index f8ed5fc..89b89ac 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -194,20 +194,23 @@ mkdir -p "$__object/files" "$__type/files/configuration.py.sh" > "$__object/files/configuration.py" "$__type/files/ldap_config.py.sh" > "$__object/files/ldap_config.py" -require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents -require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ +require="__user/netbox" __directory /opt/netbox/cdist +require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/configuration.py --mode 640 --owner netbox \ --source "$__object/files/configuration.py" if [ -f "$__object/parameter/ldap-server" ]; then - require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ --source "$__object/files/ldap_config.py" +else + require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/ldap_config.py --state absent fi # save secret -require="__user/netbox" __file /opt/netbox/.secretkey --mode 400 \ - --owner netbox --source - << SECRET +require="__directory/opt/netbox/cdist" __file /opt/netbox/cdist/secretkey \ + --mode 400 --owner netbox --source - << SECRET $SECRET_KEY SECRET From 050812305b282e65be51c3902dba691c4718ca4d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 28 Sep 2020 20:04:22 +0200 Subject: [PATCH 118/156] __netbox: add mange.py calls recommended from upstream Calls where added because there are used upstream in the `upgrade.sh`-Script, too. Upgrade-Script: https://github.com/netbox-community/netbox/blob/develop/upgrade.sh --- type/__netbox/gencode-remote | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index aa3864e..d7bc219 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -64,11 +64,17 @@ EOF # Set final permissions. chown -R netbox /opt/netbox +# NetBox manage scripts # Run database migrations. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate - # Generate static assets. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collectstatic --no-input +# Delete any stale content types +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py remove_stale_contenttypes --no-input +# Delete any expired user sessions +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py clearsessions +# Clear all cached data +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py invalidate all # Remove temporary working directory. cd / From 231f96de18cad4b53232baa504b8bf266bd015de Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 17:37:41 +0200 Subject: [PATCH 119/156] Error if expected environment variables are unset --- type/__dma/gencode-remote | 58 ++++++++++++------------- type/__dma/manifest | 2 +- type/__dma_auth/gencode-remote | 18 ++++---- type/__mail_alias/explorer/aliases | 4 +- type/__mail_alias/explorer/aliases_file | 2 +- type/__mail_alias/gencode-remote | 18 ++++---- 6 files changed, 52 insertions(+), 50 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index bcd4530..e254323 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -23,18 +23,18 @@ drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } CONF_PATH=/etc/dma # set in Makefile # Determine mailname -if test -f "${__object}/parameter/mailname" +if test -f "${__object:?}/parameter/mailname" then - mailname=$(cat "${__object}/parameter/mailname") + mailname=$(cat "${__object:?}/parameter/mailname") else - case $(cat "${__global}/explorer/os") + case $(cat "${__global:?}/explorer/os") in (debian|devuan|ubuntu) # On Debian-like systems use /etc/mailname unless --mailname is used mailname='/etc/mailname' ;; (*) - mailname=$__target_fqdn + mailname=${__target_fqdn:?} ;; esac fi @@ -42,19 +42,19 @@ fi # Generate "should" values for config conf_should=$( - if test -s "${__object}/parameter/smarthost" + if test -s "${__object:?}/parameter/smarthost" then - printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smarthost")" + printf 'SMARTHOST %s\n' "$(cat "${__object:?}/parameter/smarthost")" fi printf 'MAILNAME %s\n' "${mailname}" - if test -s "${__object}/explorer/auth_conf" + if test -s "${__object:?}/explorer/auth_conf" then - printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" + printf "AUTHPATH %s\n" "$(cat "${__object:?}/explorer/auth_conf")" fi - case $(cat "${__object}/parameter/security") + case $(cat "${__object:?}/parameter/security") in (ssl|tls) default_smtp_port=465 @@ -77,35 +77,35 @@ conf_should=$( ;; esac - if test -s "${__object}/parameter/port" + if test -s "${__object:?}/parameter/port" then - printf 'PORT %u\n' "$(cat "${__object}/parameter/port")" + printf 'PORT %u\n' "$(cat "${__object:?}/parameter/port")" elif test "${default_smtp_port}" -ne 25 # DMA uses port 25 by default then printf 'PORT %u\n' "${default_smtp_port}" fi - if test -f "${__object}/parameter/masquerade" + if test -f "${__object:?}/parameter/masquerade" then while read -r line do printf 'MASQUERADE %s\n' "${line}" - done <"${__object}/parameter/masquerade" + done <"${__object:?}/parameter/masquerade" fi - if test -f "${__object}/parameter/defer" + if test -f "${__object:?}/parameter/defer" then echo 'DEFER' fi - if test -f "${__object}/parameter/fullbounce" + if test -f "${__object:?}/parameter/fullbounce" then echo 'FULLBOUNCE' fi - if test -f "${__object}/parameter/nullclient" + if test -f "${__object:?}/parameter/nullclient" then - test -s "${__object}/parameter/smarthost" || { + test -s "${__object:?}/parameter/smarthost" || { echo '--nullclient requires a --smarthost to be defined' >&2 exit 1 } @@ -114,10 +114,10 @@ conf_should=$( fi ) # Sort conf_should to compare against "conf_is" -conf_should=$(echo "$conf_should" | sort -s -k 1,1) +conf_should=$(echo "${conf_should}" | sort -s -k 1,1) config_updated=false -if ! echo "$conf_should" | cmp -s "${__object}/explorer/conf" - +if ! echo "${conf_should}" | cmp -s "${__object:?}/explorer/conf" - then # config needs to be updated dma_conf="${CONF_PATH:?}/dma.conf" @@ -140,7 +140,7 @@ then # at the end of the file. cat <'${dma_conf}.tmp' \ +awk '$(drop_awk_comments "${__type:?}/files/update_dma_conf.awk")' '${dma_conf}' '${dma_conf}' <<'EOF' >'${dma_conf}.tmp' \ && cat '${dma_conf}.tmp' >'${dma_conf}' ${conf_should} EOF @@ -148,28 +148,28 @@ rm '${dma_conf}.tmp' CODE config_updated=true - echo 'config updated' >>"${__messages_out}" + echo 'config updated' >>"${__messages_out:?}" fi -if test -f "${__object}/parameter/send-test-email" +if test -f "${__object:?}/parameter/send-test-email" then - if grep -q '^__mail_alias/root:' "${__messages_in}" \ - || grep -q '^__dma_auth/' "${__messages_in}" \ + if grep -q '^__mail_alias/root:' "${__messages_in:?}" \ + || grep -q '^__dma_auth/' "${__messages_in:?}" \ || $config_updated then - cat <<-EOF - sendmail root <. # -os=$(cat "${__global}/explorer/os") +os=$(cat "${__global:?}/explorer/os") # Install DMA case $os diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index d8be7e8..aee4d7f 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -20,14 +20,14 @@ drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } -state_is=$(cat "${__object}/explorer/state") -state_should=$(cat "${__object}/parameter/state") +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") -server=$__object_id -login=$(cat "${__object}/parameter/login") +server=${__object_id:?} +login=$(cat "${__object:?}/parameter/login") -auth_conf=$(cat "${__object}/explorer/auth_conf") +auth_conf=$(cat "${__object:?}/explorer/auth_conf") test -n "${auth_conf}" || { echo 'Cannot determine path of dma auth.conf' >&2 exit 1 @@ -46,13 +46,13 @@ in if test "${state_is}" = 'absent' then - printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out:?}" else - printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out:?}" fi ;; (absent) - printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out:?}" ;; (*) printf 'Invalid --state: %s.\n' "${state_should}" >&2 @@ -65,7 +65,7 @@ esac cat <'${auth_conf}.tmp' \ +awk '$(drop_awk_comments "${__type:?}/files/update_dma_auth.awk")' <'${auth_conf}' >'${auth_conf}.tmp' \ && cat '${auth_conf}.tmp' >'${auth_conf}' rm -f '${auth_conf}.tmp' EOF diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index 4fffd3b..5b5d68d 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -20,9 +20,11 @@ # Find aliases for a given user name and print the aliases (each one on a # separate line) -aliases_file=$("${__type_explorer}/aliases_file") +aliases_file=$("${__type_explorer:?}/aliases_file") test -r "${aliases_file}" || exit 0 +: "${__object_id:?}" # assert __object_id is set, because it is used in AWK + awk -F ':[ \t]*' ' function print_aliases(aliases, matches) { # prints comma-separated aliases (one per line) diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file index a59bb99..7f09f88 100755 --- a/type/__mail_alias/explorer/aliases_file +++ b/type/__mail_alias/explorer/aliases_file @@ -28,7 +28,7 @@ check_file() { fi } -case $("${__explorer}/os") +case $("${__explorer:?}/os") in (freebsd|openbsd|solaris) check_file /etc/mail/aliases diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index e5bc2b7..7ef2f7c 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -20,7 +20,7 @@ drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } -aliases_file=$(cat "${__object}/explorer/aliases_file") +aliases_file=$(cat "${__object:?}/explorer/aliases_file") test -n "${aliases_file}" || { echo 'Could not determine aliases file path.' >&2 @@ -28,29 +28,29 @@ test -n "${aliases_file}" || { } -state_should=$(cat "${__object}/parameter/state") +state_should=$(cat "${__object:?}/parameter/state") case $state_should in (present) - if cmp -s "${__object}/explorer/aliases" "${__object}/parameter/alias" + if cmp -s "${__object:?}/explorer/aliases" "${__object:?}/parameter/alias" then # all good! exit 0 fi - if test -s "${__object}/explorer/aliases" + if test -s "${__object:?}/explorer/aliases" then - echo "update aliases" >>"$__messages_out" + echo "update aliases" >>"${__messages_out:?}" else - echo "add aliases" >>"$__messages_out" + echo "add aliases" >>"${__messages_out:?}" fi ;; (absent) # nothing to do if no aliases found. - test -s "${__object}/explorer/aliases" || exit 0 + test -s "${__object:?}/explorer/aliases" || exit 0 - echo "delete aliases" >>"$__messages_out" + echo "delete aliases" >>"${__messages_out:?}" ;; (*) printf 'Invalid --state: %s.\n' "${state_should}" >&2 @@ -61,7 +61,7 @@ esac cat <'${aliases_file}.tmp' \ +awk '$(drop_awk_comments "${__type:?}/files/update_aliases.awk")' <'${aliases_file}' >'${aliases_file}.tmp' \ || { rm -f '${aliases_file}.tmp' echo 'Generating new aliases file failed!' >&2 From 161e1e85f4b7696036bc1684426e33962d713fa7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 17:54:35 +0200 Subject: [PATCH 120/156] [scripts/run-shellcheck.sh] Do not shellcheck AWK and Python scripts --- scripts/run-shellcheck.sh | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/scripts/run-shellcheck.sh b/scripts/run-shellcheck.sh index 769f853..fcca722 100755 --- a/scripts/run-shellcheck.sh +++ b/scripts/run-shellcheck.sh @@ -1,21 +1,29 @@ -#!/bin/sh +#!/bin/sh -eu -SHELLCHECKCMD="shellcheck -s sh -f gcc -x" +SHELLCHECKCMD='shellcheck -s sh -f gcc -x' # Skip SC2154 for variables starting with __ since such variables are cdist # environment variables. SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]' -SHELLCHECKTMP=".shellcheck.tmp" +SHELLCHECKTMP='.shellcheck.tmp' # Move to top-level cdist-contrib directory. -cd $(dirname $0)/.. +cd "$(dirname $0)"/.. -check () { - find type/ -type f $1 $2 -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}" - test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } +check() { + find type/ -type f "$@" -exec ${SHELLCHECKCMD} {} + \ + | grep -v "${SHELLCHECK_SKIP}" >>"${SHELLCHECKTMP}" || true } -check -path "*/explorer/*" -check -path "*/files/*" +rm -f "${SHELLCHECKTMP}" + +check -path '*/explorer/*' +check -path '*/files/*' ! -name '*.awk' ! -name '*.py' check -name manifest check -name gencode-local check -name gencode-remote + +if test -s "${SHELLCHECKTMP}" +then + cat "${SHELLCHECKTMP}" >&2 + exit 1 +fi From c6b795b3f9ac5472b00818fd53b6da02feabb52a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 1 Oct 2020 12:55:38 +0200 Subject: [PATCH 121/156] [type/__mail_alias] Update man.rst and make --alias required --- type/__mail_alias/gencode-remote | 6 ++++++ type/__mail_alias/man.rst | 15 ++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 7ef2f7c..3eea452 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -39,6 +39,12 @@ in exit 0 fi + test -s "${__object:?}/parameter/alias" || { + printf 'The --alias parameter is required if --state present.\n' >&2 + printf 'Use --state absent to remove all aliases.\n' >&2 + exit 1 + } + if test -s "${__object:?}/explorer/aliases" then echo "update aliases" >>"${__messages_out:?}" diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst index 3782ffb..d6d9742 100644 --- a/type/__mail_alias/man.rst +++ b/type/__mail_alias/man.rst @@ -8,7 +8,7 @@ cdist-type__mail_alias - Manage mail aliases. DESCRIPTION ----------- -This cdist type allows you to configure mail aliases (/etc/mail/aliases). +This cdist type allows you to configure mail aliases (/etc/aliases). REQUIRED PARAMETERS @@ -21,11 +21,12 @@ OPTIONAL PARAMETERS state 'present' or 'absent', defaults to 'present' alias - the aliases where mail for the given user should be redirected to. - This parameter can be specified multiple times to redirect to more than one - recipient. - See the `aliases(5)` man page for the different forms this parameter can - take.. + an alias, i.e. a mail address where mail for the user should be redirected + to. + This parameter can be specified multiple times to redirect to multiple + recipients. + If ``--state`` is ``present`` this parameter is required. + See `aliases(5)` for the different forms this parameter can take. BOOLEAN PARAMETERS @@ -51,7 +52,7 @@ BUGS containing ``,`` (commas) are treated incorrectly (they are treated as two addresses/aliases.) Make sure that email addresses do not contain commas. - +- ``:include:`` directives in the aliases file are ignored by this type. SEE ALSO -------- From f202d11124347783dbd679ae521f1bcb09e73cbb Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 1 Oct 2020 13:45:20 +0200 Subject: [PATCH 122/156] [type/__mail_alias] Fix continuation line processing --- type/__mail_alias/explorer/aliases | 12 ++++-------- type/__mail_alias/files/update_aliases.awk | 8 +++----- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index 5b5d68d..ac13d7c 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -31,7 +31,7 @@ function print_aliases(aliases, matches) { split(aliases, matches, /,[ \t]*/) for (i in matches) { gsub(/^[ \t]*|[ \t]*$/, "", matches[i]) - print matches[i] + if (matches[i]) print matches[i] } } @@ -47,14 +47,10 @@ function print_aliases(aliases, matches) { is_cont = /^[ \t]/ || cont # detect if the line is a line to be continued (ends with a backslash) - cont = ($0 ~ /\\$/) + cont = /\\$/ - # if it is, we drop the backslash from the line and skip to next line - # (the contents have been printed above if they should) - if (cont) { - sub(/[ \t]*\\$/, "", $0) - next - } + # if it is, we drop the backslash from the line + if (cont) sub(/[ \t]*\\$/, "", $0) } is_cont { diff --git a/type/__mail_alias/files/update_aliases.awk b/type/__mail_alias/files/update_aliases.awk index 87ea202..336009f 100644 --- a/type/__mail_alias/files/update_aliases.awk +++ b/type/__mail_alias/files/update_aliases.awk @@ -67,14 +67,12 @@ BEGIN { is_cont = /^[ \t]/ || cont # detect if the line is a line to be continued (ends with a backslash) - cont = ($0 ~ /\\$/) - # if it is, we drop the backslash from the line. - if (cont) sub(/[ \t]*\\$/, "", $0) + cont = /\\$/ } is_cont { - # we ignore the line as it has been rewritten previously or is not - # interesting + # we only print the line if it has not been rewritten (select) + if (!select) print next } From 1c9ab6e07b1808ce708cc9926bce4983a7faf0d7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 1 Oct 2020 17:18:01 +0200 Subject: [PATCH 123/156] [type/__dma] Update man.rst --- type/__dma/gencode-remote | 5 ++-- type/__dma/man.rst | 56 ++++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index e254323..1987106 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -66,7 +66,7 @@ conf_should=$( echo 'STARTTLS' ;; (opportunistic) - default_smtp_port=25 # XXX: correct? + default_smtp_port=25 echo 'SECURETRANSFER' echo 'STARTTLS' echo 'OPPORTUNISTIC_TLS' @@ -152,7 +152,8 @@ CODE fi -if test -f "${__object:?}/parameter/send-test-email" +# Send a test email if enabled and necessary (=configuration changed) +if test -f "${__object:?}/parameter/send-test-mail" then if grep -q '^__mail_alias/root:' "${__messages_in:?}" \ || grep -q '^__dma_auth/' "${__messages_in:?}" \ diff --git a/type/__dma/man.rst b/type/__dma/man.rst index ba4a5a6..29a71fa 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -8,47 +8,46 @@ cdist-type__dma - Setup the DragonFly Mail Agent as the MTA. DESCRIPTION ----------- -This (singleton) type uses dma, a small Mail Transport Agent (MTA), to accept -mails from locally installed Mail User Agents (MUA) and deliver the mails -to a remote destination. - -Remote delivery happens over TLS to one or more mailboxes that are local to the -mail server configured in the ``smarthost`` parameter. +This (singleton) type uses DMA, a small Mail Transport Agent (MTA), to accept +mails from locally installed Mail User Agents (MUA) and either deliver the mails +to a remote smart host for delivery or communicate with remote SMTP servers +directly. REQUIRED PARAMETERS ------------------- -smarthost - The mail server used to send email. - It must be configured to act as a relay for the host being configured by - this type so that mail can be sent to users non-local to the smarthost. +None. BOOLEAN PARAMETERS ------------------ defer - If enabled, the mail queue has to be manually flushed with the `-q` option. -full-bounce - Enable if the bounce message should include the complete original message, + If enabled, mail will not be sent immediately, but stored in a queue. + To flush the queue and send the mails, ```dma -q`` has to be run + periodically (e.g. using a cron job.) + This type does not manage such a cron job, but some operating systems ship + such a cron job with the package. +fullbounce + Enable if bounce messages should include the complete original message, not just the headers. -null-client +nullclient Enable to bypass aliases and local delivery, and instead forward all mails to the defined ``--smarthost``. -send-test-email - If present, after setup this type will send an email to root, to allow you - to easily test your setup. +send-test-mail + If set, this type will send a test email to root after setup, to check if + the configured settings work. OPTIONAL PARAMETERS ------------------- mailname If present, this will be the hostname used to identify this host and the - remote part of the from addresses. - If not defined, it defaults to `/etc/mailname` on Debian derivatives and to - `__target_fqdn` otherwise. + remote part of the sender addresses. + If not defined, it defaults to ``/etc/mailname`` on Debian derivatives and + to ``__target_fqdn`` otherwise. See `dma(8)` for more information. - Note: on Debian derivatives the `/etc/mailname` file should be updated + Note: on Debian derivatives the ``/etc/mailname`` file should be updated instead of using this parameter. masquerade Masquerade the envelope-from addresses with this address/hostname. @@ -59,7 +58,7 @@ masquerade port The port on which to deliver email. If not provided, a sensible default port will be used based on the - `--security` argument. + ``--security`` argument. security Configures whether and how DMA should use secure connections. @@ -74,14 +73,23 @@ security messages directly to the outside mail exchangers. insecure allow plain text SMTP login over an insecure connection. - Should really not be used anymore! + Should really *not* be used anymore! +smarthost + The mail server used to send email. + It must be configured to act as a relay for the host being configured by + this type so that mail can be sent to users non-local to the smarthost. + EXAMPLES -------- .. code-block:: sh - __dma --smarthost mx1.domain.tld --send-test-email + # Install DMA and use the smarthost mx1.domain.tld to send mail. + __dma --smarthost mx1.domain.tld --send-test-mail + + # Install DMA in a default configuration. + __dma SEE ALSO From f76bcd35742d4dc0db2af247b2be51fc6ae6132c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 1 Oct 2020 17:21:19 +0200 Subject: [PATCH 124/156] [type/__dma_auth] Update man.rst --- type/__dma_auth/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__dma_auth/man.rst b/type/__dma_auth/man.rst index f56bfb7..da76883 100644 --- a/type/__dma_auth/man.rst +++ b/type/__dma_auth/man.rst @@ -27,7 +27,7 @@ password OPTIONAL PARAMETERS ------------------- state - Either `present` or `absent`. Defaults to `present`. + Either ``present`` or ``absent``. Defaults to ``present``. BOOLEAN PARAMETERS ------------------ From 14f7cf88396f8a2870b4edfdb12f9b8448ac48b8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 4 Oct 2020 17:40:01 +0200 Subject: [PATCH 125/156] __netbox: set --data-root default value The default value is set to preserve all data saved by netbox. As explained in the manpage, it sets the directory for all data directories to `~netbox/data/` (would resolve to `/opt/netbox/data/`), so upgrades will not remove this data. --- type/__netbox/man.rst | 17 ++++++++++------- type/__netbox/manifest | 11 +++++------ type/__netbox/parameter/default/data-root | 1 + 3 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 type/__netbox/parameter/default/data-root diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 3330b1b..6d50268 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -131,7 +131,7 @@ data-root This parameter set's the media, reports and scripts root to subdirectories of the given directory. Values can be overwritten by special parameters like `--media-root` for example. Use this option if you want to store persistant - data of netbox on an other partition. A trailing shlash is not needed. + data of netbox on an other partition. A trailing slash is not needed. The data directories have following predefined sub-directory names: @@ -142,23 +142,26 @@ data-root scripts root: ``$data_root/scripts`` + To preserve all data from installation upgrades - which just replace the + installation directory - the data will be kept in the netbox home directory + rather than the installation directory by default (``/opt/netbox/data/``). + This way, no data will be deleted after the installation directory + replacement because it remains outside of the installation directory. + media-root The file path to where media files (like image attachments) are stored. Change this path if you require to store data on an other partiotion. - A trailing slash is not needed. By default, it will be stored into the - installation directory (``/opt/netbox/netbox/netbox/media``). + A trailing slash is not needed. Defaults to ``$data_root/media``. reports-root The file path of where custom reports are kept. Change this path if you require to store data on an other partition. A trailing slash is not - needed. By default, it will be stored into the installation directory - (``/opt/netbox/netbox/netbox/reports``). + needed. Defaults to ``$data_root/reports``. scripts-root The file path of where custom scripts are kept. Change this path if you require to store data on an other partition. A trailing slash is not - needed. By default, it will be stored into the installation directory - (``/opt/netbox/netbox/netbox/scripts``). + needed. Defaults to ``$data_root/scripts``. BOOLEAN PARAMETERS diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 89b89ac..fbcf253 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -161,12 +161,11 @@ else fi export LOGIN_REQUIRED -if [ -f "$__object/parameter/data-root" ]; then - data_root="$(cat "$__object/parameter/data-root")" - MEDIA_ROOT="$data_root/media" - REPORTS_ROOT="$data_root/reports" - SCRIPTS_ROOT="$data_root/scripts" -fi +data_root="$(cat "$__object/parameter/data-root")" +MEDIA_ROOT="$data_root/media" +REPORTS_ROOT="$data_root/reports" +SCRIPTS_ROOT="$data_root/scripts" + if [ -f "$__object/parameter/media-root" ]; then MEDIA_ROOT="$(cat "$__object/parameter/media-root")" fi diff --git a/type/__netbox/parameter/default/data-root b/type/__netbox/parameter/default/data-root new file mode 100644 index 0000000..45faab0 --- /dev/null +++ b/type/__netbox/parameter/default/data-root @@ -0,0 +1 @@ +/opt/netbox/data From 0c85b2d3fd3e9dee6a79091064e4fb7078adcdfb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 5 Oct 2020 18:26:47 +0200 Subject: [PATCH 126/156] __netbox: uninstall not anymore required packages This code tries to remove packages not needed anymore. As it presumably not removing dependencies, it's not that perfect pice of script. --- type/__netbox/gencode-remote | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d7bc219..1cd4939 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -36,18 +36,35 @@ ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || tr EOF cat << EOF +# backup requirement files +if [ -f /opt/netbox/requirements.txt]; then + cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt +else + # preseve file-not-found errors and warnings + touch /opt/netbox/old-requirements.txt +fi +cp '$src/requirements.txt' /opt/netbox/ + # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' -cp '$src/requirements.txt' /opt/netbox/ # backup dependency info ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' + # Setup & enter python virtualenv. # forcing python3 to be sure (till python4 gets released ..) virtualenv -p python3 /opt/netbox/venv +# Uninstall packages not required anymore +# if versions not shortend, they will be ignored by pip, but not by comm +# all of this could be done with grep, too, but it's still must be shortend with awk +awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" +awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" +comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ + | xargs /opt/netbox/venv/bin/pip3 uninstall -y + # Install python dependencies. # avoid gunicorn, because it will be done in an other type grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ @@ -57,7 +74,7 @@ EOF if [ -f "$__object/parameter/ldap-server" ]; then echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" else - echo "/opt/netbox/venv/bin/pip3 uninstall django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 uninstall -y django-auth-ldap" fi cat << EOF From 8f1b56026ca75b6a9d66dd40d40c239c34f6927b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 5 Oct 2020 19:02:05 +0200 Subject: [PATCH 127/156] __netbox: force link creation for config files This adds the force flag to `ln` to avoid aborts cause the link already exists and so forth. It also adds robustness to the execution. --- type/__netbox/gencode-remote | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 1cd4939..3b4e05c 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -48,9 +48,9 @@ cp '$src/requirements.txt' /opt/netbox/ # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' - -ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' -ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' +# force links to the cdist directory +ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. From 0b3bc1453073de6e45101e9386b5de1e8f93227a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 6 Oct 2020 17:29:31 +0200 Subject: [PATCH 128/156] __netbox: adjusted command flags Mostly added -q to the pip calls. Also forced the temp directory removal. --- type/__netbox/gencode-remote | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 3b4e05c..eb3bcd4 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -63,18 +63,18 @@ virtualenv -p python3 /opt/netbox/venv awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ - | xargs /opt/netbox/venv/bin/pip3 uninstall -y + | xargs /opt/netbox/venv/bin/pip3 uninstall -qy # Install python dependencies. # avoid gunicorn, because it will be done in an other type grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ - | xargs /opt/netbox/venv/bin/pip3 install + | xargs /opt/netbox/venv/bin/pip3 install -q EOF if [ -f "$__object/parameter/ldap-server" ]; then - echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 install -q django-auth-ldap" else - echo "/opt/netbox/venv/bin/pip3 uninstall -y django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 uninstall -qy django-auth-ldap" fi cat << EOF @@ -95,7 +95,7 @@ sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py invalid # Remove temporary working directory. cd / -rm -r "\$tmpdir" +rm -rf "\$tmpdir" # Save version after successful installation printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version From 3b780c4794b1c9df9e3f6582e7ad17ad1cfd0402 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 8 Oct 2020 19:54:48 +0200 Subject: [PATCH 129/156] __netbox: fixup small bugs and venv process killing Changed flag (force to ignore a non-existant directory), typo and swapped arguments are done. Also, the process to stop all processes from the virtal environment has changed: Now, it stops all potential services and ignore errors (because a service doesn't exist). After that, it sends a kill signal to all processes and then gracefully wait since there is no option to do that with systemd. --- type/__netbox/gencode-remote | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index eb3bcd4..1ef78cd 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -28,16 +28,24 @@ EOF # Stop everything in the pyenv to update cat << EOF -# Try to kill everything in the venv -systemctl -q --wait stop netbox gunicorn-netbox uwsgi-netbox || true -# don't know if this is required since using --wait -ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true +# Try to stop everything in the venv (ignore non-existant services) +systemctl -q stop netbox gunicorn-netbox uwsgi-netbox || true + +# kill and poll till all venv processes end +# at least if they are called with full path in the cmd (like in the services) +pids="\$( ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' )" +if [ "\$pids" ]; then + kill \$pids + while ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | grep -q . ; do + sleep 0.5 + done +fi EOF cat << EOF # backup requirement files -if [ -f /opt/netbox/requirements.txt]; then +if [ -f /opt/netbox/requirements.txt ]; then cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt else # preseve file-not-found errors and warnings @@ -46,7 +54,7 @@ fi cp '$src/requirements.txt' /opt/netbox/ # Deploy sources and restore configuration. -rm -r '$install_dir' +rm -rf '$install_dir' cp -r '$src/netbox' '$install_dir' # force links to the cdist directory ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' @@ -62,7 +70,7 @@ virtualenv -p python3 /opt/netbox/venv # all of this could be done with grep, too, but it's still must be shortend with awk awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" -comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ +comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" \ | xargs /opt/netbox/venv/bin/pip3 uninstall -qy # Install python dependencies. From 13e97d171bb700f577cdcfb03dde81a193b0c10c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:39:19 +0200 Subject: [PATCH 130/156] __netbox*: added systemd socket support The Gunicorn type now supports systemd sockets only. With uWSGI, you can choose between it and the native sockets based on the parameters chosen. This is done because it could not be implemented to have multiple protocols with the systemd sockets (so you may choose). The systemd socket unit file is generally available, so both types use the same script to generate the socket unit file. --- type/__netbox/files/netbox.socket.sh | 33 ++++++++++ type/__netbox_gunicorn/files/gunicorn.py.sh | 2 +- type/__netbox_gunicorn/files/netbox.service | 3 +- type/__netbox_gunicorn/files/netbox.socket.sh | 1 + type/__netbox_gunicorn/man.rst | 8 ++- type/__netbox_gunicorn/manifest | 22 +++---- type/__netbox_uwsgi/explorer/bind-capability | 12 ++++ .../{netbox.service => netbox.service.sh} | 18 +++++- type/__netbox_uwsgi/files/netbox.socket.sh | 1 + type/__netbox_uwsgi/files/uwsgi.ini.sh | 24 ++++---- type/__netbox_uwsgi/gencode-remote | 60 ++++++++++++++++++- type/__netbox_uwsgi/man.rst | 33 +++++++++- type/__netbox_uwsgi/manifest | 38 +++++++++++- .../__netbox_uwsgi/parameter/default/protocol | 1 + type/__netbox_uwsgi/parameter/optional | 1 + 15 files changed, 218 insertions(+), 39 deletions(-) create mode 100755 type/__netbox/files/netbox.socket.sh create mode 120000 type/__netbox_gunicorn/files/netbox.socket.sh create mode 100755 type/__netbox_uwsgi/explorer/bind-capability rename type/__netbox_uwsgi/files/{netbox.service => netbox.service.sh} (70%) mode change 100644 => 100755 create mode 120000 type/__netbox_uwsgi/files/netbox.socket.sh create mode 100644 type/__netbox_uwsgi/parameter/default/protocol diff --git a/type/__netbox/files/netbox.socket.sh b/type/__netbox/files/netbox.socket.sh new file mode 100755 index 0000000..03e3d44 --- /dev/null +++ b/type/__netbox/files/netbox.socket.sh @@ -0,0 +1,33 @@ +#!/bin/sh -e +# __netbox/files/netbox.socket.sh + +# This is shared between all WSGI-server types. + +# Arguments: +# 1: File which list all sockets to listen on (sepearated by \n) + +if [ $# -ne 1 ]; then + printf "netbox.socket.sh: argument \$1 missing or too much given!\n" >&2 + exit 1 +fi + + +cat << UNIT +[Unit] +Description=Socket for NetBox via $TYPE + +[Socket] +UNIT + +# read all sockets to listen to +while read line; do + printf "ListenStream=%s\n" "$line" +done < "$1" + +cat << UNIT +SocketUser=netbox +SocketGroup=www-data + +[Install] +WantedBy=sockets.target +UNIT diff --git a/type/__netbox_gunicorn/files/gunicorn.py.sh b/type/__netbox_gunicorn/files/gunicorn.py.sh index 862ba06..c1e6ee5 100755 --- a/type/__netbox_gunicorn/files/gunicorn.py.sh +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -13,7 +13,7 @@ cores="$(cat "$__explorer/cpu_cores")" cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = [$HOST ] +#bind = done via systemd socket 'gunicorn-netbox.socket' # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index dcf1a66..28b6b45 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -3,13 +3,14 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service Requires=netbox-rq.service +Requires=gunicorn-netbox.socket Wants=network.target After=netbox.service After=network.target After=redis-server.service postgresql.service [Service] -Type=simple +Type=notify User=netbox Group=netbox diff --git a/type/__netbox_gunicorn/files/netbox.socket.sh b/type/__netbox_gunicorn/files/netbox.socket.sh new file mode 120000 index 0000000..28ce920 --- /dev/null +++ b/type/__netbox_gunicorn/files/netbox.socket.sh @@ -0,0 +1 @@ +../../__netbox/files/netbox.socket.sh \ No newline at end of file diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 1c09d78..8860294 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -14,6 +14,9 @@ sockets. Static content must be served independent of Gunicorn. The Gunicorn daemon is available as the `gunicorn-netbox` systemd service, but also available via the `netbox` wrapper service. +It will use systemd socket activation to listen to the given sockets. This +should allow to bind to privileaged ports (all below 1024) and hot reloads. + REQUIRED PARAMETERS ------------------- @@ -38,8 +41,9 @@ state bind-to The hosts the gunicorn socket should be bind to. Formats are `IP`, - `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple - times. Defaults to ``127.0.0.1:8001``. + `IP:PORT`, `PATH` or anything other that systemd socket units will + understand as stream. Parameter can be set multiple times. Defaults + to ``127.0.0.1:8001``. BOOLEAN PARAMETERS diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index 2e5ccce..5748e9d 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -25,17 +25,10 @@ case "$param_state" in esac -if [ "$state" = "present" ]; then - HOST="" - while read -r host; do - # shellcheck disable=SC2089 - HOST="$HOST '$host'," - done < "$__object/parameter/bind-to" - # shellcheck disable=SC2090 - export HOST +mkdir "$__object/files" +if [ "$state" = "present" ]; then # process template - mkdir "$__object/files" "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" # gunicorn config file @@ -49,7 +42,16 @@ else fi -# install service file +TYPE="Gunicorn" +export TYPE + +"$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \ + > "$__object/files/netbox.socket" + +# install systemd files +__systemd_unit gunicorn-netbox.socket \ + --state "$state" --enablement-state "$unit_state" \ + --source "$__object/files/netbox.socket" --restart __systemd_unit gunicorn-netbox.service \ --state "$state" --enablement-state "$unit_state" \ --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/explorer/bind-capability b/type/__netbox_uwsgi/explorer/bind-capability new file mode 100755 index 0000000..c5c0365 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/bind-capability @@ -0,0 +1,12 @@ +#!/bin/sh -e +# explorer/bind-capablility + +# Checks if the uWSGI binary have the capability to bind to privileaged ports +# as a non-root user. It's required if no systemd sockets are used (cause of +# the use of multiple protocols etc.) + +binary="/opt/netbox/venv/bin/uwsgi" +# -v verifies if capability is set +if setcap -q -v CAP_NET_BIND_SERVICE+ep "$binary"; then + echo set +fi diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service.sh old mode 100644 new mode 100755 similarity index 70% rename from type/__netbox_uwsgi/files/netbox.service rename to type/__netbox_uwsgi/files/netbox.service.sh index cd06e82..3705769 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service.sh @@ -1,15 +1,26 @@ +#!/bin/sh -e + +cat << EOF [Unit] Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service Requires=netbox-rq.service +EOF + +# Add dependency to own socket +if [ "$(cat "$__object/files/systemd_socket")" = "yes" ]; then + echo "Requires=uwsgi-netbox.socket" +fi + +cat << EOF Wants=network.target After=netbox.service After=network.target After=redis-server.service postgresql.service [Service] -Type=simple +Type=notify User=netbox Group=netbox @@ -17,8 +28,8 @@ WorkingDirectory=/opt/netbox ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi -ExecReload=kill -HUP $MAINPID -ExecStop=kill -INT $MAINPID +ExecReload=kill -HUP \$MAINPID +ExecStop=kill -INT \$MAINPID KillSignal=SIGQUIT Restart=on-failure @@ -26,3 +37,4 @@ RestartSec=30 [Install] WantedBy=netbox.service +EOF diff --git a/type/__netbox_uwsgi/files/netbox.socket.sh b/type/__netbox_uwsgi/files/netbox.socket.sh new file mode 120000 index 0000000..28ce920 --- /dev/null +++ b/type/__netbox_uwsgi/files/netbox.socket.sh @@ -0,0 +1 @@ +../../__netbox/files/netbox.socket.sh \ No newline at end of file diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 7835a3e..4bae613 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -28,20 +28,16 @@ cat << EOF ; socket(s) to bind EOF -# special protocol to bind -while read -r param; do - if [ -z "$param" ]; then continue; fi # ignore empty lines from the here-doc - - multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" - socket_changes="yes" - -done << INPUT # here-doc cause of SC2031 -$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print ) -INPUT - -# else, default bind to -if [ -z "$socket_changes" ]; then - multi_options "socket" "$__object/parameter/bind-to" +if [ "$SYSTEMD_SOCKET" != "yes" ]; then + # special protocol to bind + find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print \ + | while read -r param; do + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" + done +else + # else, systemd will offer socket + echo "; sockets managed via 'uwsgi-netbox.socket'" + printf "protocol = %s\n" "$PROTOCOL" fi diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index c219643..7c3b826 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -3,18 +3,67 @@ # control state state="$(cat "$__object/parameter/state")" +# Set capabilities to aquire privileaged ports as netbox user. Two modes are +# available to efficiently set capabilites. Assumes libcap-bin is installed as +# default on debian systems. +# +# Arguments: +# 1: mode to detect if capabilites are required to set ('set' or 'correct') +set_bind_cap() { + cap_mode="" # reset variable from the execution before + + # check if capabilites are required after given mode + case "$1" in + # assumes capabilites are not set (cause of new binaries) + set) + if [ "$SYSTEMD_SOCKET" != "yes" ]; then + cap_mode="+ep" + fi + ;; + + # check if capabilities have changed + correct) + if [ -s "$__object/explorer/bind-capability" ]; then + # capabilites are set + if [ "$SYSTEMD_SOCKET" = "yes" ]; then + cap_mode="-ep" # unset + fi + else + # capabilities are unset + if [ "$SYSTEMD_SOCKET" != "yes" ]; then + cap_mode="+ep" # set + fi + fi + ;; + + # faulty mode + *) + echo "called set_bind_cap incorrect (\$1 missing)" >&2 + ;; + esac + + # set capabilities if any + if [ "$cap_mode" ]; then + printf "setcap -q CAP_NET_BIND_SERVICE%s /opt/netbox/venv/bin/uwsgi\n" "$cap_mode" + fi +} +SYSTEMD_SOCKET="$(cat "$__object/files/systemd_socket")" + + case "$state" in # install uwsgi enabled|disabled) # not installed if ! [ -s "$__object/explorer/installed" ]; then - echo "/opt/netbox/venv/bin/pip3 install uwsgi" + echo "/opt/netbox/venv/bin/pip3 install -q uwsgi" + set_bind_cap set do_restart=yes printf "installed\n" >> "$__messages_out" # updates available elif [ -s "$__object/explorer/upgradeable" ]; then - echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + echo "/opt/netbox/venv/bin/pip3 install -q --upgrade uwsgi" + set_bind_cap set do_restart=yes printf "upgraded\n" >> "$__messages_out" fi @@ -25,6 +74,11 @@ case "$state" in printf "configured\n" >> "$__messages_out" fi + # if no capabilities were set yet, check if any are required + if [ -z "$cap_mode" ]; then + set_bind_cap correct + fi + # restart uwsgi if [ "$do_restart" ] && [ "$state" != "disabled" ]; then @@ -40,7 +94,7 @@ EOF # check if installed if [ -s "$__object/explorer/installed" ]; then # service already disabled - echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi" + echo "/opt/netbox/venv/bin/pip3 uninstall -qy uwsgi" printf "uninstalled\n" >> "$__messages_out" fi ;; diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 3fb8515..d54401e 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -15,6 +15,19 @@ protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This application is available via the `uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper service, too. +**As uWSGI will be started as netbox user, it does not have privileges to +bind to a privileaged port (all ports below 1024).** Because uWSGI will +drop privileages anyway before binding to a port, solutions are to use +the systemd sockets to activate the ports as root or set linux kernel +capabilites to bind to such a privileaged port. + +As systemd sockets (or uwsgi itself) do not allow to distinguish multiple +sockets if different protocols are used for different sockets, this type does +not use systemd sockets if it is requested from the user. Using the +``--bind-to`` and ``--protocol`` parameters, it uses the systemd socket +activation. Else, it set the different sockets and protocols natively to uwsgi +and add kernel capabilities to be able to listen to privileaged ports. + REQUIRED PARAMETERS ------------------- @@ -39,8 +52,18 @@ state bind-to - The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. - Defaults to ``127.0.0.1:3031``. Can be set multiple times. + The socket uwsgi should bind to. Must be UNIX/TCP (or anything that + systemd sockets accept as stream). Defaults to ``127.0.0.1:3031``. Can be + set multiple times. The used protocol is defined by ``--protocol``. + + **By setting up the socket via this parameter, it uses systemd sockets to + handle these.** This parameter will be ignored if a more detailed paramter + is given (``--$proto-bind``). + +protocol + The protocol which should be used for the socket given by the ``--bind-to`` + parameter. Possible values are ``uwsgi``, ``http``, ``fastcgi`` and + ``scgi``. If nothing given, it defaults to ``uwsgi``. uwsgi-bind http-bind @@ -50,6 +73,12 @@ scgi-bind ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must be a UNIX/TCP socket. Can be set multiple times. + **By using such parameters instead of ``--bind-to``, no systemd sockets + will be used because it can not handle sockets for multiple protocols.** + Instead, the native socket binding will be used. It will add kernel + capabilites to bind to privileaged ports, too. This allow binds to ports + like 80 as netbox user. + BOOLEAN PARAMETERS ------------------ diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index c5885c9..7c593e8 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -25,15 +25,30 @@ case "$param_state" in esac +mkdir "$__object/files" + +# check if systemd sockets will be used +if [ -f "$__object/parameter/bind-to" ]; then + SYSTEMD_SOCKET="yes" +fi +if find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print -quit | grep -q .; then + SYSTEMD_SOCKET="no" +fi +echo "$SYSTEMD_SOCKET" > "$__object/files/systemd_socket" + if [ "$state" = "present" ]; then - # *bind* parameters are directly processed in the gen script + # already checked outside this if-clause + export SYSTEMD_SOCKET + + PROTOCOL="$(cat "$__object/parameter/protocol")" + export PROTOCOL + if [ -f "$__object/parameter/serve-static" ]; then STATIC_MAP="yes" export STATIC_MAP fi # process template - mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" # uwsgi config file @@ -48,7 +63,24 @@ else fi +# handle the systemd socket +if [ "$SYSTEMD_SOCKET" = "yes" ]; then + TYPE="uWSGI" + export TYPE + + # generate and set the socket unit + "$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \ + > "$__object/files/netbox.socket" + __systemd_unit uwsgi-netbox.socket \ + --state "$state" --enablement-state "$unit_state" \ + --source "$__object/files/netbox.socket" --restart +else + # remove the systemd socket unit + __systemd_unit uwsgi-netbox.socket --state absent +fi + # install service file +"$__type/files/netbox.service.sh" > "$__object/files/netbox.service" __systemd_unit uwsgi-netbox.service \ --state "$state" --enablement-state "$unit_state" \ - --source "$__type/files/netbox.service" --restart + --source "$__object/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/parameter/default/protocol b/type/__netbox_uwsgi/parameter/default/protocol new file mode 100644 index 0000000..caf986e --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/protocol @@ -0,0 +1 @@ +uwsgi diff --git a/type/__netbox_uwsgi/parameter/optional b/type/__netbox_uwsgi/parameter/optional index ff72b5c..3284ccc 100644 --- a/type/__netbox_uwsgi/parameter/optional +++ b/type/__netbox_uwsgi/parameter/optional @@ -1 +1,2 @@ state +protocol From 32c95ccb220cb3302d1c43cca5ace01fa66a6755 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:50:22 +0200 Subject: [PATCH 131/156] __netbox: don't call `pip3 uninstall` if nothing available To avoid an error from pip, it will be checked if there are packages to uninstall. This is better than just doing `|| true`. --- type/__netbox/gencode-remote | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 1ef78cd..07b0fcf 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -66,12 +66,16 @@ ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Uninstall packages not required anymore -# if versions not shortend, they will be ignored by pip, but not by comm +# if versions not be shortend, they will be ignored by pip, but not by comm # all of this could be done with grep, too, but it's still must be shortend with awk awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" -comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" \ - | xargs /opt/netbox/venv/bin/pip3 uninstall -qy +comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" > "\$tmpdir/pip-uninstall.txt" + +# only uninstall if something is available (to avoid errors cause of this) +if [ -s "\$tmpdir/pip-uninstall.txt" ]; then + /opt/netbox/venv/bin/pip3 uninstall -qy -r "\$tmpdir/pip-uninstall.txt" +fi # Install python dependencies. # avoid gunicorn, because it will be done in an other type @@ -86,6 +90,7 @@ EOF fi cat << EOF + # Set final permissions. chown -R netbox /opt/netbox From 72f2996c509e64a9f35f53d7b347ae65d1e6df59 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:56:54 +0200 Subject: [PATCH 132/156] __netbox*: fix shellcheck SC2162 --- type/__netbox/files/netbox.socket.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/netbox.socket.sh b/type/__netbox/files/netbox.socket.sh index 03e3d44..2ef9e81 100755 --- a/type/__netbox/files/netbox.socket.sh +++ b/type/__netbox/files/netbox.socket.sh @@ -20,7 +20,7 @@ Description=Socket for NetBox via $TYPE UNIT # read all sockets to listen to -while read line; do +while read -r line; do printf "ListenStream=%s\n" "$line" done < "$1" From 144cae3b796831d4c96e221611b25945f4462f68 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 10:16:25 +0200 Subject: [PATCH 133/156] __netbox: use __pyenv and remove obsolete netbox stop Now, the python venv is now created via `pyvenv` or `python3 -m venv` instead of the legacy `virtualenv`. For this, not all python processes from the venv need to be stopped. Migration from previous versions of this type may be difficult, but solvable if the venv will be recreated. --- type/__netbox/gencode-remote | 32 +++++++------------------------- type/__netbox/manifest | 2 ++ 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 07b0fcf..591717f 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -24,26 +24,7 @@ cd "\$tmpdir" curl -sS -L '$url' > '$archive' tar xf '$archive' -EOF - # Stop everything in the pyenv to update - cat << EOF -# Try to stop everything in the venv (ignore non-existant services) -systemctl -q stop netbox gunicorn-netbox uwsgi-netbox || true - -# kill and poll till all venv processes end -# at least if they are called with full path in the cmd (like in the services) -pids="\$( ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' )" -if [ "\$pids" ]; then - kill \$pids - while ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | grep -q . ; do - sleep 0.5 - done -fi - -EOF - - cat << EOF # backup requirement files if [ -f /opt/netbox/requirements.txt ]; then cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt @@ -61,9 +42,7 @@ ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' -# Setup & enter python virtualenv. -# forcing python3 to be sure (till python4 gets released ..) -virtualenv -p python3 /opt/netbox/venv +# virtualenv is given already by __pyvenv, just using it # Uninstall packages not required anymore # if versions not be shortend, they will be ignored by pip, but not by comm @@ -112,6 +91,7 @@ rm -rf "\$tmpdir" # Save version after successful installation printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version + EOF # meta @@ -127,10 +107,12 @@ if grep -q "^__file/opt/netbox/" "$__messages_in"; then fi -# check for changes +# Check for changes if [ "$changes" = "yes" ]; then - # to avoid database corruption at config changes, both services must be - # stopped at the same time (noted in the manual, too). + # After the upstream upgrade.sh script, it's ok to migrate while the + # application is running ;) + + # restarting after changes cat << EOF # Restart service. All required services are included with netbox.service. systemctl restart netbox diff --git a/type/__netbox/manifest b/type/__netbox/manifest index fbcf253..475cadb 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -187,6 +187,8 @@ fi # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home +# Generate python environment (user will be set by gencode-remote) +require="__user/netbox" __pyvenv /opt/netbox/venv/ # Generate and upload netbox configuration. mkdir -p "$__object/files" From 9def5700c8a1de7e59ed0ac3c40cdd1b7c484e05 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 10:21:52 +0200 Subject: [PATCH 134/156] __netbox: replace redis DEFAULT_TIMEOUT with RQ_DEFAULT_TIMEOUT The first one was deprecated by the second one. --- type/__netbox/files/configuration.py.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 06a1c86..31ebd05 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -36,7 +36,6 @@ REDIS = { # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '$REDIS_PASSWORD', 'DATABASE': $((REDIS_DBID_OFFSET + 0)), - 'DEFAULT_TIMEOUT': 300, 'SSL': $REDIS_SSL, }, 'caching': { @@ -47,10 +46,10 @@ REDIS = { # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '$REDIS_PASSWORD', 'DATABASE': $((REDIS_DBID_OFFSET + 1)), - 'DEFAULT_TIMEOUT': 300, 'SSL': $REDIS_SSL, } } +RQ_DEFAULT_TIMEOUT = 300 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and From 558021d2eeb10697d2dec0e7dc563fbf26e63ba0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 11:08:11 +0200 Subject: [PATCH 135/156] __netbox: reorder generated code for shorter transition Now, the venv and pip things are done before the program files are replaced, which should minimize the outage where the application is not available. --- type/__netbox/gencode-remote | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 591717f..5d4b7be 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -25,30 +25,22 @@ curl -sS -L '$url' > '$archive' tar xf '$archive' +# virtualenv is given already by __pyvenv, just using it + # backup requirement files if [ -f /opt/netbox/requirements.txt ]; then - cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt + mv /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt else # preseve file-not-found errors and warnings touch /opt/netbox/old-requirements.txt fi cp '$src/requirements.txt' /opt/netbox/ -# Deploy sources and restore configuration. -rm -rf '$install_dir' -cp -r '$src/netbox' '$install_dir' -# force links to the cdist directory -ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' -ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' - - -# virtualenv is given already by __pyvenv, just using it - # Uninstall packages not required anymore # if versions not be shortend, they will be ignored by pip, but not by comm # all of this could be done with grep, too, but it's still must be shortend with awk -awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" -awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" +awk -F== '{print \$1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" +awk -F== '{print \$1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" > "\$tmpdir/pip-uninstall.txt" # only uninstall if something is available (to avoid errors cause of this) @@ -70,9 +62,17 @@ EOF cat << EOF +# Deploy sources and restore configuration. +rm -rf '$install_dir' +cp -r '$src/netbox' '$install_dir' +# force links to the cdist directory +ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' + # Set final permissions. chown -R netbox /opt/netbox + # NetBox manage scripts # Run database migrations. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate From 46bff41f6d8bb9a0fd50ac7b659690f5e94377dc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 11:22:47 +0200 Subject: [PATCH 136/156] __netbox{,_uwsgi}: updated manpages Mostly added notes of what does not work that well. --- type/__netbox/man.rst | 28 +++++++++++++--------------- type/__netbox_uwsgi/man.rst | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 6d50268..f34ead7 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -16,7 +16,7 @@ It setup systemd unit files for the services `netbox` and `netbox-rq`. The `netbox` service only wrap all netbox related services, e.g. restarting and so one will be delegated to all related services. -The application is still not accessable because WSGI server is required. To +The application is still not accessable because a WSGI server is required. To access the application through WSGI, uWSGI or Gunicorn can be used. The setup can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. @@ -236,23 +236,21 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. -If you change a configuration, the database may go corrupt if two instances of -the application are running with different configurations at the same time. -This most commonly happens when the WSGI server and RQ-worker restarts after a -configuration change. This occours in the following case for example: +You may also be interested of writing a own type which handles the creation of +the super user. To do this non-interactivly, see the ansible role as `reference +`. -.. code-block:: sh +If you change the secret key while the netbox instance is running, there is a +time frame where the access to the application corrupts the whole database. +Then, you need to restore a backup or wipe the database. - systemctl restart gunicorn-netbox # WSGI-server already online with new - # configuration after this command. - systemctl restart netbox-rq # RQ-Worker still worked with the old - # configuration till here. +Currently, the cause is not clear, but it should work if you do not touch +netbox while the configuration is done (do not shut it down, too). It only +applies for changes of the secret key, which not happen normally. -This type handles the restart of both services correctly to avoid such database -corruptions. To safely manual restart the whole netbox instance manual, simply -restart all services in one ``systemctl restart netbox`` command, as it ensures -that first all services are shut down before starting one of them. The service -``netbox`` wraps all required services that are available. +Maybe the `--restart` flag for the `__systemd_unit` types is not the best idea, +but avoids that the changes will not be applied. It could be solved if the type +would send messages from his actions. SEE ALSO diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index d54401e..c5e51ca 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -129,6 +129,7 @@ EXAMPLES # with multiple protocols # parameter `--bind-to` will be ignored + # avoids systemd sockets, but can handle multiple protocols __netbox $args require="__netbox" __netbox_uwsgi --uwsgi-bind 0.0.0.0:3031 \ --http-bind 0.0.0.0:8080 \ @@ -150,6 +151,20 @@ EXAMPLES require="__netbox_uwsgi" __systemd_service uwsgi-netbox --state stopped +NOTES +----- +If systemd sockets are used, uwsgi can not be reloaded because it does not +handle the socket correctly. It works by completly restarting uwsgi (because +it is near the same cause of the systemd socket) or tweaking the service unit +with the line ``StandardInput=socket``, which limits you to only one address +to bind to (else, the service will not start). + +Maybe someone is interested in enabling log files, because the "log to stdout" +is not the fanciest approach (because it is shown in the journal). See the +`uwsgi documentation ` for +reference. + + SEE ALSO -------- `uWSGI Documentation `_ From c1d5d5b088724c22c8b8e79f00a1b82ed88436f2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 08:52:43 +0200 Subject: [PATCH 137/156] __netbox: remove dependency to package virtualenv This is not required any more because it was completly replaced by `python3-venv`. The venv handling is already given to `__pyvenv`. --- type/__netbox/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 475cadb..4be49bd 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -6,7 +6,7 @@ case "$os" in debian|ubuntu) # Install netbox dependencies. for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl sudo; do __package $pkg done From 87e115da98425c69d8bb37397ccdf47c0f2a4dc2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 25 Oct 2020 19:13:10 +0100 Subject: [PATCH 138/156] __netbox: some manpage fixes Mostly invalid links .. --- type/__netbox/man.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index f34ead7..135304c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -100,8 +100,7 @@ redis-password redis-dbid-offset Offset to set the redis database id's. The `tasks` database id is - `offset + 0` and `caching` is `offset + 1`. The offset defaults - to ``0``. + `offset + 0` and `caching` is `offset + 1`. The offset defaults to ``0``. smtp-host Host of the SMTP email server. Defaults to ``localhost``. @@ -172,12 +171,12 @@ redis-ssl smtp-use-tls Uses TLS to connect to the SMTP email server. `See documentation - `_ for more information. smtp-use-ssl Uses implicit TLS with the SMTP email server. `See documentation - `_ for more information. login-required @@ -229,16 +228,16 @@ is a more good way to inject python code for dynamic configuration variables, you are welcome to contribute! - `Possible optional settings - ` + `_ If you not setup ldap authentification, you may be interested into how to `setting up a super user -` +`_ directly on the machine to be able to access and use NetBox. You may also be interested of writing a own type which handles the creation of the super user. To do this non-interactivly, see the ansible role as `reference -`. +`_. If you change the secret key while the netbox instance is running, there is a time frame where the access to the application corrupts the whole database. From 04076a75eb728b8d4ad50b31c1b0bd87a9d781a8 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 16 Nov 2020 13:57:34 +0100 Subject: [PATCH 139/156] [type/__mail_alias] man.rst: Make bugs a list --- type/__mail_alias/man.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst index d6d9742..de40512 100644 --- a/type/__mail_alias/man.rst +++ b/type/__mail_alias/man.rst @@ -48,11 +48,15 @@ EXAMPLES BUGS ---- -- Quoted strings are not parsed by this type. As a result, email addresses - containing ``,`` (commas) are treated incorrectly (they are treated as two - addresses/aliases.) - Make sure that email addresses do not contain commas. -- ``:include:`` directives in the aliases file are ignored by this type. +- Quoted strings are not parsed by this type. As a result, aliases + containing ``,`` (commas) are treated incorrectly (they are treated as + separate aliases.) + Make sure that email addresses, file names, and pipe commands do not contain + commas. +- ``:include:`` directives in the aliases file are not evaluated by this type. + They are treated like a regular alias, the values of the included file are + not expanded. + SEE ALSO -------- From 7e20d13b9ff5126580c0ac9a9231c2ebea966985 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 16 Nov 2020 13:58:05 +0100 Subject: [PATCH 140/156] [type/__mail_alias] Use explicit line variables in update_aliases.awk --- type/__mail_alias/files/update_aliases.awk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/type/__mail_alias/files/update_aliases.awk b/type/__mail_alias/files/update_aliases.awk index 336009f..11a4c85 100644 --- a/type/__mail_alias/files/update_aliases.awk +++ b/type/__mail_alias/files/update_aliases.awk @@ -18,11 +18,11 @@ # along with cdist. If not, see . # -function getvalue(path) { +function getvalue(path, line) { # Reads the first line of the file located at path and returns it. - getline < path + getline line < path close(path) - return $0 + return line } function sepafter(f, def, _) { @@ -31,14 +31,14 @@ function sepafter(f, def, _) { return _ ? _ : def } -function write_aliases() { +function write_aliases( line) { if (aliases_written) return # print aliases line printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") - while ((getline < aliases_should_file) > 0) { + while ((getline line < aliases_should_file) > 0) { if (aliases_written) printf ", " - printf "%s", $0 + printf "%s", line aliases_written = 1 } printf "\n" From 0932c9ccde66bdcb75e998f7a730624fdbd3068e Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 16 Nov 2020 14:10:30 +0100 Subject: [PATCH 141/156] [type/{__dma,__dma_auth,__mail_alias}] Quote things properly --- type/__dma/files/update_dma_conf.awk | 2 +- type/__dma/gencode-remote | 17 +++++++++-------- type/__dma_auth/gencode-remote | 11 ++++++----- type/__mail_alias/gencode-remote | 17 +++++++++-------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/type/__dma/files/update_dma_conf.awk b/type/__dma/files/update_dma_conf.awk index 67661fd..2f60a3d 100644 --- a/type/__dma/files/update_dma_conf.awk +++ b/type/__dma/files/update_dma_conf.awk @@ -18,7 +18,7 @@ # along with cdist. If not, see . function comment_line(line) { - # returns the position in line at which the comment'\''s text starts + # returns the position in line at which the comment's text starts # (0 if the line is not a comment) match(line, /^[ \t]*\#+[ \t]*/) return RSTART ? (RLENGTH + 1) : 0 diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 1987106..a33388d 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -18,7 +18,8 @@ # along with cdist. If not, see . # -drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } +quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; } +drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; } CONF_PATH=/etc/dma # set in Makefile @@ -139,13 +140,13 @@ then # options are grouped by word (the first word in the line) and appended # at the end of the file. - cat <'${dma_conf}.tmp' \ -&& cat '${dma_conf}.tmp' >'${dma_conf}' -${conf_should} -EOF -rm '${dma_conf}.tmp' -CODE + cat <<-CODE + awk $(drop_awk_comments "${__type:?}/files/update_dma_conf.awk") $(quote "${dma_conf}") $(quote "${dma_conf}") <<'EOF' >$(quote "${dma_conf}.tmp") \ + && cat $(quote "${dma_conf}.tmp") >$(quote "${dma_conf}") + ${conf_should} + EOF + rm $(quote "${dma_conf}.tmp") + CODE config_updated=true echo 'config updated' >>"${__messages_out:?}" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index aee4d7f..4279b7a 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,7 +18,8 @@ # along with cdist. If not, see . # -drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } +quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; } +drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; } state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") @@ -63,9 +64,9 @@ esac cat <'${auth_conf}.tmp' \ -&& cat '${auth_conf}.tmp' >'${auth_conf}' -rm -f '${auth_conf}.tmp' +awk $(drop_awk_comments "${__type:?}/files/update_dma_auth.awk") <$(quote "${auth_conf}") >$(quote "${auth_conf}.tmp") \ +&& cat $(quote "${auth_conf}.tmp") >$(quote "${auth_conf}") +rm -f $(quote "${auth_conf}.tmp") EOF diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 3eea452..4a8f889 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -18,7 +18,8 @@ # along with cdist. If not, see . # -drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } +quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; } +drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; } aliases_file=$(cat "${__object:?}/explorer/aliases_file") @@ -30,7 +31,7 @@ test -n "${aliases_file}" || { state_should=$(cat "${__object:?}/parameter/state") -case $state_should +case ${state_should} in (present) if cmp -s "${__object:?}/explorer/aliases" "${__object:?}/parameter/alias" @@ -65,22 +66,22 @@ in esac cat <'${aliases_file}.tmp' \ +awk $(drop_awk_comments "${__type:?}/files/update_aliases.awk") <$(quote "${aliases_file}") >$(quote "${aliases_file}.tmp") \ || { - rm -f '${aliases_file}.tmp' + rm -f $(quote "${aliases_file}.tmp") echo 'Generating new aliases file failed!' >&2 exit 1 } -if ! cmp -s '${aliases_file}' '${aliases_file}.tmp' +if ! cmp -s $(quote "${aliases_file}") $(quote "${aliases_file}.tmp") then # aliases file was modified, replace: - cat '${aliases_file}.tmp' >'${aliases_file}' + cat $(quote "${aliases_file}.tmp") >$(quote "${aliases_file}") # then, run newaliases if present ("missing" on Alpine Linux because of typo) command -v newaliases >/dev/null 2>&1 && newaliases || true fi -rm -f '${aliases_file}.tmp' +rm -f $(quote "${aliases_file}.tmp") EOF From 487574c865c0b150d83bc37bfda44c0051f91973 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 16 Nov 2020 14:15:52 +0100 Subject: [PATCH 142/156] [type/__dma] Convert AWK to loop over same file twice --- type/__dma/files/update_dma_conf.awk | 8 ++++++++ type/__dma/gencode-remote | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/type/__dma/files/update_dma_conf.awk b/type/__dma/files/update_dma_conf.awk index 2f60a3d..15ef7bf 100644 --- a/type/__dma/files/update_dma_conf.awk +++ b/type/__dma/files/update_dma_conf.awk @@ -88,6 +88,12 @@ BEGIN { FS = "\n" EQS = "[ \t]" # copied from dma/conf.c + if (ARGV[2]) exit (e=1) + + # Loop over file twice! + ARGV[2] = ARGV[1] + ARGC++ + # read the "should" state into the `conf` array. while (getline < "/dev/stdin") { word = first($0, EQS) @@ -165,6 +171,8 @@ NR > FNR { } END { + if (e) exit + # print rest of config options ( for (word in conf) print_confs(word) } diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index a33388d..fa676d4 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -141,7 +141,7 @@ then # at the end of the file. cat <<-CODE - awk $(drop_awk_comments "${__type:?}/files/update_dma_conf.awk") $(quote "${dma_conf}") $(quote "${dma_conf}") <<'EOF' >$(quote "${dma_conf}.tmp") \ + awk $(drop_awk_comments "${__type:?}/files/update_dma_conf.awk") $(quote "${dma_conf}") <<'EOF' >$(quote "${dma_conf}.tmp") \ && cat $(quote "${dma_conf}.tmp") >$(quote "${dma_conf}") ${conf_should} EOF From 9f4a85901ef8d58e95924dd9f584ca6437934030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 11:04:07 +0100 Subject: [PATCH 143/156] Import html/man doc generation logic from upstream cdist --- .gitlab-ci.yml | 10 +- Makefile | 70 ++++++++++ docs/src/Makefile | 235 ++++++++++++++++++++++++++++++++++ docs/src/conf.py | 101 +++++++++++++++ docs/src/index.rst.sh | 45 +++++++ docs/src/manpage.py | 87 +++++++++++++ type/__matrix_element/man.rst | 2 +- type/__matterbridge/man.rst | 4 +- type/__netbox/man.rst | 5 +- type/__netbox_uwsgi/man.rst | 5 +- 10 files changed, 552 insertions(+), 12 deletions(-) create mode 100644 Makefile create mode 100644 docs/src/Makefile create mode 100644 docs/src/conf.py create mode 100755 docs/src/index.rst.sh create mode 100644 docs/src/manpage.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dba7864..79c532e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,20 @@ stages: - test + - doc image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest shellcheck: stage: test script: - - ./scripts/run-shellcheck.sh + - make lint manpages: stage: test script: - - ./scripts/run-manpage-checks.sh + - make check-manpages + +docs: + stage: doc + script: + - make docs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1a0cfb3 --- /dev/null +++ b/Makefile @@ -0,0 +1,70 @@ +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo "man build only man user documentation" + @echo "html build only html user documentation" + @echo "docs build both man and html user documentation" + @echo "check-manpages check for manpage in types" + @echo "lint run shellcheck on types" + @echo "check run both type manpage checks and linting" + @echo "clean clean" + +DOCS_SRC_DIR=./docs/src +TYPEDIR=./type + +SPHINXM=make -C $(DOCS_SRC_DIR) man +SPHINXH=make -C $(DOCS_SRC_DIR) html +SPHINXC=make -C $(DOCS_SRC_DIR) clean + +################################################################################ +# Manpages +# +MAN7DSTDIR=$(DOCS_SRC_DIR)/man7 + +# Use shell / ls to get complete list - $(TYPEDIR)/*/man.rst does not work +# Using ls does not work if no file with given pattern exist, so use wildcard +MANTYPESRC=$(wildcard $(TYPEDIR)/*/man.rst) +MANTYPEPREFIX=$(subst $(TYPEDIR)/,$(MAN7DSTDIR)/cdist-type,$(MANTYPESRC)) +MANTYPES=$(subst /man.rst,.rst,$(MANTYPEPREFIX)) + +# Link manpage: do not create man.html but correct named file +$(MAN7DSTDIR)/cdist-type%.rst: $(TYPEDIR)/%/man.rst + mkdir -p $(MAN7DSTDIR) + ln -sf "../../../$^" $@ + +DOCSINDEX=$(MAN7DSTDIR)/index.rst +DOCSINDEXH=$(DOCS_SRC_DIR)/index.rst.sh + +$(DOCSINDEX): $(DOCSINDEXH) + $(DOCSINDEXH) + +# Manpages: .cdist Types +DOT_CDIST_PATH=${HOME}/.cdist +DOTMAN7DSTDIR=$(MAN7DSTDIR) +DOTTYPEDIR=$(DOT_CDIST_PATH)/type + +# Link manpage: do not create man.html but correct named file +$(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst + ln -sf "$^" $@ + +man: $(MANTYPES) $(DOCSINDEX) + $(SPHINXM) + +html: $(MANTYPES) $(DOCSINDEX) + $(SPHINXH) + +docs: man html + +check-manpages: + ./scripts/run-manpage-checks.sh + +lint: + ./scripts/run-shellcheck.sh + +check: check-manpages lint + +clean: + $(SPHINXC) + rm -f docs/src/index.rst + rm -rf docs/src/man7/ + rm -rf docs/src/__pycache__/ diff --git a/docs/src/Makefile b/docs/src/Makefile new file mode 100644 index 0000000..2e9d6ce --- /dev/null +++ b/docs/src/Makefile @@ -0,0 +1,235 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +PAPER ?= +BUILDDIR ?= ../dist +# for cache, etc. +_BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) + $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(_BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + rm -rf $(_BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/cdist-docs.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/cdist-docs.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/cdist-docs" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/cdist-docs" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b cman $(ALLSPHINXOPTS) $(BUILDDIR)/man + mkdir -p $(BUILDDIR)/man/man7 + mv -f $(BUILDDIR)/man/*.7 $(BUILDDIR)/man/man7/ + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/src/conf.py b/docs/src/conf.py new file mode 100644 index 0000000..19b2dfd --- /dev/null +++ b/docs/src/conf.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import sys +import os +import sphinx_rtd_theme + +from datetime import date + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", ".."))) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'docs.src.manpage', + 'sphinx.ext.extlinks', +] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = ['.rst'] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'cdist-contrib' +copyright = 'cdist-contrib contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + +version = str(date.today()) +release = os.popen('git rev-parse HEAD').read() + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Output file base name for HTML help builder. +htmlhelp_basename = 'cdistcontribdoc' + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +root_mandir = os.path.dirname(os.path.realpath(__file__)) +mandirs = [] +for mansubdir in ('man7',): + mandirs.append((os.path.join(root_mandir, mansubdir), mansubdir[-1])) +man_pages = [] +for mandir, section in mandirs: + for root, dirs, files in os.walk(mandir): + for fname in files: + froot, fext = os.path.splitext(fname) + if fext == '.rst': + man_page = (os.path.join('man' + str(section), froot), + froot, '', [], section) + man_pages.append(man_page) + +# man_pages = [ +# ('cdist-type', 'cdist-type', 'cdist-type documentation', +# [author], 1), +# ('man7/cdist-type__file', 'cdist-type__file', +# '', [], 1), +# ('cdist-type__directory', 'cdist-type__directory', +# 'cdist-type__directory documentation', [author], 1), +# ] + +# If true, show URL addresses after external links. +# man_show_urls = False diff --git a/docs/src/index.rst.sh b/docs/src/index.rst.sh new file mode 100755 index 0000000..28c5078 --- /dev/null +++ b/docs/src/index.rst.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +__cdist_pwd="$(pwd -P)" +__cdist_mydir="${0%/*}"; +__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)" +__cdist_myname=${0##*/}; +__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" + +filename="${__cdist_myname%.sh}" +dest="$__cdist_abs_mydir/$filename" + +cd "$__cdist_abs_mydir" + +exec > "$dest" +cat << EOF +cdist-contrib - Community maintained cdist types +================================================ + +This project extends the \`cdist \`_ configuration management +tool with community-maitained types which are either too specific to fit/be +maintained in cdist itself or were not accepted in code cdist but could still +be useful. + +Please note this project is a **rolling release**! The documentation you're +reading has been generated from the |version| state (commit |release|). +Sources are available on \`code.ungleich.ch +\`_. + + +.. toctree:: + :hidden: + +EOF + +# If there is no such file then ls prints error to stderr, +# so redirect stderr to /dev/null. +for type in $(ls man7/cdist-type__*.rst 2>/dev/null | LC_ALL=C sort); do + no_dir="${type#man7/}"; + no_type="${no_dir#cdist-type}"; + name="${no_type%.rst}"; + manref="${no_dir%.rst}" + man="${manref}(7)" + + echo " $name" "" +done diff --git a/docs/src/manpage.py b/docs/src/manpage.py new file mode 100644 index 0000000..1f8ac4f --- /dev/null +++ b/docs/src/manpage.py @@ -0,0 +1,87 @@ +import sphinx.builders.manpage +import sphinx.writers.manpage +from docutils.frontend import OptionParser +from sphinx.util.console import bold, darkgreen +from six import string_types +from docutils.io import FileOutput +from os import path +from sphinx.util.nodes import inline_all_toctrees +from sphinx import addnodes +from sphinx.util import logging + +""" + Extension based on sphinx builtin manpage. + It does not write its own .SH NAME based on config, + but leaves everything to actual reStructuredText file content. +""" + + +logger = logging.getLogger(__name__) + + +class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator): + + def header(self): + tmpl = (".TH \"%(title_upper)s\" \"%(manual_section)s\"" + " \"%(date)s\" \"%(version)s\" \"%(manual_group)s\"\n") + return tmpl % self._docinfo + + +class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): + + def __init__(self, builder): + super().__init__(builder) + self.translator_class = ( + self.builder.get_translator_class() or ManualPageTranslator) + + +class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): + + name = 'cman' + default_translator_class = ManualPageTranslator + + def write(self, *ignored): + docwriter = ManualPageWriter(self) + docsettings = OptionParser( + defaults=self.env.settings, + components=(docwriter,), + read_config_files=True).get_default_values() + + logger.info(bold('writing... '), nonl=True) + + for info in self.config.man_pages: + docname, name, description, authors, section = info + if isinstance(authors, string_types): + if authors: + authors = [authors] + else: + authors = [] + + targetname = '%s.%s' % (name, section) + logger.info(darkgreen(targetname) + ' { ', nonl=True) + destination = FileOutput( + destination_path=path.join(self.outdir, targetname), + encoding='utf-8') + + tree = self.env.get_doctree(docname) + docnames = set() + largetree = inline_all_toctrees(self, docnames, docname, tree, + darkgreen, [docname]) + logger.info('} ', nonl=True) + self.env.resolve_references(largetree, docname, self) + # remove pending_xref nodes + for pendingnode in largetree.traverse(addnodes.pending_xref): + pendingnode.replace_self(pendingnode.children) + + largetree.settings = docsettings + largetree.settings.title = name + largetree.settings.subtitle = description + largetree.settings.authors = authors + largetree.settings.section = section + + docwriter.write(largetree, destination) + logger.info("") + + +def setup(app): + app.add_builder(ManualPageBuilder) diff --git a/type/__matrix_element/man.rst b/type/__matrix_element/man.rst index c3e0d5a..05f0685 100644 --- a/type/__matrix_element/man.rst +++ b/type/__matrix_element/man.rst @@ -1,5 +1,5 @@ cdist-type__matrix_element(7) -====================== +============================= NAME ---- diff --git a/type/__matterbridge/man.rst b/type/__matterbridge/man.rst index 02eef2c..babf34b 100644 --- a/type/__matterbridge/man.rst +++ b/type/__matterbridge/man.rst @@ -33,9 +33,9 @@ EXAMPLES .. code-block:: sh - __matterbridge --version 1.16.3 --config - << EOF + __matterbridge --version 1.16.3 --config - <<- EOF [...] -EOF + EOF SEE ALSO diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 135304c..5f78f1d 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -122,7 +122,6 @@ basepath webroot ``/``. For example, if installed at https://example.com/netbox/, set the value ``netbox/``. -http-proxy https-proxy Proxy which will be used with any HTTP request like webhooks. @@ -171,12 +170,12 @@ redis-ssl smtp-use-tls Uses TLS to connect to the SMTP email server. `See documentation - `_ + `__ for more information. smtp-use-ssl Uses implicit TLS with the SMTP email server. `See documentation - `_ + `__ for more information. login-required diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index c5e51ca..13dc6dc 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -65,10 +65,7 @@ protocol parameter. Possible values are ``uwsgi``, ``http``, ``fastcgi`` and ``scgi``. If nothing given, it defaults to ``uwsgi``. -uwsgi-bind -http-bind -fastcgi-bind -scgi-bind +scgi-bind, uwsgi-bind, http-bind, fastcgi-bind Bind the application to a specific protocol instead of implicit uwsgi via ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must be a UNIX/TCP socket. Can be set multiple times. From cd833363224493e8a63d7b0b02a83958ffc97663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 11:26:45 +0100 Subject: [PATCH 144/156] Use our own image for CI --- .gitlab-ci.yml | 2 +- scripts/ci-container/Dockerfile | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 scripts/ci-container/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79c532e..31505a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ stages: - test - doc -image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest +image: code.ungleich.ch:5050/ungleich-public/cdist-contrib/ci-container:latest shellcheck: stage: test diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile new file mode 100644 index 0000000..33646f4 --- /dev/null +++ b/scripts/ci-container/Dockerfile @@ -0,0 +1,7 @@ +# This image is used in the cdist-contrib CI for linting and generating the +# documentation. +FROM fedora:latest +MAINTAINER Timothée Floure + +RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ + ShellCheck From 2bedbe96870d492d7f7d6cd868c210a7ba0e4f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 10:55:25 +0100 Subject: [PATCH 145/156] Automatically deploy docs from master (at contrib.cdi.st) --- .gitlab-ci.yml | 11 +++++++++++ scripts/ci-container/Dockerfile | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31505a5..4b61568 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,5 +16,16 @@ manpages: docs: stage: doc + only: + - master + before_script: + - eval $(ssh-agent -s) + - echo "$CD_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - mkdir -p ~/.ssh + - echo "$CD_SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts script: - make docs + - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<- EOF + put -r docs/dist/html/* + EOF diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile index 33646f4..daa0f26 100644 --- a/scripts/ci-container/Dockerfile +++ b/scripts/ci-container/Dockerfile @@ -4,4 +4,4 @@ FROM fedora:latest MAINTAINER Timothée Floure RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ - ShellCheck + ShellCheck openssh-clients From ec41ef34901bf164b5decb5fa8fa9f150c131a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:05:09 +0100 Subject: [PATCH 146/156] Patch sftp call for doc deployment in CI --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4b61568..9392cd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,4 @@ docs: - chmod 644 ~/.ssh/known_hosts script: - make docs - - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<- EOF - put -r docs/dist/html/* - EOF + - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<< "put -r docs/dist/html/*" From ec0dc30c87a1b770c647641bb3f3abaa1f735032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:34:54 +0100 Subject: [PATCH 147/156] Generate documentation homepage from README --- .gitlab-ci.yml | 2 +- README.md | 3 ++- docs/src/index.rst.sh | 21 ++++++++------------- scripts/ci-container/Dockerfile | 2 +- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9392cd9..81db798 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,5 +25,5 @@ docs: - echo "$CD_SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts script: - - make docs + - make html - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<< "put -r docs/dist/html/*" diff --git a/README.md b/README.md index 4231de2..4d20c22 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ maintained in cdist itself or were not accepted in code cdist but could still be useful. This project does not have releases and is continously updated: see git history -for change log. +for change log. You will find HTML documentation at +[contrib.cdi.st](https://contrib.cdi.st). ## Using cdist-contrib diff --git a/docs/src/index.rst.sh b/docs/src/index.rst.sh index 28c5078..babc1d9 100755 --- a/docs/src/index.rst.sh +++ b/docs/src/index.rst.sh @@ -9,23 +9,18 @@ __cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" filename="${__cdist_myname%.sh}" dest="$__cdist_abs_mydir/$filename" +if ! command -v pandoc > /dev/null; then + echo "Pandoc is required to generate HTML index from README." >&2 + exit 1 +fi + cd "$__cdist_abs_mydir" exec > "$dest" + +pandoc -f markdown -t rst ../../README.md + cat << EOF -cdist-contrib - Community maintained cdist types -================================================ - -This project extends the \`cdist \`_ configuration management -tool with community-maitained types which are either too specific to fit/be -maintained in cdist itself or were not accepted in code cdist but could still -be useful. - -Please note this project is a **rolling release**! The documentation you're -reading has been generated from the |version| state (commit |release|). -Sources are available on \`code.ungleich.ch -\`_. - .. toctree:: :hidden: diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile index daa0f26..9900322 100644 --- a/scripts/ci-container/Dockerfile +++ b/scripts/ci-container/Dockerfile @@ -4,4 +4,4 @@ FROM fedora:latest MAINTAINER Timothée Floure RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ - ShellCheck openssh-clients + ShellCheck openssh-clients pandoc From 73e31e6d1ee69181f762847e6844c77e6a0b9db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:56:37 +0100 Subject: [PATCH 148/156] Mention known external cdist resources in README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 4d20c22..d9c5e22 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,11 @@ Every type in cdist-contrib must: * Have a `man.rst` documentation page. * Pass [shellcheck](http://shellcheck.net/) without errors. + +## Other resources + +Some people/organizations are known to keep some cdist types that might be of +interest to others: + +* [cdist-evilham](https://git.sr.ht/~evilham/cdist-evilham): Evilham's cdist-types +* [cdist-recycledcloud](https://code.recycled.cloud/e-Durable/cdist-recycledcloud): e-Durable SA / Recycled Cloud public types From f01f73f33d428ecc2e09bc8754d63e512862a61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:58:21 +0100 Subject: [PATCH 149/156] Remove mention to legacy mattermost room from README --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index d9c5e22..28f54db 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,11 @@ And you would run [cdist][cdist] from the same directory as follows: ## Participating in the [cdist][cdist] community -Join us on [#cdist:ungleich.ch][cdistmatrix] on matrix or on -[#cdist over mattermost][cdistmattermost]. - +Join us on [#cdist:ungleich.ch][cdistmatrix] on matrix! [cdist]: https://www.cdi.st/ [cdistconfig]: https://www.cdi.st/manual/latest/cdist-configuration.html [cdistmatrix]: https://matrix.to/#/#cdist:ungleich.ch -[cdistmattermost]: https://chat.ungleich.ch/ungleich/channels/cdist ## Contributing From f9f5c578f744cd157c138fca09ff84616cf805b6 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 11 Jan 2021 12:16:09 +0100 Subject: [PATCH 150/156] [type/__dma*] Fix shellcheck errors --- type/__dma/gencode-remote | 2 +- type/__dma/manifest | 2 +- type/__dma_auth/explorer/state | 2 +- type/__dma_auth/gencode-remote | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index fa676d4..580b22e 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -158,7 +158,7 @@ if test -f "${__object:?}/parameter/send-test-mail" then if grep -q '^__mail_alias/root:' "${__messages_in:?}" \ || grep -q '^__dma_auth/' "${__messages_in:?}" \ - || $config_updated + || ${config_updated} then cat <<-CODE sendmail root <<'EOF' diff --git a/type/__dma/manifest b/type/__dma/manifest index 2cbd1a5..530ad09 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -21,7 +21,7 @@ os=$(cat "${__global:?}/explorer/os") # Install DMA -case $os +case ${os} in (alpine) __package dma --state present diff --git a/type/__dma_auth/explorer/state b/type/__dma_auth/explorer/state index 621e5a2..c829cd4 100755 --- a/type/__dma_auth/explorer/state +++ b/type/__dma_auth/explorer/state @@ -25,7 +25,7 @@ # different_password: a line exists but with a different password # multiple: multiple lines matching host exist (should not happen) -auth_conf=$("${__type_explorer}/auth_conf") +auth_conf=$("${__type_explorer:?}/auth_conf") test -r "${auth_conf}" || exit 0 awk -F'\n' ' diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 4279b7a..b6a0100 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -40,7 +40,7 @@ then exit 0 fi -case $state_should +case ${state_should} in (present) test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1; } From 7cef989b1fb4c92e53c20768d65c5e8d4f3406eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 Jan 2021 07:14:24 +0100 Subject: [PATCH 151/156] Fix run-shellcheck.sh following 'conflict resolution' from gitlab web ui --- scripts/run-shellcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-shellcheck.sh b/scripts/run-shellcheck.sh index fcca722..d6c2db6 100755 --- a/scripts/run-shellcheck.sh +++ b/scripts/run-shellcheck.sh @@ -17,7 +17,7 @@ check() { rm -f "${SHELLCHECKTMP}" check -path '*/explorer/*' -check -path '*/files/*' ! -name '*.awk' ! -name '*.py' +check -path '*/files/*' -name '*.sh' check -name manifest check -name gencode-local check -name gencode-remote From 0e4bc443e2a1dcf703174df6c885281a0a6a64d3 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 14 Aug 2020 11:43:41 +0200 Subject: [PATCH 152/156] Add a type for wikijs. --- type/__wikijs/files/config.yml.sh | 34 ++++++++++++++++++ type/__wikijs/files/wikijs-openrc | 10 ++++++ type/__wikijs/gencode-remote | 25 +++++++++++++ type/__wikijs/man.rst | 55 ++++++++++++++++++++++++++++ type/__wikijs/manifest | 60 +++++++++++++++++++++++++++++++ type/__wikijs/parameter/boolean | 1 + type/__wikijs/parameter/optional | 3 ++ type/__wikijs/parameter/required | 1 + type/__wikijs/singleton | 0 9 files changed, 189 insertions(+) create mode 100755 type/__wikijs/files/config.yml.sh create mode 100644 type/__wikijs/files/wikijs-openrc create mode 100644 type/__wikijs/gencode-remote create mode 100644 type/__wikijs/man.rst create mode 100644 type/__wikijs/manifest create mode 100644 type/__wikijs/parameter/boolean create mode 100644 type/__wikijs/parameter/optional create mode 100644 type/__wikijs/parameter/required create mode 100644 type/__wikijs/singleton diff --git a/type/__wikijs/files/config.yml.sh b/type/__wikijs/files/config.yml.sh new file mode 100755 index 0000000..6f9943f --- /dev/null +++ b/type/__wikijs/files/config.yml.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +if [ $# -ne 1 ]; +then + echo "You have to give me the database password as an argument:" + echo "on some systems, anyone can read env(1)." + exit 1; +fi + +cat << EOF +port: 80 +db: + type: postgres + host: localhost + port: 5432 + user: ${DB_USER:?} + pass: $1 + db: ${DB_NAME:?} + ssl: false +ssl: + enabled: ${SSL} + port: 443 + provider: letsencrypt + domain: ${__target_host:?} + subscriberEmail: ${LE_EMAIL:?} +pool: + min: 2 + max: 10 +bindIP: 0.0.0.0 +logLevel: warn +offline: false +ha: false +dataPath: ./data +EOF diff --git a/type/__wikijs/files/wikijs-openrc b/type/__wikijs/files/wikijs-openrc new file mode 100644 index 0000000..e484647 --- /dev/null +++ b/type/__wikijs/files/wikijs-openrc @@ -0,0 +1,10 @@ +#!/sbin/openrc-run + +command='/usr/bin/node' +command_args='server' +command_background=true +description="Run wiki.js" +directory='/var/wiki' +error_log=/var/log/"$RC_SVCNAME".err +output_log=/var/log/"$RC_SVCNAME".log +pidfile="/run/$RC_SVCNAME.pid" diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote new file mode 100644 index 0000000..2faf559 --- /dev/null +++ b/type/__wikijs/gencode-remote @@ -0,0 +1,25 @@ +#!/bin/sh + +# Check for installation +cat << EOF +if [ -f '/var/wiki/LICENSE' ]; +then + # Assume everything is done already. + exit 0; +fi +EOF + +# Download and copy source +cat << EOF +TMPDIR=\$(mktemp -d) +cd \$TMPDIR || exit 1 +wget https://github.com/Requarks/wiki/releases/download/2.4.107/wiki-js.tar.gz +tar xf wiki-js.tar.gz -C /var/wiki +EOF + +# Install deps and launch +cat << EOF +cd /var/wiki || exit 1 +npm install +service start wikijs +EOF diff --git a/type/__wikijs/man.rst b/type/__wikijs/man.rst new file mode 100644 index 0000000..6573b60 --- /dev/null +++ b/type/__wikijs/man.rst @@ -0,0 +1,55 @@ +cdist-type__wikijs(7) +======================== + +NAME +---- +cdist-type__wikijs - Deploy the wiki.js software. + +DESCRIPTION +----------- + +See wiki.js.org for more information. This type deploys with a postgresql +database, since it is the upstream recommended for production, and they seem to +strongly suggest that in thenext releases, they will not support anything else. + +Currently, this type servers wikijs as standalone, listening on ports 80 and +443, and with a service file for OpenRC. Feel free to contribute a +generalisation if you require one. + +REQUIRED PARAMETERS +------------------- + +database-password + The password to the PSQL database. + +OPTIONAL PARAMETERS +------------------- + +database + The name of the PSQL database to connect to. If omitted, then 'wikijs' is + used. + +database-user + The name of the PSQL database user to connec as. If omitted, then 'wikijs' is + used. + +letsencrypt-mail + If the SSL parameter is passed, then we setup wikijs to automatically obtain + certificates: this is the email used to sign up to a LE account. + +BOOLEAN PARAMETERS +------------------ + +ssl + Whether or not to enable the wikijs automatic obtention of LE certificates. + +AUTHORS +------- +Joachim Desroches + +COPYING +------- +Copyright \(C) 2020 Joachim Desroches. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__wikijs/manifest b/type/__wikijs/manifest new file mode 100644 index 0000000..7ee42ac --- /dev/null +++ b/type/__wikijs/manifest @@ -0,0 +1,60 @@ +#!/bin/sh + +os="$(cat "${__global:?}"/explorer/os)" + +case "$os" in + alpine) + : + ;; + *) + echo "This type has no implementation for $os. Aborting." >&2; + exit 1; +esac + +DB_USER=wikijs +if [ -f "${__object:?}/parameter/database-user" ]; +then + DB_USER="$(cat "${__object:?}/parameter/database-user")" +fi +export DB_USER + +DB_NAME=wikijs +if [ -f "${__object:?}/parameter/database" ]; +then + DB_NAME="$(cat "${__object:?}/parameter/database")" +fi +export DB_NAME + +SSL=false +if [ -f "${__object:?}/parameter/ssl" ]; +then + SSL=true +fi +export SSL + +if ! [ "$SSL" = "false" ]; +then + if [ -f "${__object:?}/parameter/letsencrypt-mail" ]; + then + LE_EMAIL="$(cat "${__object:?}/parameter/letsencrypt-mail")" + export LE_EMAIL + else + echo "You must specify an email account if you request SSL." + echo "Hit me." + exit 1 + fi +fi + +db_pass="$(cat "${__object:?}/parameter/database-password")" + +__package nodejs +__package nghttp2-dev # Required for some reason, else a symbol is missing +__package npm +__directory /var/wiki/ +__file /etc/init.d/wikijs --source "${__files:?}/files/wikijs-openrc" +require='__file/etc/init.d/wikijs' __start_on_boot wikijs + +mkdir -p "${__object:?}/files" +"${__type:?}/files/config.yml.sh" "$db_pass" > "${__object:?}/files/config.yml" +require='__directory/var/wiki' \ + __file /var/wiki/config.yml --source "${__object:?}/files/config.yml" diff --git a/type/__wikijs/parameter/boolean b/type/__wikijs/parameter/boolean new file mode 100644 index 0000000..a2647ce --- /dev/null +++ b/type/__wikijs/parameter/boolean @@ -0,0 +1 @@ +ssl diff --git a/type/__wikijs/parameter/optional b/type/__wikijs/parameter/optional new file mode 100644 index 0000000..9c309c9 --- /dev/null +++ b/type/__wikijs/parameter/optional @@ -0,0 +1,3 @@ +database +database-user +letsencrypt-mail diff --git a/type/__wikijs/parameter/required b/type/__wikijs/parameter/required new file mode 100644 index 0000000..8a109a1 --- /dev/null +++ b/type/__wikijs/parameter/required @@ -0,0 +1 @@ +database-password diff --git a/type/__wikijs/singleton b/type/__wikijs/singleton new file mode 100644 index 0000000..e69de29 From ef748cf8e248f4f3f4eed88e433fc34ab24b0cb5 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 14 Aug 2020 13:02:17 +0200 Subject: [PATCH 153/156] Fix typo and drop use of useless tempdir. --- type/__wikijs/gencode-remote | 5 +---- type/__wikijs/man.rst | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote index 2faf559..66d7df7 100644 --- a/type/__wikijs/gencode-remote +++ b/type/__wikijs/gencode-remote @@ -11,10 +11,7 @@ EOF # Download and copy source cat << EOF -TMPDIR=\$(mktemp -d) -cd \$TMPDIR || exit 1 -wget https://github.com/Requarks/wiki/releases/download/2.4.107/wiki-js.tar.gz -tar xf wiki-js.tar.gz -C /var/wiki +wget -O - https://github.com/Requarks/wiki/releases/download/2.4.107/wiki-js.tar.gz | tar xz -C /var/wiki EOF # Install deps and launch diff --git a/type/__wikijs/man.rst b/type/__wikijs/man.rst index 6573b60..50dcd1d 100644 --- a/type/__wikijs/man.rst +++ b/type/__wikijs/man.rst @@ -10,7 +10,7 @@ DESCRIPTION See wiki.js.org for more information. This type deploys with a postgresql database, since it is the upstream recommended for production, and they seem to -strongly suggest that in thenext releases, they will not support anything else. +strongly suggest that in the next releases, they will not support anything else. Currently, this type servers wikijs as standalone, listening on ports 80 and 443, and with a service file for OpenRC. Feel free to contribute a From 647833580df813bdd105a7e0c28ca1d004ded668 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 14 Aug 2020 13:46:05 +0200 Subject: [PATCH 154/156] Split out service management from the installation type. --- type/__wikijs/gencode-remote | 1 - type/__wikijs/manifest | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote index 66d7df7..a45ac22 100644 --- a/type/__wikijs/gencode-remote +++ b/type/__wikijs/gencode-remote @@ -18,5 +18,4 @@ EOF cat << EOF cd /var/wiki || exit 1 npm install -service start wikijs EOF diff --git a/type/__wikijs/manifest b/type/__wikijs/manifest index 7ee42ac..2e3e96c 100644 --- a/type/__wikijs/manifest +++ b/type/__wikijs/manifest @@ -48,11 +48,12 @@ fi db_pass="$(cat "${__object:?}/parameter/database-password")" __package nodejs -__package nghttp2-dev # Required for some reason, else a symbol is missing __package npm __directory /var/wiki/ + +# These things are Alpine-dependant. __file /etc/init.d/wikijs --source "${__files:?}/files/wikijs-openrc" -require='__file/etc/init.d/wikijs' __start_on_boot wikijs +__package nghttp2-dev # Required for some reason, else a symbol is missing mkdir -p "${__object:?}/files" "${__type:?}/files/config.yml.sh" "$db_pass" > "${__object:?}/files/config.yml" From 7122fe1bee6977794bb5e9cc5d2eb8c4f0380067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 Jan 2021 07:57:34 +0100 Subject: [PATCH 155/156] __wikijs: add version management and HTTP(S) port configuration --- type/__wikijs/files/config.yml.sh | 26 ++++++++++++++++------ type/__wikijs/gencode-remote | 14 ++++++++++-- type/__wikijs/man.rst | 9 ++++++++ type/__wikijs/manifest | 8 +++++-- type/__wikijs/parameter/default/http-port | 1 + type/__wikijs/parameter/default/https-port | 1 + type/__wikijs/parameter/optional | 2 ++ type/__wikijs/parameter/required | 1 + 8 files changed, 51 insertions(+), 11 deletions(-) mode change 100644 => 100755 type/__wikijs/gencode-remote create mode 100644 type/__wikijs/parameter/default/http-port create mode 100644 type/__wikijs/parameter/default/https-port diff --git a/type/__wikijs/files/config.yml.sh b/type/__wikijs/files/config.yml.sh index 6f9943f..b66687a 100755 --- a/type/__wikijs/files/config.yml.sh +++ b/type/__wikijs/files/config.yml.sh @@ -7,8 +7,25 @@ then exit 1; fi +generate_ssl_section () { + + cat << EOF +ssl: + enabled: ${SSL} +EOF + +if [ "$SSL" = "true" ]; then + cat << EOF + port: $HTTPS_PORT + provider: letsencrypt + domain: ${__target_host:?} + subscriberEmail: ${LE_EMAIL:?} +EOF + fi +} + cat << EOF -port: 80 +port: $HTTP_PORT db: type: postgres host: localhost @@ -17,12 +34,7 @@ db: pass: $1 db: ${DB_NAME:?} ssl: false -ssl: - enabled: ${SSL} - port: 443 - provider: letsencrypt - domain: ${__target_host:?} - subscriberEmail: ${LE_EMAIL:?} +$(generate_ssl_section) pool: min: 2 max: 10 diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote old mode 100644 new mode 100755 index a45ac22..81055e2 --- a/type/__wikijs/gencode-remote +++ b/type/__wikijs/gencode-remote @@ -1,17 +1,22 @@ #!/bin/sh +VERSION_FILE=/var/wiki/version +version=$(cat "${__object:?}/parameter/version") + # Check for installation cat << EOF -if [ -f '/var/wiki/LICENSE' ]; +if [ -f $VERSION_FILE ] && [ "\$(cat $VERSION_FILE)" = "$version" ]; then # Assume everything is done already. exit 0; +else + echo "$version" > $VERSION_FILE fi EOF # Download and copy source cat << EOF -wget -O - https://github.com/Requarks/wiki/releases/download/2.4.107/wiki-js.tar.gz | tar xz -C /var/wiki +wget -O - https://github.com/Requarks/wiki/releases/download/$version/wiki-js.tar.gz | tar xz -C /var/wiki EOF # Install deps and launch @@ -19,3 +24,8 @@ cat << EOF cd /var/wiki || exit 1 npm install EOF + +# Restart service. +cat << EOF +service wikijs restart +EOF diff --git a/type/__wikijs/man.rst b/type/__wikijs/man.rst index 50dcd1d..b259c90 100644 --- a/type/__wikijs/man.rst +++ b/type/__wikijs/man.rst @@ -22,6 +22,9 @@ REQUIRED PARAMETERS database-password The password to the PSQL database. +version + 'wikijs' version to be deployed. + OPTIONAL PARAMETERS ------------------- @@ -37,6 +40,12 @@ letsencrypt-mail If the SSL parameter is passed, then we setup wikijs to automatically obtain certificates: this is the email used to sign up to a LE account. +http-port + Specify HTTP port, defaults to 80. + +http-port + Specify HTTPS port, defaults to 443. Only relevant if the SSL flag is enabled. + BOOLEAN PARAMETERS ------------------ diff --git a/type/__wikijs/manifest b/type/__wikijs/manifest index 2e3e96c..04a21af 100644 --- a/type/__wikijs/manifest +++ b/type/__wikijs/manifest @@ -32,7 +32,7 @@ then fi export SSL -if ! [ "$SSL" = "false" ]; +if [ "$SSL" = "true" ]; then if [ -f "${__object:?}/parameter/letsencrypt-mail" ]; then @@ -45,6 +45,10 @@ then fi fi +HTTP_PORT=$(cat "${__object:?}/parameter/http-port") +HTTPS_PORT=$(cat "${__object:?}/parameter/https-port") +export HTTP_PORT HTTPS_PORT + db_pass="$(cat "${__object:?}/parameter/database-password")" __package nodejs @@ -52,7 +56,7 @@ __package npm __directory /var/wiki/ # These things are Alpine-dependant. -__file /etc/init.d/wikijs --source "${__files:?}/files/wikijs-openrc" +__file /etc/init.d/wikijs --source "${__type:?}/files/wikijs-openrc" __package nghttp2-dev # Required for some reason, else a symbol is missing mkdir -p "${__object:?}/files" diff --git a/type/__wikijs/parameter/default/http-port b/type/__wikijs/parameter/default/http-port new file mode 100644 index 0000000..d15a2cc --- /dev/null +++ b/type/__wikijs/parameter/default/http-port @@ -0,0 +1 @@ +80 diff --git a/type/__wikijs/parameter/default/https-port b/type/__wikijs/parameter/default/https-port new file mode 100644 index 0000000..6a13cf6 --- /dev/null +++ b/type/__wikijs/parameter/default/https-port @@ -0,0 +1 @@ +443 diff --git a/type/__wikijs/parameter/optional b/type/__wikijs/parameter/optional index 9c309c9..be19c92 100644 --- a/type/__wikijs/parameter/optional +++ b/type/__wikijs/parameter/optional @@ -1,3 +1,5 @@ database database-user letsencrypt-mail +http-port +https-port diff --git a/type/__wikijs/parameter/required b/type/__wikijs/parameter/required index 8a109a1..ae542bc 100644 --- a/type/__wikijs/parameter/required +++ b/type/__wikijs/parameter/required @@ -1 +1,2 @@ database-password +version From 8929c566fcc0493ee50ca18974ea5488aa2f196b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 Jan 2021 08:33:50 +0100 Subject: [PATCH 156/156] __wikijs: remove uneeded npm install call See https://github.com/Requarks/wiki/issues/1325. --- type/__wikijs/gencode-remote | 5 ----- type/__wikijs/manifest | 1 - 2 files changed, 6 deletions(-) diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote index 81055e2..37c7df7 100755 --- a/type/__wikijs/gencode-remote +++ b/type/__wikijs/gencode-remote @@ -22,10 +22,5 @@ EOF # Install deps and launch cat << EOF cd /var/wiki || exit 1 -npm install -EOF - -# Restart service. -cat << EOF service wikijs restart EOF diff --git a/type/__wikijs/manifest b/type/__wikijs/manifest index 04a21af..b047223 100644 --- a/type/__wikijs/manifest +++ b/type/__wikijs/manifest @@ -52,7 +52,6 @@ export HTTP_PORT HTTPS_PORT db_pass="$(cat "${__object:?}/parameter/database-password")" __package nodejs -__package npm __directory /var/wiki/ # These things are Alpine-dependant.