From 05a6b8aa42e0c3d34dd9f4512a8e631c7c28f05b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 15 May 2020 07:57:16 +0200 Subject: [PATCH 001/332] Import __matterbridge type from ungleich's dot-cdist --- .../files/matterbridge.service.sh | 18 ++++ type/__matterbridge/man.rst | 56 ++++++++++ type/__matterbridge/manifest | 101 ++++++++++++++++++ type/__matterbridge/parameter/required | 2 + type/__matterbridge/singleton | 0 5 files changed, 177 insertions(+) create mode 100755 type/__matterbridge/files/matterbridge.service.sh create mode 100644 type/__matterbridge/man.rst create mode 100755 type/__matterbridge/manifest create mode 100644 type/__matterbridge/parameter/required create mode 100644 type/__matterbridge/singleton diff --git a/type/__matterbridge/files/matterbridge.service.sh b/type/__matterbridge/files/matterbridge.service.sh new file mode 100755 index 0000000..9dbd1cb --- /dev/null +++ b/type/__matterbridge/files/matterbridge.service.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +cat <`_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__matterbridge/manifest b/type/__matterbridge/manifest new file mode 100755 index 0000000..56f470a --- /dev/null +++ b/type/__matterbridge/manifest @@ -0,0 +1,101 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$(cat "$__global/explorer/os") +case "$os" in + debian) + # This type assume systemd for service installation. + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +# Required parameters. +VERSION=$(cat "$__object/parameter/version") +if [ -f "$__object/parameter/config" ]; then + CONFIG="$(cat "$__object/parameter/config")" + if [ "$CONFIG" = "-" ]; then + CONFIG=$(cat "$__object/stdin") + fi +fi + +# Hardcoded values used in templates. +export BINARY_PATH=/usr/local/bin/matterbridge +export CONFIG_PATH=/etc/matterbridge/matterbridge.toml +export USER=matterbridge +export GROUP=$USER + +# Internal variables. +artefact="matterbridge-$VERSION-linux-64bit" +checksum_file="checksums.txt" +release_download_url=https://github.com/42wim/matterbridge/releases/download +binary_url="$release_download_url/v$VERSION/$artefact" +checksum_file_url="$release_download_url/v$VERSION/$checksum_file" +config_dir=$(dirname $CONFIG_PATH) +systemd_unit_path='/etc/systemd/system/matterbridge.service' + +# Check if curl is available. +if [ ! -x "$(which curl)" ]; then + echo "curl is required for this type, but could not be found. Exiting." &>2 + exit 1 +fi + +# Initialize working directory. +mkdir -p "$__object/files" + +# Download and check matterbridge binary. +curl -L $binary_url -o "$__object/files/$artefact" +curl -Ls $checksum_file_url | grep $artefact > "$__object/files/$checksum_file" +ls $__object/files/ >&2 +cat $__object/files/checksums.txt >&2 +(cd "$__object/files"; sha256sum --check $checksum_file) +if [ $? -ne 0 ]; then + echo "Matterbridge binary checksum failed." >&2 + exit 1 +fi + +# Create service user. +__user $USER --home "/var/lib/$USER" + +# Deploy matterbridge binary. +require="__user/$USER" __file "$BINARY_PATH" \ + --source "$__object/files/$artefact" \ + --owner "$USER" --mode 755 + +# Generate and deploy configuration file. +"$__type/files/matterbridge.service.sh" > "$__object/files/matterbridge.service" + +require="__user/$USER" __directory "$config_dir" \ + --owner "$USER" --mode 0755 --parents \ + +require="__directory/$config_dir" __file "$CONFIG_PATH" \ + --owner "$USER" \ + --mode 0640 \ + --source "$CONFIG" + +__file "$systemd_unit_path" \ + --source "$__object/files/matterbridge.service" + +# Deal with init system. +require="__file/$systemd_unit_path" __start_on_boot matterbridge +require="__file/$BINARY_PATH __file/$CONFIG_PATH __file/$systemd_unit_path" __service matterbridge --action restart diff --git a/type/__matterbridge/parameter/required b/type/__matterbridge/parameter/required new file mode 100644 index 0000000..ed5d8b3 --- /dev/null +++ b/type/__matterbridge/parameter/required @@ -0,0 +1,2 @@ +version +config diff --git a/type/__matterbridge/singleton b/type/__matterbridge/singleton new file mode 100644 index 0000000..e69de29 From 43562ff09c7ab44f42e37893df5dc48af1b1e15e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 15 May 2020 08:08:45 +0200 Subject: [PATCH 002/332] Make __matterbridge shellcheck-friendly --- type/__matterbridge/manifest | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/type/__matterbridge/manifest b/type/__matterbridge/manifest index 56f470a..ef02112 100755 --- a/type/__matterbridge/manifest +++ b/type/__matterbridge/manifest @@ -55,8 +55,8 @@ config_dir=$(dirname $CONFIG_PATH) systemd_unit_path='/etc/systemd/system/matterbridge.service' # Check if curl is available. -if [ ! -x "$(which curl)" ]; then - echo "curl is required for this type, but could not be found. Exiting." &>2 +if ! command -v curl; then + echo "curl is required for this type, but could not be found. Exiting." >&2 exit 1 fi @@ -64,12 +64,9 @@ fi mkdir -p "$__object/files" # Download and check matterbridge binary. -curl -L $binary_url -o "$__object/files/$artefact" -curl -Ls $checksum_file_url | grep $artefact > "$__object/files/$checksum_file" -ls $__object/files/ >&2 -cat $__object/files/checksums.txt >&2 -(cd "$__object/files"; sha256sum --check $checksum_file) -if [ $? -ne 0 ]; then +curl -L "$binary_url" -o "$__object/files/$artefact" +curl -Ls "$checksum_file_url" | grep "$artefact" > "$__object/files/$checksum_file" +if ! (cd "$__object/files"; sha256sum --check $checksum_file); then echo "Matterbridge binary checksum failed." >&2 exit 1 fi From f4671691bee30b89054a60523f9177ac15fa8790 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 29 May 2020 17:33:23 +0200 Subject: [PATCH 003/332] Rename __root_mail_dma to __dma to prepare for new types --- type/{__root_mail_dma => __dma}/files/aliases | 0 type/{__root_mail_dma => __dma}/gencode-remote | 0 type/{__root_mail_dma => __dma}/man.rst | 6 +++--- type/{__root_mail_dma => __dma}/manifest | 0 type/{__root_mail_dma => __dma}/parameter/boolean | 0 type/{__root_mail_dma => __dma}/parameter/optional | 0 type/{__root_mail_dma => __dma}/parameter/required | 0 type/{__root_mail_dma => __dma}/parameter/required_multiple | 0 type/{__root_mail_dma => __dma}/singleton | 0 9 files changed, 3 insertions(+), 3 deletions(-) rename type/{__root_mail_dma => __dma}/files/aliases (100%) rename type/{__root_mail_dma => __dma}/gencode-remote (100%) rename type/{__root_mail_dma => __dma}/man.rst (94%) rename type/{__root_mail_dma => __dma}/manifest (100%) rename type/{__root_mail_dma => __dma}/parameter/boolean (100%) rename type/{__root_mail_dma => __dma}/parameter/optional (100%) rename type/{__root_mail_dma => __dma}/parameter/required (100%) rename type/{__root_mail_dma => __dma}/parameter/required_multiple (100%) rename type/{__root_mail_dma => __dma}/singleton (100%) diff --git a/type/__root_mail_dma/files/aliases b/type/__dma/files/aliases similarity index 100% rename from type/__root_mail_dma/files/aliases rename to type/__dma/files/aliases diff --git a/type/__root_mail_dma/gencode-remote b/type/__dma/gencode-remote similarity index 100% rename from type/__root_mail_dma/gencode-remote rename to type/__dma/gencode-remote diff --git a/type/__root_mail_dma/man.rst b/type/__dma/man.rst similarity index 94% rename from type/__root_mail_dma/man.rst rename to type/__dma/man.rst index ecf2885..a10c6c2 100644 --- a/type/__root_mail_dma/man.rst +++ b/type/__dma/man.rst @@ -1,9 +1,9 @@ -cdist-type__root_mail_dma(7) +cdist-type__dma(7) ============================ NAME ---- -cdist-type__root_mail_dma - Setup root email with the DragonFly Mail Agent +cdist-type__dma - Setup the DragonFly Mail Agent as the MTA. DESCRIPTION @@ -57,7 +57,7 @@ EXAMPLES # Send root email to both our BOFH and the nice-admin. # That way they can figure things out together. - __root_mail_dma \ + __dma \ --root-email bofh@domain.tld \ --root-email nice-admin@domain.tld \ --smart-host mx1.domain.tld \ diff --git a/type/__root_mail_dma/manifest b/type/__dma/manifest similarity index 100% rename from type/__root_mail_dma/manifest rename to type/__dma/manifest diff --git a/type/__root_mail_dma/parameter/boolean b/type/__dma/parameter/boolean similarity index 100% rename from type/__root_mail_dma/parameter/boolean rename to type/__dma/parameter/boolean diff --git a/type/__root_mail_dma/parameter/optional b/type/__dma/parameter/optional similarity index 100% rename from type/__root_mail_dma/parameter/optional rename to type/__dma/parameter/optional diff --git a/type/__root_mail_dma/parameter/required b/type/__dma/parameter/required similarity index 100% rename from type/__root_mail_dma/parameter/required rename to type/__dma/parameter/required diff --git a/type/__root_mail_dma/parameter/required_multiple b/type/__dma/parameter/required_multiple similarity index 100% rename from type/__root_mail_dma/parameter/required_multiple rename to type/__dma/parameter/required_multiple diff --git a/type/__root_mail_dma/singleton b/type/__dma/singleton similarity index 100% rename from type/__root_mail_dma/singleton rename to type/__dma/singleton From a491e8739efb6753d52ede2ee58ba5a93247d1b4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 29 May 2020 17:33:40 +0200 Subject: [PATCH 004/332] Add __dma_auth type --- type/__dma_auth/gencode-remote | 20 +++++++++++++ type/__dma_auth/man.rst | 53 ++++++++++++++++++++++++++++++++++ type/__dma_auth/manifest | 30 +++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100755 type/__dma_auth/gencode-remote create mode 100644 type/__dma_auth/man.rst create mode 100755 type/__dma_auth/manifest diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote new file mode 100755 index 0000000..77ad9d2 --- /dev/null +++ b/type/__dma_auth/gencode-remote @@ -0,0 +1,20 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + diff --git a/type/__dma_auth/man.rst b/type/__dma_auth/man.rst new file mode 100644 index 0000000..cd4f647 --- /dev/null +++ b/type/__dma_auth/man.rst @@ -0,0 +1,53 @@ +cdist-type__dma_auth(7) +======================= + +NAME +---- +cdist-type__dma_auth - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __dma_auth + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Dennis Camera + + +COPYING +------- +Copyright \(C) 2020 Dennis Camera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__dma_auth/manifest b/type/__dma_auth/manifest new file mode 100755 index 0000000..e0c809a --- /dev/null +++ b/type/__dma_auth/manifest @@ -0,0 +1,30 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac From 98496aa8e5c8463b59263f20daa162bed9b63dfa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 29 May 2020 17:33:52 +0200 Subject: [PATCH 005/332] Add __mail_alias type --- type/__mail_alias/gencode-remote | 20 ++++++++++++ type/__mail_alias/man.rst | 53 ++++++++++++++++++++++++++++++++ type/__mail_alias/manifest | 30 ++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100755 type/__mail_alias/gencode-remote create mode 100644 type/__mail_alias/man.rst create mode 100755 type/__mail_alias/manifest diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote new file mode 100755 index 0000000..77ad9d2 --- /dev/null +++ b/type/__mail_alias/gencode-remote @@ -0,0 +1,20 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst new file mode 100644 index 0000000..d9563a6 --- /dev/null +++ b/type/__mail_alias/man.rst @@ -0,0 +1,53 @@ +cdist-type__mail_alias(7) +========================= + +NAME +---- +cdist-type__mail_alias - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __mail_alias + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Dennis Camera + + +COPYING +------- +Copyright \(C) 2020 Dennis Camera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__mail_alias/manifest b/type/__mail_alias/manifest new file mode 100755 index 0000000..e0c809a --- /dev/null +++ b/type/__mail_alias/manifest @@ -0,0 +1,30 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac From 3adc4f160998776033934d65ba22f1666c9e0b4b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 30 May 2020 17:24:26 +0200 Subject: [PATCH 006/332] [type/__mail_alias] Implement type --- type/__mail_alias/explorer/aliases | 59 ++++++++++ .../{manifest => explorer/aliases_file} | 38 ++++-- type/__mail_alias/gencode-remote | 108 ++++++++++++++++++ type/__mail_alias/man.rst | 21 +++- type/__mail_alias/parameter/default/state | 1 + type/__mail_alias/parameter/optional | 1 + type/__mail_alias/parameter/optional_multiple | 1 + 7 files changed, 215 insertions(+), 14 deletions(-) create mode 100755 type/__mail_alias/explorer/aliases rename type/__mail_alias/{manifest => explorer/aliases_file} (56%) mode change 100755 => 100644 create mode 100644 type/__mail_alias/parameter/default/state create mode 100644 type/__mail_alias/parameter/optional create mode 100644 type/__mail_alias/parameter/optional_multiple diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases new file mode 100755 index 0000000..ce1a439 --- /dev/null +++ b/type/__mail_alias/explorer/aliases @@ -0,0 +1,59 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Find aliases for a given name and print the aliases line separated + +aliases_file=$("${__type_explorer}/aliases_file") +test -r "${aliases_file}" || exit 0 + +name=$__object_id + +awk -F ':[[:blank:]]*' ' +function print_aliases (aliases, matches) { + split(aliases, matches, /,[[:blank:]]*/) + for (i in matches) { + gsub(/^[[:blank:]]*|[[:blank:]]*$/, "", matches[i]) + print matches[i] + } +} + +/^#/ { + # comment + select = 0; cont = 0; next +} + +{ + cont = ($0 ~ /\\$/) + if (cont) sub(/[[:blank:]]*\\$/, "", $0) +} + +/^[[:blank:]]/ || cont { + # continuation line + if (select) print_aliases($0) + next +} + +$1 == ENVIRON["__object_id"] { + select = 1 + print_aliases($2) + next +} + +{ select = 0 } +' "${aliases_file}" diff --git a/type/__mail_alias/manifest b/type/__mail_alias/explorer/aliases_file old mode 100755 new mode 100644 similarity index 56% rename from type/__mail_alias/manifest rename to type/__mail_alias/explorer/aliases_file index e0c809a..f7c4596 --- a/type/__mail_alias/manifest +++ b/type/__mail_alias/explorer/aliases_file @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,14 +17,36 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer tries to find the correct aliases file. +found() { echo "$*"; exit 0; } -os=$(cat "$__global/explorer/os") +check_file() { + if test -f "$1" + then + found "$1" + fi +} -case "$os" in - *) - printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; +case $("$__explorer/os") +in + (freebsd|openbsd|solaris) + check_file /etc/mail/aliases + + # default + found /etc/mail/aliases + ;; + (debian|devuan|ubuntu) + check_file /etc/aliases + + # default + found /etc/aliases + ;; + (*) + check_file /etc/mail/aliases + check_file /etc/aliases + + # default + found /etc/mail/aliases + ;; esac diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 77ad9d2..a93dff2 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -18,3 +18,111 @@ # along with cdist. If not, see . # +state_should=$(cat "${__object}/parameter/state") + +case $state_should +in + (present) + if cmp "${__object}/explorer/aliases" "${__object}/parameter/alias" + then + # all good! + exit 0 + fi + + echo "set aliases" >>"$__messages_out" + mode=1 + ;; + (absent) + # nothing to do if no aliases found. + test -s "${__object}/explorer/aliases" || exit 0 + + echo "delete aliases" >>"$__messages_out" + mode=0 + ;; + (*) + printf 'Invalid --state given: %s\n' "$state_should" >&2 + exit 1 +esac + +aliases_file=$(cat "${__object}/explorer/aliases_file") + +if test -z "${aliases_file}" +then + echo 'Could not determine aliases file path.' >&2 + exit 1 +fi + +# "export" variables to remote +printf 'mode=%u\n' "${mode}" +printf "aliases_file='%s'\n" "${aliases_file}" + +cat <<'EOF' +awk -F ':[[:blank:]]*' -v mode="${mode}" ' +function sepafter(f, default, _) { + _ = substr($0, length($f) + 1, index(substr($0, length($f)+1), $(f+1)) - 1) + if (_) return _ + else return default +} + +function write_aliases() { + if (aliases_written) return + + printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") + while ((getline < aliases_should_file) > 0) { + if (aliases_written) printf ", " + printf "%s", $0 + aliases_written = 1 + } + printf "\n" + close(aliases_should_file) +} + +BEGIN { + aliases_should_file = (ENVIRON["__object"] "/parameter/alias") +} + +/^#/ { + # comment + select = 0; cont = 0 + print + next +} + +{ + cont = ($0 ~ /\\$/) + if (cont) sub(/[[:blank:]]*\\$/, "", $0) +} + +/^[[:blank:]]/ || cont { + # continuation line + if (select) next +} + +$1 == ENVIRON["__object_id"] { + in_list = 1 + if (mode) write_aliases() + next +} + +{ + in_list = 0 + print +} + +END { + # if the last line as an alias definition, the separator will be reused + if (mode && !aliases_written) write_aliases() +} +' <"${aliases_file}" >"${aliases_file}.tmp" || { + echo 'Generating new aliases file failed!' >&2 + exit 1 +} + +if ! cmp "${aliases_file}" "${aliases_file}.tmp" +then + mv "${aliases_file}.tmp" "${aliases_file}" + newaliases +else + rm "${aliases_file}.tmp" +fi +EOF diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst index d9563a6..d6c7873 100644 --- a/type/__mail_alias/man.rst +++ b/type/__mail_alias/man.rst @@ -3,12 +3,12 @@ cdist-type__mail_alias(7) NAME ---- -cdist-type__mail_alias - TODO +cdist-type__mail_alias - Manage mail aliases. DESCRIPTION ----------- -This space intentionally left blank. +This cdist type allows you to configure mail aliases (/etc/mail/aliases). REQUIRED PARAMETERS @@ -18,7 +18,14 @@ None. OPTIONAL PARAMETERS ------------------- -None. +state + 'present' or 'absent', defaults to 'present' +alias + the aliases where mail for the given user should be redirected to. + This parameter can be specified multiple times to redirect to more than one + recipient. + See the `aliases(5)` man page for the different forms this parameter can + take.. BOOLEAN PARAMETERS @@ -31,13 +38,15 @@ EXAMPLES .. code-block:: sh - # TODO - __mail_alias + # Redirect root mail to a "real" email address + __mail_alias root --alias admin@example.com + # Disable redirection of mail for joe + __mail_alias joe --state absent SEE ALSO -------- -:strong:`TODO`\ (7) +:strong:`aliases`\ (5) AUTHORS diff --git a/type/__mail_alias/parameter/default/state b/type/__mail_alias/parameter/default/state new file mode 100644 index 0000000..e7f6134 --- /dev/null +++ b/type/__mail_alias/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/type/__mail_alias/parameter/optional b/type/__mail_alias/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__mail_alias/parameter/optional @@ -0,0 +1 @@ +state diff --git a/type/__mail_alias/parameter/optional_multiple b/type/__mail_alias/parameter/optional_multiple new file mode 100644 index 0000000..d077ed8 --- /dev/null +++ b/type/__mail_alias/parameter/optional_multiple @@ -0,0 +1 @@ +alias From a5f3f3cdafe77f5aae075d1a0c053a9a36475463 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 30 May 2020 18:27:13 +0200 Subject: [PATCH 007/332] [type/__dma_auth] Implement type --- type/__dma_auth/{manifest => explorer/logins} | 29 ++++++--- type/__dma_auth/gencode-remote | 62 +++++++++++++++++++ type/__dma_auth/man.rst | 27 +++++--- type/__dma_auth/parameter/default/state | 1 + type/__dma_auth/parameter/optional | 2 + type/__dma_auth/parameter/required | 2 + 6 files changed, 106 insertions(+), 17 deletions(-) rename type/__dma_auth/{manifest => explorer/logins} (55%) mode change 100755 => 100644 create mode 100644 type/__dma_auth/parameter/default/state create mode 100644 type/__dma_auth/parameter/optional create mode 100644 type/__dma_auth/parameter/required diff --git a/type/__dma_auth/manifest b/type/__dma_auth/explorer/logins old mode 100755 new mode 100644 similarity index 55% rename from type/__dma_auth/manifest rename to type/__dma_auth/explorer/logins index e0c809a..0ed6bc1 --- a/type/__dma_auth/manifest +++ b/type/__dma_auth/explorer/logins @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,14 +17,25 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer lines matching user + server in /etc/dma/auth.conf and reports +# their cksum. +test -r /etc/dma/auth.conf || exit 0 -os=$(cat "$__global/explorer/os") +if test -f "${__object}/parameter/login" +then + login=$(cat "${__object}/parameter/login") +else + login=$__object_id +fi +server=$(cat "${__object}/parameter/server") -case "$os" in - *) - printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; -esac +regex=$(printf '^%s|%s:' "$login" "$server") + +grep -e "${regex}" /etc/dma/auth.conf \ +| while read -r line + do + echo "${line}" \ + | cksum - \ + | cut -d ' ' -f 1 + done diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 77ad9d2..0951c16 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,3 +18,65 @@ # along with cdist. If not, see . # +logins=$(cat "${__object}/explorer/logins") +state_should=$(cat "${__object}/parameter/state") + +if test -f "${__object}/parameter/login" +then + login=$(cat "${__object}/parameter/login") +else + login=$__object_id +fi +password=$(cat "${__object}/parameter/password") +server=$(cat "${__object}/parameter/server") + +case $state_should +in + (present) + conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "${password}") + cksum_should=$(echo "${conf_line}" | cksum - | cut -d ' ' -f 1) + if echo "$logins" | grep -qxF "${cksum_should}" + then + # correct line already present -> nothing to do + exit 0 + fi + + mode=1 + ;; + (absent) + if test -z "$logins" + then + # no logins present -> nothing to do + exit 0 + fi + + # NOTE: password is not needed to delete + conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "") + + mode=0 + ;; + (*) + printf 'Invalid --state: %s' "${state_should}" >&2 + exit 1 + ;; +esac + +cat </etc/dma/auth.conf.tmp \ +&& mv /etc/dma/auth.conf.tmp /etc/dma/auth.conf +EOF diff --git a/type/__dma_auth/man.rst b/type/__dma_auth/man.rst index cd4f647..bd077d6 100644 --- a/type/__dma_auth/man.rst +++ b/type/__dma_auth/man.rst @@ -3,23 +3,29 @@ cdist-type__dma_auth(7) NAME ---- -cdist-type__dma_auth - TODO +cdist-type__dma_auth - Configure SMTP logins for the DragonFly Mail Agent MTA. DESCRIPTION ----------- -This space intentionally left blank. +This cdist type allows you to set up credentials to log in to remote SMTP +servers. REQUIRED PARAMETERS ------------------- -None. +password + The user's password (in plain text.) +server + The SMTP server on which the login is valid. OPTIONAL PARAMETERS ------------------- -None. - +login + The user's LOGIN name on the SMTP server. Defaults to `__object_id`. +state + Either `present` or `absent`. Defaults to `present`. BOOLEAN PARAMETERS ------------------ @@ -31,13 +37,18 @@ EXAMPLES .. code-block:: sh - # TODO - __dma_auth + # Set the password for smarthost + __dma_auth joe --server smarthost --password hunter2 + # Set credentials for user at an external provider + __dma_auth paul@example.com --server mail.provider.com --password letmein + + # Delete credentials for example.com + __dma_auth paul --server example.com --state absent SEE ALSO -------- -:strong:`TODO`\ (7) +:strong:`cdist-type__dma`\ (7), :strong:`dma`\ (8) AUTHORS diff --git a/type/__dma_auth/parameter/default/state b/type/__dma_auth/parameter/default/state new file mode 100644 index 0000000..e7f6134 --- /dev/null +++ b/type/__dma_auth/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/type/__dma_auth/parameter/optional b/type/__dma_auth/parameter/optional new file mode 100644 index 0000000..c35dbef --- /dev/null +++ b/type/__dma_auth/parameter/optional @@ -0,0 +1,2 @@ +login +state diff --git a/type/__dma_auth/parameter/required b/type/__dma_auth/parameter/required new file mode 100644 index 0000000..8f1a1c9 --- /dev/null +++ b/type/__dma_auth/parameter/required @@ -0,0 +1,2 @@ +password +server From 988f277ad63c8c3b465b5e31571e470009c6ee9a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 30 May 2020 22:07:20 +0200 Subject: [PATCH 008/332] [type/__mail_alias] Fixes (mostly mawk compatibility) --- type/__mail_alias/explorer/aliases | 12 +++++------- type/__mail_alias/explorer/aliases_file | 0 type/__mail_alias/gencode-remote | 11 ++++++----- 3 files changed, 11 insertions(+), 12 deletions(-) mode change 100644 => 100755 type/__mail_alias/explorer/aliases_file diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index ce1a439..66940d5 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -22,13 +22,11 @@ aliases_file=$("${__type_explorer}/aliases_file") test -r "${aliases_file}" || exit 0 -name=$__object_id - -awk -F ':[[:blank:]]*' ' -function print_aliases (aliases, matches) { - split(aliases, matches, /,[[:blank:]]*/) +awk -F ':[ \t]*' ' +function print_aliases(aliases, matches) { + split(aliases, matches, /,[ \t]*/) for (i in matches) { - gsub(/^[[:blank:]]*|[[:blank:]]*$/, "", matches[i]) + gsub(/^[ \t]*|[ \t]*$/, "", matches[i]) print matches[i] } } @@ -40,7 +38,7 @@ function print_aliases (aliases, matches) { { cont = ($0 ~ /\\$/) - if (cont) sub(/[[:blank:]]*\\$/, "", $0) + if (cont) sub(/[ \t]*\\$/, "", $0) } /^[[:blank:]]/ || cont { diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file old mode 100644 new mode 100755 diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index a93dff2..7778536 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -23,7 +23,7 @@ state_should=$(cat "${__object}/parameter/state") case $state_should in (present) - if cmp "${__object}/explorer/aliases" "${__object}/parameter/alias" + if cmp -s "${__object}/explorer/aliases" "${__object}/parameter/alias" then # all good! exit 0 @@ -57,7 +57,8 @@ printf 'mode=%u\n' "${mode}" printf "aliases_file='%s'\n" "${aliases_file}" cat <<'EOF' -awk -F ':[[:blank:]]*' -v mode="${mode}" ' +test -f "${aliases_file}" || touch "${aliases_file}" +awk -F ':[ \t]*' -v mode="${mode}" ' function sepafter(f, default, _) { _ = substr($0, length($f) + 1, index(substr($0, length($f)+1), $(f+1)) - 1) if (_) return _ @@ -90,10 +91,10 @@ BEGIN { { cont = ($0 ~ /\\$/) - if (cont) sub(/[[:blank:]]*\\$/, "", $0) + if (cont) sub(/[ \t]*\\$/, "", $0) } -/^[[:blank:]]/ || cont { +/^[ \t]/ || cont { # continuation line if (select) next } @@ -118,7 +119,7 @@ END { exit 1 } -if ! cmp "${aliases_file}" "${aliases_file}.tmp" +if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" then mv "${aliases_file}.tmp" "${aliases_file}" newaliases From 59059a200a27cd750794439544e3aa5a45b65f09 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 31 May 2020 11:57:54 +0200 Subject: [PATCH 009/332] [type/__dma_auth] Use host as key --- .../__dma_auth/explorer/{logins => auth_conf} | 48 +++++++++-------- type/__dma_auth/explorer/authusers | 51 +++++++++++++++++++ type/__dma_auth/gencode-remote | 26 +++++----- type/__dma_auth/man.rst | 16 +++--- type/__dma_auth/parameter/optional | 2 +- type/__dma_auth/parameter/required | 2 +- 6 files changed, 102 insertions(+), 43 deletions(-) rename type/__dma_auth/explorer/{logins => auth_conf} (54%) mode change 100644 => 100755 create mode 100755 type/__dma_auth/explorer/authusers diff --git a/type/__dma_auth/explorer/logins b/type/__dma_auth/explorer/auth_conf old mode 100644 new mode 100755 similarity index 54% rename from type/__dma_auth/explorer/logins rename to type/__dma_auth/explorer/auth_conf index 0ed6bc1..cef0aca --- a/type/__dma_auth/explorer/logins +++ b/type/__dma_auth/explorer/auth_conf @@ -17,25 +17,33 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer lines matching user + server in /etc/dma/auth.conf and reports -# their cksum. +# This explorer determines the path of dma's auth.conf file -test -r /etc/dma/auth.conf || exit 0 +# No dma.conf -> use default +test -f /etc/dma/dma.conf || { + echo /etc/dma/auth.conf + exit 0 +} +test -r /etc/dma/dma.conf || { + echo 'Cannot read /etc/dma/dma.conf' >&2 + exit 1 +} -if test -f "${__object}/parameter/login" -then - login=$(cat "${__object}/parameter/login") -else - login=$__object_id -fi -server=$(cat "${__object}/parameter/server") - -regex=$(printf '^%s|%s:' "$login" "$server") - -grep -e "${regex}" /etc/dma/auth.conf \ -| while read -r line - do - echo "${line}" \ - | cksum - \ - | cut -d ' ' -f 1 - done +# Get AUTHPATH from dma.conf +awk -F'[ \t]' ' +{ + sub(/#.*$/, "", $0) # remove comments + if (!$0) next # ignore empty lines +} +$1 == "AUTHPATH" { + # Store authpath. In dma conf parsing last wins. + if ($2) authpath = substr($0, index($0, " ") + 1) +} +END { + if (authpath) { + print authpath + exit 0 + } else exit 1 +} +' /etc/dma/dma.conf \ +|| echo /etc/dma/auth.conf # default diff --git a/type/__dma_auth/explorer/authusers b/type/__dma_auth/explorer/authusers new file mode 100755 index 0000000..5fc6b4e --- /dev/null +++ b/type/__dma_auth/explorer/authusers @@ -0,0 +1,51 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# This explorer looks for lines matching the server parameter in dma's auth.conf +# and reports the login and server fields (password is stripped off) + +auth_conf=$("${__type_explorer}/auth_conf") +test -r "${auth_conf}" || exit 0 + +if test -f "${__object}/parameter/server" +then + server=$(cat "${__object}/parameter/server") +else + server=$__object_id +fi + +awk -F'\n' -v server="${server}" ' +BEGIN { + DP = "[: \t]" # copied from dma/conf.c +} + +# skip comments and empty lines +/^#/ || /^$/ { next } + +{ + login = substr($0, 1, index($0, "|") - 1) + host = substr($0, length(login) + 2) + if (match(host, DP)) { + host = substr(host, 1, RSTART - 1) + endpos = length(login) + RSTART + } else endpos = length +} + +host == server { print substr($0, 0, endpos) } +' "${auth_conf}" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 0951c16..989e176 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,24 +18,24 @@ # along with cdist. If not, see . # -logins=$(cat "${__object}/explorer/logins") +authusers=$(cat "${__object}/explorer/authusers") state_should=$(cat "${__object}/parameter/state") -if test -f "${__object}/parameter/login" +if test -f "${__object}/parameter/server" then - login=$(cat "${__object}/parameter/login") + server=$(cat "${__object}/parameter/server") else - login=$__object_id + server=$__object_id fi +login=$(cat "${__object}/parameter/login") password=$(cat "${__object}/parameter/password") -server=$(cat "${__object}/parameter/server") case $state_should in (present) conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "${password}") cksum_should=$(echo "${conf_line}" | cksum - | cut -d ' ' -f 1) - if echo "$logins" | grep -qxF "${cksum_should}" + if echo "$authusers" | grep -qxF "${cksum_should}" then # correct line already present -> nothing to do exit 0 @@ -44,11 +44,8 @@ in mode=1 ;; (absent) - if test -z "$logins" - then - # no logins present -> nothing to do - exit 0 - fi + # no logins present -> nothing to do + test -n "$authusers" || exit 0 # NOTE: password is not needed to delete conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "") @@ -66,11 +63,14 @@ read -r CONF_LINE <<'EOL' ${conf_line} EOL export CONF_LINE +export mode=${mode} +EOF -awk -F: -v print=$mode ' +cat <<'EOF' +awk -F: -v mode=$mode ' BEGIN { split(ENVIRON["CONF_LINE"], conf, ":") } $1 == conf[1] { - if (print && !found) { + if (mode && !found) { # remove duplicates print ENVIRON["CONF_LINE"] found = 1 diff --git a/type/__dma_auth/man.rst b/type/__dma_auth/man.rst index bd077d6..9c3ad7a 100644 --- a/type/__dma_auth/man.rst +++ b/type/__dma_auth/man.rst @@ -14,16 +14,16 @@ servers. REQUIRED PARAMETERS ------------------- +login + The user's LOGIN name on the SMTP server. password The user's password (in plain text.) -server - The SMTP server on which the login is valid. OPTIONAL PARAMETERS ------------------- -login - The user's LOGIN name on the SMTP server. Defaults to `__object_id`. +server + The SMTP server on which the login is valid. Defaults to `__object_id`. state Either `present` or `absent`. Defaults to `present`. @@ -38,13 +38,13 @@ EXAMPLES .. code-block:: sh # Set the password for smarthost - __dma_auth joe --server smarthost --password hunter2 + __dma_auth smarthost.example.com --login joe --password hunter2 # Set credentials for user at an external provider - __dma_auth paul@example.com --server mail.provider.com --password letmein + __dma_auth mail.provider.com --login paul@example.com --password letmein - # Delete credentials for example.com - __dma_auth paul --server example.com --state absent + # Delete credentials for example.com (for all users) + __dma_auth example.com --login '' --password '' --state absent SEE ALSO -------- diff --git a/type/__dma_auth/parameter/optional b/type/__dma_auth/parameter/optional index c35dbef..3e42ed3 100644 --- a/type/__dma_auth/parameter/optional +++ b/type/__dma_auth/parameter/optional @@ -1,2 +1,2 @@ -login +server state diff --git a/type/__dma_auth/parameter/required b/type/__dma_auth/parameter/required index 8f1a1c9..ae3c622 100644 --- a/type/__dma_auth/parameter/required +++ b/type/__dma_auth/parameter/required @@ -1,2 +1,2 @@ +login password -server From b848fca9299befe36b78624cf622d6df286db053 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 31 May 2020 15:01:40 +0200 Subject: [PATCH 010/332] [type/__dma_auth] Finish code to rewrite auth.conf --- type/__dma_auth/explorer/authusers | 12 +++- type/__dma_auth/gencode-remote | 90 +++++++++++++++++++++--------- 2 files changed, 73 insertions(+), 29 deletions(-) diff --git a/type/__dma_auth/explorer/authusers b/type/__dma_auth/explorer/authusers index 5fc6b4e..db83482 100755 --- a/type/__dma_auth/explorer/authusers +++ b/type/__dma_auth/explorer/authusers @@ -18,7 +18,7 @@ # along with cdist. If not, see . # # This explorer looks for lines matching the server parameter in dma's auth.conf -# and reports the login and server fields (password is stripped off) +# and reports the login and server fields (password is cksummed) auth_conf=$("${__type_explorer}/auth_conf") test -r "${auth_conf}" || exit 0 @@ -47,5 +47,11 @@ BEGIN { } else endpos = length } -host == server { print substr($0, 0, endpos) } -' "${auth_conf}" +host == server { print endpos, $0 } +' "${auth_conf}" \ +| while read pos line + do + printf '%s:%s\n' \ + "$(printf '%s' "$line" | cut -c $((-pos)))" \ + "$(printf '%s' "$line" | cut -c $((pos+2))- | cksum | cut -d' ' -f1)" + done diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 989e176..d75d611 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,7 +18,6 @@ # along with cdist. If not, see . # -authusers=$(cat "${__object}/explorer/authusers") state_should=$(cat "${__object}/parameter/state") if test -f "${__object}/parameter/server" @@ -28,27 +27,26 @@ else server=$__object_id fi login=$(cat "${__object}/parameter/login") -password=$(cat "${__object}/parameter/password") case $state_should in (present) - conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "${password}") - cksum_should=$(echo "${conf_line}" | cksum - | cut -d ' ' -f 1) - if echo "$authusers" | grep -qxF "${cksum_should}" + line_should=$(printf '%s|%s:%s\n' \ + "${login}" "${server}" \ + "$(cksum "${__object}/parameter/password" | cut -d' ' -f1)") + if grep -qxF "${line_should}" "${__object}/explorer/authusers" then # correct line already present -> nothing to do exit 0 fi + test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1 } + mode=1 ;; (absent) - # no logins present -> nothing to do - test -n "$authusers" || exit 0 - - # NOTE: password is not needed to delete - conf_line=$(printf '%s|%s:%s\n' "${login}" "${server}" "") + # no matching logins present -> nothing to do + test -s "${__object}/explorer/authusers" || exit 0 mode=0 ;; @@ -58,25 +56,65 @@ in ;; esac +auth_conf=$(cat "${__object}/explorer/auth_conf") + +if test -z "${auth_conf}" +then + echo 'Cannot determine path of dma auth.conf' >&2 + exit 1 +fi + cat </etc/dma/auth.conf.tmp \ -&& mv /etc/dma/auth.conf.tmp /etc/dma/auth.conf + +BEGIN { + DP = "[: \t]" # copied from dma/conf.c +} + +# skip comments and empty lines +/^#/ || /^$/ { print; next } + +{ + login = substr($0, 1, index($0, "|") - 1) + host = substr($0, length(login) + 2) + if (match(host, DP)) { + host = substr(host, 1, RSTART - 1) + endpos = length(login) + RSTART + } else endpos = length +} + +host == ENVIRON["server"] { + if (mode) { + if (login == ENVIRON["login"] && !written) { + printf "%s%s\n", substr($0, 1, endpos+1), getpw() + written = 1 + next + } + } else if (!ENVIRON["login"] || login == ENVIRON["login"]) next +} + +{ print } + +END { + if (mode && !written) { + printf "%s|%s:%s\n", ENVIRON["login"], ENVIRON["server"], getpw() + } +} +' <"${auth_conf}" >"${auth_conf}.tmp" \ + && mv "${auth_conf}.tmp" "${auth_conf}" EOF From 3f72ca134108163f3208b709f84037312d460e5c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 31 May 2020 15:18:11 +0200 Subject: [PATCH 011/332] [type/__dma_auth] Send messages --- type/__dma/gencode-remote | 45 +++++++++++++++++++++------------- type/__dma_auth/gencode-remote | 11 ++++++++- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 2961c09..2e3a80d 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -1,20 +1,31 @@ #!/bin/sh -e -if [ -f "${__object}/parameter/send-test-email" ]; then - SEND_EMAIL="YES" +if test -f "${__object}/parameter/send-test-email" +then + modified=false + + if grep -q '^__mail_alias/root:' "${__messages_in}" + then + modified=true + elif grep -q '^__dma_auth/' "${__messages_in}" + then + modified=true + elif grep -q '^__dma/' "${__messages_in}" + then + modified=true + fi + + if $modified + then + cat <<-EOF + sendmail root <&2; exit 1 } mode=1 - ;; + + if test -s "${__object}/explorer/authusers" + then + printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + else + printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + fi + ;; (absent) # no matching logins present -> nothing to do test -s "${__object}/explorer/authusers" || exit 0 mode=0 + + printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" ;; (*) printf 'Invalid --state: %s' "${state_should}" >&2 From b87b67597efb8349709e8f5153bb73bb08f39337 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 31 May 2020 15:06:06 +0200 Subject: [PATCH 012/332] [type/__dma] Remove mail aliases functionality --- type/__dma/files/aliases | 68 -------------------------- type/__dma/man.rst | 22 ++------- type/__dma/manifest | 22 +-------- type/__dma/parameter/required_multiple | 1 - 4 files changed, 6 insertions(+), 107 deletions(-) delete mode 100644 type/__dma/files/aliases delete mode 100644 type/__dma/parameter/required_multiple diff --git a/type/__dma/files/aliases b/type/__dma/files/aliases deleted file mode 100644 index d341318..0000000 --- a/type/__dma/files/aliases +++ /dev/null @@ -1,68 +0,0 @@ -# Based off FreeBSD's /etc/aliases -# -# >>>>>>>>>> The program "newaliases" must be run after -# >> NOTE >> this file is updated for any changes to -# >>>>>>>>>> show through to sendmail. -# -# -# See also RFC 2142, `MAILBOX NAMES FOR COMMON SERVICES, ROLES -# AND FUNCTIONS', May 1997 -# http://tools.ietf.org/html/rfc2142 - -# Pretty much everything else in this file points to "root", so -# you would do well in either reading root's mailbox or forwarding -# root's email from here. - -# root: me@my.domain - - -# Basic system aliases -- these MUST be present -MAILER-DAEMON: postmaster -postmaster: root - -# General redirections for pseudo accounts -_dhcp: root -_pflogd: root -auditdistd: root -bin: root -bind: root -daemon: root -games: root -hast: root -kmem: root -mailnull: postmaster -man: root -news: root -nobody: root -operator: root -pop: root -proxy: root -smmsp: postmaster -sshd: root -system: root -toor: root -tty: root -usenet: news -uucp: root - -# Well-known aliases -- these should be filled in! -manager: root -dumper: root - -# BUSINESS-RELATED MAILBOX NAMES -info: root -marketing: root -sales: root -support: root - -# NETWORK OPERATIONS MAILBOX NAMES -abuse: root -noc: root -security: root - -# SUPPORT MAILBOX NAMES FOR SPECIFIC INTERNET SERVICES -ftp: root -ftp-bugs: ftp -hostmaster: root -webmaster: root -www: webmaster diff --git a/type/__dma/man.rst b/type/__dma/man.rst index a10c6c2..af9298e 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -19,18 +19,9 @@ email server configured in the `smart-host` parameter. REQUIRED PARAMETERS ------------------- smart-host - The destination email server. The addresses passed in `root-email` must be - either local to the `smart-host` or it must be configured to act as a relay - for the host being configured by this type. - - -REQUIRED MULTIPLE PARAMETERS ----------------------------- -root-email - Destination email address. Can be specified multiple times or just once - with each address separated by commas. - This will be setup in `/etc/aliases` as the destination for the local - root mailbox. + The email server used to send email. + It must be configured to act as a relay for the host being configured by + this type so that mail can be sent to users non-local to the smart-host. BOOLEAN PARAMETERS @@ -55,11 +46,7 @@ EXAMPLES .. code-block:: sh - # Send root email to both our BOFH and the nice-admin. - # That way they can figure things out together. __dma \ - --root-email bofh@domain.tld \ - --root-email nice-admin@domain.tld \ --smart-host mx1.domain.tld \ --send-test-email @@ -73,11 +60,12 @@ SEE ALSO AUTHORS ------- Evilham +Dennis Camera COPYING ------- -Copyright \(C) 2020 Evilham. You can redistribute it +Copyright \(C) 2020 Evilham and Dennis Camera. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/type/__dma/manifest b/type/__dma/manifest index abcaa5b..e07fbfc 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -1,8 +1,7 @@ #!/bin/sh -e -os="$(cat "${__global}/explorer/os")" +os=$(cat "${__global}/explorer/os") -root_email="$(tr '\n' ',' < "${__object}/parameter/root-email" | sed -E 's/,+$//')" smart_host="$(cat "${__object}/parameter/smart-host")" if [ -f "${__object}/parameter/mailname" ]; then @@ -21,14 +20,12 @@ else esac fi -aliases_file=/etc/mail/aliases case ${os} in debian|devuan|ubuntu) # Debian-like requires installing DMA __package dma # Moving forward without DMA doesn't make much sense export require="__package/dma" - aliases_file=/etc/aliases ;; freebsd) # Disable sendmail + stop if necessary @@ -134,20 +131,3 @@ EOF __file /etc/dma/dma.conf --mode 0644 --source '-' < Date: Mon, 1 Jun 2020 17:07:35 +0200 Subject: [PATCH 013/332] [type/__dma] Implement config file updates --- type/__dma/explorer/conf | 30 ++++ type/__dma/gencode-remote | 197 ++++++++++++++++++++++++- type/__dma/man.rst | 24 +++ type/__dma/manifest | 165 ++++++--------------- type/__dma/parameter/default/security | 1 + type/__dma/parameter/optional | 2 + type/__dma/parameter/optional_multiple | 1 + 7 files changed, 302 insertions(+), 118 deletions(-) create mode 100755 type/__dma/explorer/conf create mode 100644 type/__dma/parameter/default/security create mode 100644 type/__dma/parameter/optional_multiple diff --git a/type/__dma/explorer/conf b/type/__dma/explorer/conf new file mode 100755 index 0000000..129e3c3 --- /dev/null +++ b/type/__dma/explorer/conf @@ -0,0 +1,30 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# This explorer looks for lines matching the server parameter in dma's auth.conf +# and reports the login and server fields (password is cksummed) + +CONF_PATH=/etc/dma # set in Makefile +dma_conf="${CONF_PATH:?}/dma.conf" + +test -f "${dma_conf}" || exit 0 + +grep -v -e '^[ \t]*#\|^$' "${dma_conf}" \ +| sed -e 's/[ \t]*#.*$//' \ +| sort -s -k 1,1 diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 2e3a80d..1091aeb 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -1,5 +1,200 @@ #!/bin/sh -e +CONF_PATH=/etc/dma # set in Makefile + +# Generate config +conf_should=$( + if test -s "${__object}/parameter/smart-host" + then + printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smart-host")" + #printf 'AUTHPATH %s\n' "$(cat "${__type}/ + fi + + case $(cat "${__object}/parameter/security") + in + (ssl|tls) + default_smtp_port=465 + echo 'SECURETRANSFER' + ;; + (starttls) + default_smtp_port=587 + echo 'SECURETRANSFER' + echo 'STARTTLS' + ;; + (opportunistic) + default_smtp_port=25 # XXX: correct? + echo 'SECURETRANSFER' + echo 'STARTTLS' + echo 'OPPORTUNISTIC_TLS' + ;; + (insecure) + default_smtp_port=25 + echo 'INSECURE' + ;; + esac + + if test -s "${__object}/parameter/port" + then + printf 'PORT %u\n' "$(cat "${__object}/parameter/port")" + elif test "${default_smtp_port}" -ne 25 + then + printf 'PORT %u\n' "${default_smtp_port}" + fi + + if test -f "${__object}/parameter/masquerade" + then + while read -r line + do + printf 'MASQUERADE %s\n' "${line}" + done <"${__object}/parameter/masquerade" + fi + + +) +conf_should=$(echo "$conf_should" | sort -s -k 1,1) + +config_updated=false +if ! echo "$conf_should" | cmp -s "${__object}/explorer/conf" - +then + # config needs to be updated + echo "dma_conf='${CONF_PATH:?}/dma.conf'" + cat <<'EOF' +awk -F '\n' ' +function comment_line(line) { return match(line, /^[ \t]*#+[ \t]*/) } +function empty_line(line) { return match(line, /^[ \t]*$/) } +function is_word(s) { return s ~ /^[A-Z_]+$/ } + +function first(line, sep) { + if (!sep) sep = SUBSEP + return index(line, sep) ? substr(line, 0, index(line, sep)) : line +} + +function rest(line, sep) { + if (!sep) sep = SUBSEP + if (index(line, sep)) + return substr(line, index(line, sep) + 1) +} + +function conf_pop(word, value) { + if (!(word in conf)) return 0 + if (!value) { + if (index(conf[word], SUBSEP)) # more than one element? + value = substr(conf[word], 0, index(conf[word], SUBSEP)) + else + value = conf[word] + } + + if (index(conf[word], SUBSEP)) { + if (index(conf[word], value SUBSEP) != 1) return 0 + conf[word] = substr(conf[word], length(value) + 2) + } else { + if (conf[word] != value) return 0 + delete conf[word] + } + return value +} + +function print_conf(word, value) { + printf "%s", word + if (value) printf " %s", value + printf "\n" +} + +function print_confs(word, value) { + if (!(word in conf)) return + if (conf[word]) { + while (value = conf_pop(word)) + print_conf(word, value) + } else { + print_conf(word) + delete conf[word] + } +} + +BEGIN { + while (getline < "/dev/stdin") { + word = first($0, " ") + if ((word in conf)) + conf[word] = conf[word] SUBSEP rest($0, " ") + else + conf[word] = rest($0, " ") + } +} + +# first pass, gather information +NR == FNR { + if (comment_line($0)) { + # comment line + word = first(substr($0, RLENGTH + 1), " ") + if (is_word(word)) last_occ["#" word] = FNR + } else { + word = first($0, " ") + if (is_word(word)) last_occ[word] = FNR + } +} + +NR > FNR && FNR == 1 { + # before second pass prepare hashes + + for (k in last_occ) + if (k ~ /^\#/ && (substr(k, 2) in last_occ)) + delete last_occ[k] + + for (k in last_occ) { + line_map[last_occ[k]] = k + } +} + +# second pass, output new config +NR > FNR { + if (comment_line($0) || empty_line($0)) { + # comment or empty line + print + + if ((FNR in line_map)) { + if (line_map[FNR] ~ /^\#/) { + # the "matching" comment line is here + k = substr(line_map[FNR], 2) + if ((k in conf)) print_confs(k) + } + + if (("INSECURE" in conf) && line_map[FNR] ~ /^\#?SECURE$/) { + # INSECURE goes where SECURE comment is + print_confs("INSECURE") + } + } + } else { + sub(/[ \t]*\#.*$/, "", $0) # ignore comments + word = first($0, " ") + + if ((word in conf) && rest($0, " ") == first(conf[word])) { + # keep config options we want + conf_pop(word) + print + } + + if ((FNR in line_map) && line_map[FNR] == word) { + # rest of config options should be here + print_confs(word) + } + } +} + +END { + # print rest of config options + for (word in conf) print_confs(word) +} +' "${dma_conf}" "${dma_conf}" <<'EOF' >"${dma_conf}.tmp" \ + && mv "${dma_conf}.tmp" "${dma_conf}" +EOF + echo "${conf_should}" + echo 'EOF' + + config_updated=true + echo 'config updated' >>"${__messages_out}" +fi + + if test -f "${__object}/parameter/send-test-email" then modified=false @@ -10,7 +205,7 @@ then elif grep -q '^__dma_auth/' "${__messages_in}" then modified=true - elif grep -q '^__dma/' "${__messages_in}" + elif $config_updated then modified=true fi diff --git a/type/__dma/man.rst b/type/__dma/man.rst index af9298e..cb3c35f 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -39,7 +39,31 @@ mailname If not defined, it defaults to `/etc/mailname` on Debian-derived Operating Systems and to `__target_host` otherwise. See `dma(8)` for more information. +masquerade + Masquerade the envelope-from addresses with this address/hostname. + Use this setting if mails are not accepted by destination mail servers + because your sender domain is invalid. + This option can be used multiple times. + For more information see the `dma(8)` man page. +port + The port on which to deliver email. + If not provided, a sensible default port will be used based on the + `--security` argument. +security + Configures whether and how DMA should use secure connections. + ssl/tls + Enable TLS/SSL secured transfer. + starttls + Use STARTTLS to establish a secure connection. + opportunistic (default) + Will try to establish a secure connection using STARTTLS, but allow + unencrypted transfer if STARTTLS fails. + Most useful when dma is used without a smarthost, delivering remote + messages directly to the outside mail exchangers. + insecure + allow plain text SMTP login over an insecure connection. + Should really not be used anymore! EXAMPLES -------- diff --git a/type/__dma/manifest b/type/__dma/manifest index e07fbfc..814e3ef 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -2,54 +2,57 @@ os=$(cat "${__global}/explorer/os") -smart_host="$(cat "${__object}/parameter/smart-host")" - -if [ -f "${__object}/parameter/mailname" ]; then - mailname="$(cat "${__object}/parameter/mailname")" +# mailname: default behaviour is different on certain systems +if test -f "${__object}/parameter/mailname" +then + mailname=$(cat "${__object}/parameter/mailname") else - # default mailname behaviour is different in certain systems - case ${os} in - debian|devuan|ubuntu) - # Debian-like default to /etc/mailname - mailname="/etc/mailname" - ;; - *) - # Otherwise let's use the hostname - mailname="${__target_host}" - ;; - esac + # Otherwise use the hostname + mailname=$(cat "${__global}/explorer/hostname") fi -case ${os} in - debian|devuan|ubuntu) - # Debian-like requires installing DMA - __package dma - # Moving forward without DMA doesn't make much sense - export require="__package/dma" - ;; - freebsd) - # Disable sendmail + stop if necessary - __key_value \ - --file "/etc/rc.conf" \ - --comment "# Disable sendmail " \ - --key "sendmail_enable" \ - --delimiter "=" \ - --value "NONE" \ - --onchange "service sendmail onestop || true" \ - "sendmail_enable" +case $os +in + (debian|devuan|ubuntu) + # On Debian-like systems use /etc/mailname + if test -f "${__object}/parameter/mailname" + then + echo "$mailname" | __file '/etc/mailname' --state present \ + --mode 0644 --owner root --group root --source - + fi + + mailname='/etc/mailname' + ;; +esac + +# Install DMA +case $os +in + (debian|devuan|ubuntu) + __package dma --state present + export require='__package/dma' + ;; + (freebsd) + # Stop sendmail if necessary + __process 'sendmail' --name 'sendmail.*' --state absent \ + --stop '/etc/rc.d/sendmail onestop' + + # ... and disable it + __key_value 'rcconf-sendmail-enable' --file '/etc/rc.conf' \ + --key 'sendmail_enable' --delimiter '=' --value '"NONE"' \ + --exact_delimiter + # Setup mailwrapper accordingly - __file /etc/mail/mailer.conf \ - --mode 0644 \ - --source '-' < /dev/stderr <&2 Your OS (${os}) is not supported yet. Maybe adding support is as simple as adapting the packages or allowing it, @@ -57,77 +60,5 @@ we highly encourage you to open a PR with the necessary changes. See: https://code.ungleich.ch/ungleich-public/cdist-contrib/ EOF exit 1 - ;; + ;; esac - -DMA_CONF="$(cat < Date: Mon, 1 Jun 2020 17:16:04 +0200 Subject: [PATCH 014/332] [type/__dma] Detect AUTHPATH --- type/__dma/explorer/auth_conf | 1 + type/__dma/gencode-remote | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 120000 type/__dma/explorer/auth_conf diff --git a/type/__dma/explorer/auth_conf b/type/__dma/explorer/auth_conf new file mode 120000 index 0000000..db038ae --- /dev/null +++ b/type/__dma/explorer/auth_conf @@ -0,0 +1 @@ +../../__dma_auth/explorer/auth_conf \ No newline at end of file diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 1091aeb..105edba 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -49,7 +49,10 @@ conf_should=$( done <"${__object}/parameter/masquerade" fi - + if test -s "${__object}/explorer/auth_conf" + then + printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" + fi ) conf_should=$(echo "$conf_should" | sort -s -k 1,1) From 4fdddfd738dac87a1e33e359e1ede6ee22a16918 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 17:23:31 +0200 Subject: [PATCH 015/332] [type/__dma] Add --defer, --full-bounce, and --null-client --- type/__dma/gencode-remote | 20 ++++++++++++++++++++ type/__dma/man.rst | 8 ++++++++ type/__dma/parameter/boolean | 3 +++ 3 files changed, 31 insertions(+) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 105edba..1f252a4 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -49,6 +49,26 @@ conf_should=$( done <"${__object}/parameter/masquerade" fi + if test -f "${__object}/parameter/defer" + then + echo 'DEFER' + fi + + if test -f "${__object}/parameter/full-bounce" + then + echo 'FULLBOUNCE' + fi + + if test -f "${__object}/parameter/null-client" + then + test -s "${__object}/parameter/smart-host" || { + echo '--null-client requires a --smart-host to be defined' >&2 + exit 1 + } + + echo 'NULLCLIENT' + fi + if test -s "${__object}/explorer/auth_conf" then printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" diff --git a/type/__dma/man.rst b/type/__dma/man.rst index cb3c35f..a8ba546 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -26,6 +26,14 @@ smart-host BOOLEAN PARAMETERS ------------------ +defer + If enabled, the mail queue has to be manually flushed with the `-q` option. +full-bounce + Enable if the bounce message should include the complete original message, + not just the headers. +null-client + Enable to bypass aliases and local delivery, and instead forward all mails + to the defined `--smart-host`. send-test-email If present, after setup this type will send an email to root, to allow you to easily test your setup. diff --git a/type/__dma/parameter/boolean b/type/__dma/parameter/boolean index d1af563..ede7dda 100644 --- a/type/__dma/parameter/boolean +++ b/type/__dma/parameter/boolean @@ -1 +1,4 @@ +defer +full-bounce +null-client send-test-email From 99d58672c4987757429fc783b66d96eb6fa90f2b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 19:20:05 +0200 Subject: [PATCH 016/332] [type/__dma_auth] Add semicolon --- type/__dma_auth/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 48e7dce..b6b7f63 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -40,7 +40,7 @@ in exit 0 fi - test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1 } + test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1; } mode=1 From 0657ac4f115ee575f0bb7780fa9ac759c472ec97 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 19:21:41 +0200 Subject: [PATCH 017/332] [type/__dma] Fix mailname --- type/__dma/gencode-remote | 31 +++++++++++++++++++++++++------ type/__dma/manifest | 27 +-------------------------- 2 files changed, 26 insertions(+), 32 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 1f252a4..4100d39 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -2,12 +2,36 @@ CONF_PATH=/etc/dma # set in Makefile +# Determine mailname +if test -f "${__object}/parameter/mailname" +then + mailname=$(cat "${__object}/parameter/mailname") +else + case $(cat "${__global}/explorer/os") + in + (debian|devuan|ubuntu) + # On Debian-like systems use /etc/mailname unless --mailname is used + mailname='/etc/mailname' + ;; + (*) + mailname=$__target_fqdn + ;; + esac +fi + + # Generate config conf_should=$( if test -s "${__object}/parameter/smart-host" then printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smart-host")" - #printf 'AUTHPATH %s\n' "$(cat "${__type}/ + fi + + printf 'MAILNAME %s\n' "${mailname}" + + if test -s "${__object}/explorer/auth_conf" + then + printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" fi case $(cat "${__object}/parameter/security") @@ -68,11 +92,6 @@ conf_should=$( echo 'NULLCLIENT' fi - - if test -s "${__object}/explorer/auth_conf" - then - printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" - fi ) conf_should=$(echo "$conf_should" | sort -s -k 1,1) diff --git a/type/__dma/manifest b/type/__dma/manifest index 814e3ef..60038b1 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -1,32 +1,7 @@ #!/bin/sh -e -os=$(cat "${__global}/explorer/os") - -# mailname: default behaviour is different on certain systems -if test -f "${__object}/parameter/mailname" -then - mailname=$(cat "${__object}/parameter/mailname") -else - # Otherwise use the hostname - mailname=$(cat "${__global}/explorer/hostname") -fi - -case $os -in - (debian|devuan|ubuntu) - # On Debian-like systems use /etc/mailname - if test -f "${__object}/parameter/mailname" - then - echo "$mailname" | __file '/etc/mailname' --state present \ - --mode 0644 --owner root --group root --source - - fi - - mailname='/etc/mailname' - ;; -esac - # Install DMA -case $os +case $(cat "${__global}/explorer/os") in (debian|devuan|ubuntu) __package dma --state present From 7183bb3cd191dfece36e32db86eb6ee42b49bb5d Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 19:24:32 +0200 Subject: [PATCH 018/332] [type/__dma] Fixes for FreeBSD --- type/__dma/gencode-remote | 13 ++++++++----- type/__dma/manifest | 3 +++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 4100d39..1b00f04 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -108,7 +108,7 @@ function is_word(s) { return s ~ /^[A-Z_]+$/ } function first(line, sep) { if (!sep) sep = SUBSEP - return index(line, sep) ? substr(line, 0, index(line, sep)) : line + return index(line, sep) ? substr(line, 1, index(line, sep) - 1) : line } function rest(line, sep) { @@ -121,7 +121,7 @@ function conf_pop(word, value) { if (!(word in conf)) return 0 if (!value) { if (index(conf[word], SUBSEP)) # more than one element? - value = substr(conf[word], 0, index(conf[word], SUBSEP)) + value = substr(conf[word], 1, index(conf[word], SUBSEP) - 1) else value = conf[word] } @@ -256,10 +256,13 @@ then then cat <<-EOF sendmail root < Date: Mon, 1 Jun 2020 20:25:10 +0200 Subject: [PATCH 019/332] [type/__dma_auth] Fix SC2162 --- type/__dma_auth/explorer/authusers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__dma_auth/explorer/authusers b/type/__dma_auth/explorer/authusers index db83482..c76667b 100755 --- a/type/__dma_auth/explorer/authusers +++ b/type/__dma_auth/explorer/authusers @@ -49,7 +49,7 @@ BEGIN { host == server { print endpos, $0 } ' "${auth_conf}" \ -| while read pos line +| while read -r pos line do printf '%s:%s\n' \ "$(printf '%s' "$line" | cut -c $((-pos)))" \ From bf822f3f8ca53852554750003296b9163122c424 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 1 Jun 2020 20:26:52 +0200 Subject: [PATCH 020/332] [type/__dma] Fix SC2154 --- type/__dma/manifest | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/type/__dma/manifest b/type/__dma/manifest index 4a78dab..7abd7c8 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -1,7 +1,9 @@ #!/bin/sh -e +os=$(cat "${__global}/explorer/os") + # Install DMA -case $(cat "${__global}/explorer/os") +case $os in (debian|devuan|ubuntu) __package dma --state present From 3adb5ac4cabe65b24c0a5139a62b38b35b9191bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 14:58:43 +0200 Subject: [PATCH 021/332] Import __ungleich_unbound type for ungleich's dot-cdist --- type/__ungleich_unbound/files/unbound.conf.sh | 1018 +++++++++++++++++ type/__ungleich_unbound/man.rst | 74 ++ type/__ungleich_unbound/manifest | 79 ++ type/__ungleich_unbound/parameter/boolean | 2 + .../parameter/optional_multiple | 2 + type/__ungleich_unbound/parameter/required | 1 + .../parameter/required_multiple | 1 + type/__ungleich_unbound/singleton | 0 8 files changed, 1177 insertions(+) create mode 100755 type/__ungleich_unbound/files/unbound.conf.sh create mode 100644 type/__ungleich_unbound/man.rst create mode 100755 type/__ungleich_unbound/manifest create mode 100644 type/__ungleich_unbound/parameter/boolean create mode 100644 type/__ungleich_unbound/parameter/optional_multiple create mode 100644 type/__ungleich_unbound/parameter/required create mode 100644 type/__ungleich_unbound/parameter/required_multiple create mode 100644 type/__ungleich_unbound/singleton diff --git a/type/__ungleich_unbound/files/unbound.conf.sh b/type/__ungleich_unbound/files/unbound.conf.sh new file mode 100755 index 0000000..e713843 --- /dev/null +++ b/type/__ungleich_unbound/files/unbound.conf.sh @@ -0,0 +1,1018 @@ +#!/bin/sh + +generate_interface() { + for i in $INTERFACES; do + echo " interface: $i" + done +} + +generate_access_control() { + echo "$ACCESS_CONTROLS" | while read entry; do + echo " access-control: $entry" + done +} + +generate_forward_addr() { + for addr in $FORWARD_ADDRS; do + echo " forward-addr: $addr" + done +} + +cat << EOF +# +# THIS FILE HAS BEEN GENERATED BY CDIST, DO NOT EDIT BY HAND. +# +# See unbound.conf(5) man page, version 1.9.1. +# +# this is a comment. + +#Use this to include other text into the file. +#include: "otherfile.conf" + +# The server clause sets the main parameters. +server: + # whitespace is not necessary, but looks cleaner. + + # verbosity number, 0 is least verbose. 1 is default. + verbosity: 1 + + # print statistics to the log (for every thread) every N seconds. + # Set to "" or 0 to disable. Default is disabled. + # statistics-interval: 0 + + # enable shm for stats, default no. if you enable also enable + # statistics-interval, every time it also writes stats to the + # shared memory segment keyed with shm-key. + # shm-enable: no + + # shm for stats uses this key, and key+1 for the shared mem segment. + # shm-key: 11777 + + # enable cumulative statistics, without clearing them after printing. + # statistics-cumulative: no + + # enable extended statistics (query types, answer codes, status) + # printed from unbound-control. default off, because of speed. + # extended-statistics: no + + # number of threads to create. 1 disables threading. + # num-threads: 1 + + # specify the interfaces to answer queries from by ip-address. + # The default is to listen to localhost (127.0.0.1 and ::1). + # specify 0.0.0.0 and ::0 to bind to all available interfaces. + # specify every interface[@port] on a new 'interface:' labelled line. + # The listen interfaces are not changed on reload, only on restart. + # interface: 192.0.2.153 + # interface: 192.0.2.154 + # interface: 192.0.2.154@5003 + # interface: 2001:DB8::5 +$(generate_interface) + + # enable this feature to copy the source address of queries to reply. + # Socket options are not supported on all platforms. experimental. + # interface-automatic: no + + # port to answer queries from + # port: 53 + + # specify the interfaces to send outgoing queries to authoritative + # server from by ip-address. If none, the default (all) interface + # is used. Specify every interface on a 'outgoing-interface:' line. + # outgoing-interface: 192.0.2.153 + # outgoing-interface: 2001:DB8::5 + # outgoing-interface: 2001:DB8::6 + + # Specify a netblock to use remainder 64 bits as random bits for + # upstream queries. Uses freebind option (Linux). + # outgoing-interface: 2001:DB8::/64 + # Also (Linux:) ip -6 addr add 2001:db8::/64 dev lo + # And: ip -6 route add local 2001:db8::/64 dev lo + # And set prefer-ip6: yes to use the ip6 randomness from a netblock. + # Set this to yes to prefer ipv6 upstream servers over ipv4. + # prefer-ip6: no + + # number of ports to allocate per thread, determines the size of the + # port range that can be open simultaneously. About double the + # num-queries-per-thread, or, use as many as the OS will allow you. + # outgoing-range: 4096 + + # permit unbound to use this port number or port range for + # making outgoing queries, using an outgoing interface. + # outgoing-port-permit: 32768 + + # deny unbound the use this of port number or port range for + # making outgoing queries, using an outgoing interface. + # Use this to make sure unbound does not grab a UDP port that some + # other server on this computer needs. The default is to avoid + # IANA-assigned port numbers. + # If multiple outgoing-port-permit and outgoing-port-avoid options + # are present, they are processed in order. + # outgoing-port-avoid: "3200-3208" + + # number of outgoing simultaneous tcp buffers to hold per thread. + # outgoing-num-tcp: 10 + + # number of incoming simultaneous tcp buffers to hold per thread. + # incoming-num-tcp: 10 + + # buffer size for UDP port 53 incoming (SO_RCVBUF socket option). + # 0 is system default. Use 4m to catch query spikes for busy servers. + # so-rcvbuf: 0 + + # buffer size for UDP port 53 outgoing (SO_SNDBUF socket option). + # 0 is system default. Use 4m to handle spikes on very busy servers. + # so-sndbuf: 0 + + # use SO_REUSEPORT to distribute queries over threads. + # so-reuseport: yes + + # use IP_TRANSPARENT so the interface: addresses can be non-local + # and you can config non-existing IPs that are going to work later on + # (uses IP_BINDANY on FreeBSD). + # ip-transparent: no + + # use IP_FREEBIND so the interface: addresses can be non-local + # and you can bind to nonexisting IPs and interfaces that are down. + # Linux only. On Linux you also have ip-transparent that is similar. + # ip-freebind: no + + # EDNS reassembly buffer to advertise to UDP peers (the actual buffer + # is set with msg-buffer-size). 1472 can solve fragmentation (timeouts) + # edns-buffer-size: 4096 + + # Maximum UDP response size (not applied to TCP response). + # Suggested values are 512 to 4096. Default is 4096. 65536 disables it. + # max-udp-size: 4096 + + # max memory to use for stream(tcp and tls) waiting result buffers. + # stream-wait-size: 4m + + # buffer size for handling DNS data. No messages larger than this + # size can be sent or received, by UDP or TCP. In bytes. + # msg-buffer-size: 65552 + + # the amount of memory to use for the message cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # msg-cache-size: 4m + + # the number of slabs to use for the message cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # msg-cache-slabs: 4 + + # the number of queries that a thread gets to service. + # num-queries-per-thread: 1024 + + # if very busy, 50% queries run to completion, 50% get timeout in msec + # jostle-timeout: 200 + + # msec to wait before close of port on timeout UDP. 0 disables. + # delay-close: 0 + + # msec for waiting for an unknown server to reply. Increase if you + # are behind a slow satellite link, to eg. 1128. + # unknown-server-time-limit: 376 + + # the amount of memory to use for the RRset cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # rrset-cache-size: 4m + + # the number of slabs to use for the RRset cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # rrset-cache-slabs: 4 + + # the time to live (TTL) value lower bound, in seconds. Default 0. + # If more than an hour could easily give trouble due to stale data. + # cache-min-ttl: 0 + + # the time to live (TTL) value cap for RRsets and messages in the + # cache. Items are not cached for longer. In seconds. + # cache-max-ttl: 86400 + + # the time to live (TTL) value cap for negative responses in the cache + # cache-max-negative-ttl: 3600 + + # the time to live (TTL) value for cached roundtrip times, lameness and + # EDNS version information for hosts. In seconds. + # infra-host-ttl: 900 + + # minimum wait time for responses, increase if uplink is long. In msec. + # infra-cache-min-rtt: 50 + + # the number of slabs to use for the Infrastructure cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # infra-cache-slabs: 4 + + # the maximum number of hosts that are cached (roundtrip, EDNS, lame). + # infra-cache-numhosts: 10000 + + # define a number of tags here, use with local-zone, access-control. + # repeat the define-tag statement to add additional tags. + # define-tag: "tag1 tag2 tag3" + + # Enable IPv4, "yes" or "no". + do-ip4: $DO_IP4 + + # Enable IPv6, "yes" or "no". + do-ip6: $DO_IP6 + + # Enable UDP, "yes" or "no". + # do-udp: yes + + # Enable TCP, "yes" or "no". + # do-tcp: yes + + # upstream connections use TCP only (and no UDP), "yes" or "no" + # useful for tunneling scenarios, default no. + # tcp-upstream: no + + # upstream connections also use UDP (even if do-udp is no). + # useful if if you want UDP upstream, but don't provide UDP downstream. + # udp-upstream-without-downstream: no + + # Maximum segment size (MSS) of TCP socket on which the server + # responds to queries. Default is 0, system default MSS. + # tcp-mss: 0 + + # Maximum segment size (MSS) of TCP socket for outgoing queries. + # Default is 0, system default MSS. + # outgoing-tcp-mss: 0 + + # Idle TCP timeout, connection closed in milliseconds + # tcp-idle-timeout: 30000 + + # Enable EDNS TCP keepalive option. + # edns-tcp-keepalive: no + + # Timeout for EDNS TCP keepalive, in msec. + # edns-tcp-keepalive-timeout: 120000 + + # Use systemd socket activation for UDP, TCP, and control sockets. + # use-systemd: no + + # Detach from the terminal, run in background, "yes" or "no". + # Set the value to "no" when unbound runs as systemd service. + # do-daemonize: yes + + # control which clients are allowed to make (recursive) queries + # to this server. Specify classless netblocks with /size and action. + # By default everything is refused, except for localhost. + # Choose deny (drop message), refuse (polite error reply), + # allow (recursive ok), allow_setrd (recursive ok, rd bit is forced on), + # allow_snoop (recursive and nonrecursive ok) + # deny_non_local (drop queries unless can be answered from local-data) + # refuse_non_local (like deny_non_local but polite error reply). + # access-control: 0.0.0.0/0 refuse + # access-control: 127.0.0.0/8 allow + # access-control: ::0/0 refuse + # access-control: ::1 allow + # access-control: ::ffff:127.0.0.1 allow +$(generate_access_control) + + # tag access-control with list of tags (in "" with spaces between) + # Clients using this access control element use localzones that + # are tagged with one of these tags. + # access-control-tag: 192.0.2.0/24 "tag2 tag3" + + # set action for particular tag for given access control element + # if you have multiple tag values, the tag used to lookup the action + # is the first tag match between access-control-tag and local-zone-tag + # where "first" comes from the order of the define-tag values. + # access-control-tag-action: 192.0.2.0/24 tag3 refuse + + # set redirect data for particular tag for access control element + # access-control-tag-data: 192.0.2.0/24 tag2 "A 127.0.0.1" + + # Set view for access control element + # access-control-view: 192.0.2.0/24 viewname + + # if given, a chroot(2) is done to the given directory. + # i.e. you can chroot to the working directory, for example, + # for extra security, but make sure all files are in that directory. + # + # If chroot is enabled, you should pass the configfile (from the + # commandline) as a full path from the original root. After the + # chroot has been performed the now defunct portion of the config + # file path is removed to be able to reread the config after a reload. + # + # All other file paths (working dir, logfile, roothints, and + # key files) can be specified in several ways: + # o as an absolute path relative to the new root. + # o as a relative path to the working directory. + # o as an absolute path relative to the original root. + # In the last case the path is adjusted to remove the unused portion. + # + # The pid file can be absolute and outside of the chroot, it is + # written just prior to performing the chroot and dropping permissions. + # + # Additionally, unbound may need to access /dev/random (for entropy). + # How to do this is specific to your OS. + # + # If you give "" no chroot is performed. The path must not end in a /. + # chroot: "" + + # if given, user privileges are dropped (after binding port), + # and the given username is assumed. Default is user "unbound". + # If you give "" no privileges are dropped. + # username: "unbound" + + # the working directory. The relative files in this config are + # relative to this directory. If you give "" the working directory + # is not changed. + # If you give a server: directory: dir before include: file statements + # then those includes can be relative to the working directory. + # directory: "" + + # the log file, "" means log to stderr. + # Use of this option sets use-syslog to "no". + # logfile: "" + + # Log to syslog(3) if yes. The log facility LOG_DAEMON is used to + # log to. If yes, it overrides the logfile. + # use-syslog: yes + + # Log identity to report. if empty, defaults to the name of argv[0] + # (usually "unbound"). + # log-identity: "" + + # print UTC timestamp in ascii to logfile, default is epoch in seconds. + # log-time-ascii: no + + # print one line with time, IP, name, type, class for every query. + # log-queries: no + + # print one line per reply, with time, IP, name, type, class, rcode, + # timetoresolve, fromcache and responsesize. + # log-replies: no + + # log with tag 'query' and 'reply' instead of 'info' for + # filtering log-queries and log-replies from the log. + # log-tag-queryreply: no + + # log the local-zone actions, like local-zone type inform is enabled + # also for the other local zone types. + # log-local-actions: no + + # print log lines that say why queries return SERVFAIL to clients. + # log-servfail: no + + # file to read root hints from. + # get one from https://www.internic.net/domain/named.cache + root-hints: /usr/share/dns-root-hints/named.root + + # enable to not answer id.server and hostname.bind queries. + # hide-identity: no + + # enable to not answer version.server and version.bind queries. + # hide-version: no + + # enable to not answer trustanchor.unbound queries. + # hide-trustanchor: no + + # the identity to report. Leave "" or default to return hostname. + # identity: "" + + # the version to report. Leave "" or default to return package version. + # version: "" + + # the target fetch policy. + # series of integers describing the policy per dependency depth. + # The number of values in the list determines the maximum dependency + # depth the recursor will pursue before giving up. Each integer means: + # -1 : fetch all targets opportunistically, + # 0: fetch on demand, + # positive value: fetch that many targets opportunistically. + # Enclose the list of numbers between quotes (""). + # target-fetch-policy: "3 2 1 0 0" + + # Harden against very small EDNS buffer sizes. + # harden-short-bufsize: no + + # Harden against unseemly large queries. + # harden-large-queries: no + + # Harden against out of zone rrsets, to avoid spoofing attempts. + # harden-glue: yes + + # Harden against receiving dnssec-stripped data. If you turn it + # off, failing to validate dnskey data for a trustanchor will + # trigger insecure mode for that zone (like without a trustanchor). + # Default on, which insists on dnssec data for trust-anchored zones. + # harden-dnssec-stripped: yes + + # Harden against queries that fall under dnssec-signed nxdomain names. + # harden-below-nxdomain: yes + + # Harden the referral path by performing additional queries for + # infrastructure data. Validates the replies (if possible). + # Default off, because the lookups burden the server. Experimental + # implementation of draft-wijngaards-dnsext-resolver-side-mitigation. + # harden-referral-path: no + + # Harden against algorithm downgrade when multiple algorithms are + # advertised in the DS record. If no, allows the weakest algorithm + # to validate the zone. + # harden-algo-downgrade: no + + # Sent minimum amount of information to upstream servers to enhance + # privacy. Only sent minimum required labels of the QNAME and set QTYPE + # to A when possible. + # qname-minimisation: yes + + # QNAME minimisation in strict mode. Do not fall-back to sending full + # QNAME to potentially broken nameservers. A lot of domains will not be + # resolvable when this option in enabled. + # This option only has effect when qname-minimisation is enabled. + # qname-minimisation-strict: no + + # Aggressive NSEC uses the DNSSEC NSEC chain to synthesize NXDOMAIN + # and other denials, using information from previous NXDOMAINs answers. + # aggressive-nsec: no + + # Use 0x20-encoded random bits in the query to foil spoof attempts. + # This feature is an experimental implementation of draft dns-0x20. + # use-caps-for-id: no + + # Domains (and domains in them) without support for dns-0x20 and + # the fallback fails because they keep sending different answers. + # caps-whitelist: "licdn.com" + # caps-whitelist: "senderbase.org" + + # Enforce privacy of these addresses. Strips them away from answers. + # It may cause DNSSEC validation to additionally mark it as bogus. + # Protects against 'DNS Rebinding' (uses browser as network proxy). + # Only 'private-domain' and 'local-data' names are allowed to have + # these private addresses. No default. + # private-address: 10.0.0.0/8 + # private-address: 172.16.0.0/12 + # private-address: 192.168.0.0/16 + # private-address: 169.254.0.0/16 + # private-address: fd00::/8 + # private-address: fe80::/10 + # private-address: ::ffff:0:0/96 + + # Allow the domain (and its subdomains) to contain private addresses. + # local-data statements are allowed to contain private addresses too. + # private-domain: "example.com" + + # If nonzero, unwanted replies are not only reported in statistics, + # but also a running total is kept per thread. If it reaches the + # threshold, a warning is printed and a defensive action is taken, + # the cache is cleared to flush potential poison out of it. + # A suggested value is 10000000, the default is 0 (turned off). + # unwanted-reply-threshold: 0 + + # Do not query the following addresses. No DNS queries are sent there. + # List one address per entry. List classless netblocks with /size, + # do-not-query-address: 127.0.0.1/8 + # do-not-query-address: ::1 + + # if yes, the above default do-not-query-address entries are present. + # if no, localhost can be queried (for testing and debugging). + # do-not-query-localhost: yes + + # if yes, perform prefetching of almost expired message cache entries. + # prefetch: no + + # if yes, perform key lookups adjacent to normal lookups. + # prefetch-key: no + + # deny queries of type ANY with an empty response. + # deny-any: no + + # if yes, Unbound rotates RRSet order in response. + # rrset-roundrobin: no + + # if yes, Unbound doesn't insert authority/additional sections + # into response messages when those sections are not required. + # minimal-responses: yes + + # true to disable DNSSEC lameness check in iterator. + # disable-dnssec-lame-check: no + + # module configuration of the server. A string with identifiers + # separated by spaces. Syntax: "[dns64] [validator] iterator" + # most modules have to be listed at the beginning of the line, + # except cachedb(just before iterator), and python (at the beginning, + # or, just before the iterator). + module-config: "dns64 validator iterator" + + # File with trusted keys, kept uptodate using RFC5011 probes, + # initial file like trust-anchor-file, then it stores metadata. + # Use several entries, one per domain name, to track multiple zones. + # + # If you want to perform DNSSEC validation, run unbound-anchor before + # you start unbound (i.e. in the system boot scripts). And enable: + # Please note usage of unbound-anchor root anchor is at your own risk + # and under the terms of our LICENSE (see that file in the source). + # auto-trust-anchor-file: "" + + # trust anchor signaling sends a RFC8145 key tag query after priming. + # trust-anchor-signaling: yes + + # Root key trust anchor sentinel (draft-ietf-dnsop-kskroll-sentinel) + # root-key-sentinel: yes + + # File with DLV trusted keys. Same format as trust-anchor-file. + # There can be only one DLV configured, it is trusted from root down. + # DLV is going to be decommissioned. Please do not use it any more. + # dlv-anchor-file: "dlv.isc.org.key" + + # File with trusted keys for validation. Specify more than one file + # with several entries, one file per entry. + # Zone file format, with DS and DNSKEY entries. + # Note this gets out of date, use auto-trust-anchor-file please. + trust-anchor-file: "/usr/share/dnssec-root/trusted-key.key" + + # Trusted key for validation. DS or DNSKEY. specify the RR on a + # single line, surrounded by "". TTL is ignored. class is IN default. + # Note this gets out of date, use auto-trust-anchor-file please. + # (These examples are from August 2007 and may not be valid anymore). + # trust-anchor: "nlnetlabs.nl. DNSKEY 257 3 5 AQPzzTWMz8qSWIQlfRnPckx2BiVmkVN6LPupO3mbz7FhLSnm26n6iG9N Lby97Ji453aWZY3M5/xJBSOS2vWtco2t8C0+xeO1bc/d6ZTy32DHchpW 6rDH1vp86Ll+ha0tmwyy9QP7y2bVw5zSbFCrefk8qCUBgfHm9bHzMG1U BYtEIQ==" + # trust-anchor: "jelte.nlnetlabs.nl. DS 42860 5 1 14D739EB566D2B1A5E216A0BA4D17FA9B038BE4A" + + # File with trusted keys for validation. Specify more than one file + # with several entries, one file per entry. Like trust-anchor-file + # but has a different file format. Format is BIND-9 style format, + # the trusted-keys { name flag proto algo "key"; }; clauses are read. + # you need external update procedures to track changes in keys. + # trusted-keys-file: "" + + # Ignore chain of trust. Domain is treated as insecure. + # domain-insecure: "example.com" + + # Override the date for validation with a specific fixed date. + # Do not set this unless you are debugging signature inception + # and expiration. "" or "0" turns the feature off. -1 ignores date. + # val-override-date: "" + + # The time to live for bogus data, rrsets and messages. This avoids + # some of the revalidation, until the time interval expires. in secs. + # val-bogus-ttl: 60 + + # The signature inception and expiration dates are allowed to be off + # by 10% of the signature lifetime (expir-incep) from our local clock. + # This leeway is capped with a minimum and a maximum. In seconds. + # val-sig-skew-min: 3600 + # val-sig-skew-max: 86400 + + # Should additional section of secure message also be kept clean of + # unsecure data. Useful to shield the users of this validator from + # potential bogus data in the additional section. All unsigned data + # in the additional section is removed from secure messages. + # val-clean-additional: yes + + # Turn permissive mode on to permit bogus messages. Thus, messages + # for which security checks failed will be returned to clients, + # instead of SERVFAIL. It still performs the security checks, which + # result in interesting log files and possibly the AD bit in + # replies if the message is found secure. The default is off. + # val-permissive-mode: no + + # Ignore the CD flag in incoming queries and refuse them bogus data. + # Enable it if the only clients of unbound are legacy servers (w2008) + # that set CD but cannot validate themselves. + # ignore-cd-flag: no + + # Serve expired responses from cache, with TTL 0 in the response, + # and then attempt to fetch the data afresh. + # serve-expired: no + # + # Limit serving of expired responses to configured seconds after + # expiration. 0 disables the limit. + # serve-expired-ttl: 0 + # + # Set the TTL of expired records to the serve-expired-ttl value after a + # failed attempt to retrieve the record from upstream. This makes sure + # that the expired records will be served as long as there are queries + # for it. + # serve-expired-ttl-reset: no + + # Have the validator log failed validations for your diagnosis. + # 0: off. 1: A line per failed user query. 2: With reason and bad IP. + # val-log-level: 0 + + # It is possible to configure NSEC3 maximum iteration counts per + # keysize. Keep this table very short, as linear search is done. + # A message with an NSEC3 with larger count is marked insecure. + # List in ascending order the keysize and count values. + # val-nsec3-keysize-iterations: "1024 150 2048 500 4096 2500" + + # instruct the auto-trust-anchor-file probing to add anchors after ttl. + # add-holddown: 2592000 # 30 days + + # instruct the auto-trust-anchor-file probing to del anchors after ttl. + # del-holddown: 2592000 # 30 days + + # auto-trust-anchor-file probing removes missing anchors after ttl. + # If the value 0 is given, missing anchors are not removed. + # keep-missing: 31622400 # 366 days + + # debug option that allows very small holddown times for key rollover, + # otherwise the RFC mandates probe intervals must be at least 1 hour. + # permit-small-holddown: no + + # the amount of memory to use for the key cache. + # plain value in bytes or you can append k, m or G. default is "4Mb". + # key-cache-size: 4m + + # the number of slabs to use for the key cache. + # the number of slabs must be a power of 2. + # more slabs reduce lock contention, but fragment memory usage. + # key-cache-slabs: 4 + + # the amount of memory to use for the negative cache (used for DLV). + # plain value in bytes or you can append k, m or G. default is "1Mb". + # neg-cache-size: 1m + + # By default, for a number of zones a small default 'nothing here' + # reply is built-in. Query traffic is thus blocked. If you + # wish to serve such zone you can unblock them by uncommenting one + # of the nodefault statements below. + # You may also have to use domain-insecure: zone to make DNSSEC work, + # unless you have your own trust anchors for this zone. + # local-zone: "localhost." nodefault + # local-zone: "127.in-addr.arpa." nodefault + # local-zone: "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa." nodefault + # local-zone: "onion." nodefault + # local-zone: "test." nodefault + # local-zone: "invalid." nodefault + # local-zone: "10.in-addr.arpa." nodefault + # local-zone: "16.172.in-addr.arpa." nodefault + # local-zone: "17.172.in-addr.arpa." nodefault + # local-zone: "18.172.in-addr.arpa." nodefault + # local-zone: "19.172.in-addr.arpa." nodefault + # local-zone: "20.172.in-addr.arpa." nodefault + # local-zone: "21.172.in-addr.arpa." nodefault + # local-zone: "22.172.in-addr.arpa." nodefault + # local-zone: "23.172.in-addr.arpa." nodefault + # local-zone: "24.172.in-addr.arpa." nodefault + # local-zone: "25.172.in-addr.arpa." nodefault + # local-zone: "26.172.in-addr.arpa." nodefault + # local-zone: "27.172.in-addr.arpa." nodefault + # local-zone: "28.172.in-addr.arpa." nodefault + # local-zone: "29.172.in-addr.arpa." nodefault + # local-zone: "30.172.in-addr.arpa." nodefault + # local-zone: "31.172.in-addr.arpa." nodefault + # local-zone: "168.192.in-addr.arpa." nodefault + # local-zone: "0.in-addr.arpa." nodefault + # local-zone: "254.169.in-addr.arpa." nodefault + # local-zone: "2.0.192.in-addr.arpa." nodefault + # local-zone: "100.51.198.in-addr.arpa." nodefault + # local-zone: "113.0.203.in-addr.arpa." nodefault + # local-zone: "255.255.255.255.in-addr.arpa." nodefault + # local-zone: "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa." nodefault + # local-zone: "d.f.ip6.arpa." nodefault + # local-zone: "8.e.f.ip6.arpa." nodefault + # local-zone: "9.e.f.ip6.arpa." nodefault + # local-zone: "a.e.f.ip6.arpa." nodefault + # local-zone: "b.e.f.ip6.arpa." nodefault + # local-zone: "8.b.d.0.1.0.0.2.ip6.arpa." nodefault + # And for 64.100.in-addr.arpa. to 127.100.in-addr.arpa. + + # If unbound is running service for the local host then it is useful + # to perform lan-wide lookups to the upstream, and unblock the + # long list of local-zones above. If this unbound is a dns server + # for a network of computers, disabled is better and stops information + # leakage of local lan information. + # unblock-lan-zones: no + + # The insecure-lan-zones option disables validation for + # these zones, as if they were all listed as domain-insecure. + # insecure-lan-zones: no + + # a number of locally served zones can be configured. + # local-zone: + # local-data: "" + # o deny serves local data (if any), else, drops queries. + # o refuse serves local data (if any), else, replies with error. + # o static serves local data, else, nxdomain or nodata answer. + # o transparent gives local data, but resolves normally for other names + # o redirect serves the zone data for any subdomain in the zone. + # o nodefault can be used to normally resolve AS112 zones. + # o typetransparent resolves normally for other types and other names + # o inform acts like transparent, but logs client IP address + # o inform_deny drops queries and logs client IP address + # o inform_redirect redirects queries and logs client IP address + # o always_transparent, always_refuse, always_nxdomain, resolve in + # that way but ignore local data for that name + # o noview breaks out of that view towards global local-zones. + # + # defaults are localhost address, reverse for 127.0.0.1 and ::1 + # and nxdomain for AS112 zones. If you configure one of these zones + # the default content is omitted, or you can omit it with 'nodefault'. + # + # If you configure local-data without specifying local-zone, by + # default a transparent local-zone is created for the data. + # + # You can add locally served data with + # local-zone: "local." static + # local-data: "mycomputer.local. IN A 192.0.2.51" + # local-data: 'mytext.local TXT "content of text record"' + # + # You can override certain queries with + # local-data: "adserver.example.com A 127.0.0.1" + # + # You can redirect a domain to a fixed address with + # (this makes example.com, www.example.com, etc, all go to 192.0.2.3) + # local-zone: "example.com" redirect + # local-data: "example.com A 192.0.2.3" + # + # Shorthand to make PTR records, "IPv4 name" or "IPv6 name". + # You can also add PTR records using local-data directly, but then + # you need to do the reverse notation yourself. + # local-data-ptr: "192.0.2.3 www.example.com" + + # tag a localzone with a list of tag names (in "" with spaces between) + # local-zone-tag: "example.com" "tag2 tag3" + + # add a netblock specific override to a localzone, with zone type + # local-zone-override: "example.com" 192.0.2.0/24 refuse + + # service clients over TLS (on the TCP sockets), with plain DNS inside + # the TLS stream. Give the certificate to use and private key. + # default is "" (disabled). requires restart to take effect. + # tls-service-key: "path/to/privatekeyfile.key" + # tls-service-pem: "path/to/publiccertfile.pem" + # tls-port: 853 + + # cipher setting for TLSv1.2 + # tls-ciphers: "DHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256" + # cipher setting for TLSv1.3 + # tls-ciphersuites: "TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_8_SHA256:TLS_AES_128_CCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256" + + # Add the secret file for TLS Session Ticket. + # Secret file must be 80 bytes of random data. + # First key use to encrypt and decrypt TLS session tickets. + # Other keys use to decrypt only. + # requires restart to take effect. + # tls-session-ticket-keys: "path/to/secret_file1" + # tls-session-ticket-keys: "path/to/secret_file2" + + # request upstream over TLS (with plain DNS inside the TLS stream). + # Default is no. Can be turned on and off with unbound-control. + # tls-upstream: no + + # Certificates used to authenticate connections made upstream. + # tls-cert-bundle: "" + + # Add system certs to the cert bundle, from the Windows Cert Store + # tls-win-cert: no + + # Also serve tls on these port numbers (eg. 443, ...), by listing + # tls-additional-port: portno for each of the port numbers. + + # DNS64 prefix. Must be specified when DNS64 is use. + # Enable dns64 in module-config. Used to synthesize IPv6 from IPv4. + dns64-prefix: $DNS64_PREFIX + + # DNS64 ignore AAAA records for these domains and use A instead. + # dns64-ignore-aaaa: "example.com" + + # ratelimit for uncached, new queries, this limits recursion effort. + # ratelimiting is experimental, and may help against randomqueryflood. + # if 0(default) it is disabled, otherwise state qps allowed per zone. + # ratelimit: 0 + + # ratelimits are tracked in a cache, size in bytes of cache (or k,m). + # ratelimit-size: 4m + # ratelimit cache slabs, reduces lock contention if equal to cpucount. + # ratelimit-slabs: 4 + + # 0 blocks when ratelimited, otherwise let 1/xth traffic through + # ratelimit-factor: 10 + + # override the ratelimit for a specific domain name. + # give this setting multiple times to have multiple overrides. + # ratelimit-for-domain: example.com 1000 + # override the ratelimits for all domains below a domain name + # can give this multiple times, the name closest to the zone is used. + # ratelimit-below-domain: com 1000 + + # global query ratelimit for all ip addresses. + # feature is experimental. + # if 0(default) it is disabled, otherwise states qps allowed per ip address + # ip-ratelimit: 0 + + # ip ratelimits are tracked in a cache, size in bytes of cache (or k,m). + # ip-ratelimit-size: 4m + # ip ratelimit cache slabs, reduces lock contention if equal to cpucount. + # ip-ratelimit-slabs: 4 + + # 0 blocks when ip is ratelimited, otherwise let 1/xth traffic through + # ip-ratelimit-factor: 10 + + # Limit the number of connections simultaneous from a netblock + # tcp-connection-limit: 192.0.2.0/24 12 + + # select from the fastest servers this many times out of 1000. 0 means + # the fast server select is disabled. prefetches are not sped up. + # fast-server-permil: 0 + # the number of servers that will be used in the fast server selection. + # fast-server-num: 3 + + # Specific options for ipsecmod. unbound needs to be configured with + # --enable-ipsecmod for these to take effect. + # + # Enable or disable ipsecmod (it still needs to be defined in + # module-config above). Can be used when ipsecmod needs to be + # enabled/disabled via remote-control(below). + # ipsecmod-enabled: yes + # + # Path to executable external hook. It must be defined when ipsecmod is + # listed in module-config (above). + # ipsecmod-hook: "./my_executable" + # + # When enabled unbound will reply with SERVFAIL if the return value of + # the ipsecmod-hook is not 0. + # ipsecmod-strict: no + # + # Maximum time to live (TTL) for cached A/AAAA records with IPSECKEY. + # ipsecmod-max-ttl: 3600 + # + # Reply with A/AAAA even if the relevant IPSECKEY is bogus. Mainly used for + # testing. + # ipsecmod-ignore-bogus: no + # + # Domains for which ipsecmod will be triggered. If not defined (default) + # all domains are treated as being whitelisted. + # ipsecmod-whitelist: "example.com" + # ipsecmod-whitelist: "nlnetlabs.nl" + + +# Python config section. To enable: +# o use --with-pythonmodule to configure before compiling. +# o list python in the module-config string (above) to enable. +# It can be at the start, it gets validated results, or just before +# the iterator and process before DNSSEC validation. +# o and give a python-script to run. +python: + # Script file to load + # python-script: "/ubmodule-tst.py" + +# Remote control config section. +remote-control: + # Enable remote control with unbound-control(8) here. + # set up the keys and certificates with unbound-control-setup. + # control-enable: no + + # what interfaces are listened to for remote control. + # give 0.0.0.0 and ::0 to listen to all interfaces. + # set to an absolute path to use a unix local name pipe, certificates + # are not used for that, so key and cert files need not be present. + # control-interface: 127.0.0.1 + # control-interface: ::1 + + # port number for remote control operations. + # control-port: 8953 + + # for localhost, you can disable use of TLS by setting this to "no" + # For local sockets this option is ignored, and TLS is not used. + # control-use-cert: "yes" + + # unbound server key file. + # server-key-file: "/unbound_server.key" + + # unbound server certificate file. + # server-cert-file: "/unbound_server.pem" + + # unbound-control key file. + # control-key-file: "/unbound_control.key" + + # unbound-control certificate file. + # control-cert-file: "/unbound_control.pem" + +# Stub zones. +# Create entries like below, to make all queries for 'example.com' and +# 'example.org' go to the given list of nameservers. list zero or more +# nameservers by hostname or by ipaddress. If you set stub-prime to yes, +# the list is treated as priming hints (default is no). +# With stub-first yes, it attempts without the stub if it fails. +# Consider adding domain-insecure: name and local-zone: name nodefault +# to the server: section if the stub is a locally served zone. +# stub-zone: +# name: "example.com" +# stub-addr: 192.0.2.68 +# stub-prime: no +# stub-first: no +# stub-tls-upstream: no +# stub-no-cache: no +# stub-zone: +# name: "example.org" +# stub-host: ns.example.com. + +# Forward zones +# Create entries like below, to make all queries for 'example.com' and +# 'example.org' go to the given list of servers. These servers have to handle +# recursion to other nameservers. List zero or more nameservers by hostname +# or by ipaddress. Use an entry with name "." to forward all queries. +# If you enable forward-first, it attempts without the forward if it fails. +# forward-zone: +# name: "example.com" +# forward-addr: 192.0.2.68 +# forward-addr: 192.0.2.73@5355 # forward to port 5355. +# forward-first: no +# forward-tls-upstream: no +# forward-no-cache: no +# forward-zone: +# name: "example.org" +# forward-host: fwd.example.com + +forward-zone: + name: "." +$(generate_forward_addr) + +# Authority zones +# The data for these zones is kept locally, from a file or downloaded. +# The data can be served to downstream clients, or used instead of the +# upstream (which saves a lookup to the upstream). The first example +# has a copy of the root for local usage. The second serves example.org +# authoritatively. zonefile: reads from file (and writes to it if you also +# download it), master: fetches with AXFR and IXFR, or url to zonefile. +# With allow-notify: you can give additional (apart from masters) sources of +# notifies. +# auth-zone: +# name: "." +# master: 199.9.14.201 # b.root-servers.net +# master: 192.33.4.12 # c.root-servers.net +# master: 199.7.91.13 # d.root-servers.net +# master: 192.5.5.241 # f.root-servers.net +# master: 192.112.36.4 # g.root-servers.net +# master: 193.0.14.129 # k.root-servers.net +# master: 192.0.47.132 # xfr.cjr.dns.icann.org +# master: 192.0.32.132 # xfr.lax.dns.icann.org +# master: 2001:500:200::b # b.root-servers.net +# master: 2001:500:2::c # c.root-servers.net +# master: 2001:500:2d::d # d.root-servers.net +# master: 2001:500:2f::f # f.root-servers.net +# master: 2001:500:12::d0d # g.root-servers.net +# master: 2001:7fd::1 # k.root-servers.net +# master: 2620:0:2830:202::132 # xfr.cjr.dns.icann.org +# master: 2620:0:2d0:202::132 # xfr.lax.dns.icann.org +# fallback-enabled: yes +# for-downstream: no +# for-upstream: yes +# auth-zone: +# name: "example.org" +# for-downstream: yes +# for-upstream: yes +# zonefile: "example.org.zone" + +# Views +# Create named views. Name must be unique. Map views to requests using +# the access-control-view option. Views can contain zero or more local-zone +# and local-data options. Options from matching views will override global +# options. Global options will be used if no matching view is found. +# With view-first yes, it will try to answer using the global local-zone and +# local-data elements if there is no view specific match. +# view: +# name: "viewname" +# local-zone: "example.com" redirect +# local-data: "example.com A 192.0.2.3" +# local-data-ptr: "192.0.2.3 www.example.com" +# view-first: no +# view: +# name: "anotherview" +# local-zone: "example.com" refuse + +# DNSCrypt +# Caveats: +# 1. the keys/certs cannot be produced by unbound. You can use dnscrypt-wrapper +# for this: https://github.com/cofyc/dnscrypt-wrapper/blob/master/README.md#usage +# 2. dnscrypt channel attaches to an interface. you MUST set interfaces to +# listen on `dnscrypt-port` with the follo0wing snippet: +# server: +# interface: 0.0.0.0@443 +# interface: ::0@443 +# +# Finally, `dnscrypt` config has its own section. +# dnscrypt: +# dnscrypt-enable: yes +# dnscrypt-port: 443 +# dnscrypt-provider: 2.dnscrypt-cert.example.com. +# dnscrypt-secret-key: /path/unbound-conf/keys1/1.key +# dnscrypt-secret-key: /path/unbound-conf/keys2/1.key +# dnscrypt-provider-cert: /path/unbound-conf/keys1/1.cert +# dnscrypt-provider-cert: /path/unbound-conf/keys2/1.cert + +# CacheDB +# Enable external backend DB as auxiliary cache. Specify the backend name +# (default is "testframe", which has no use other than for debugging and +# testing) and backend-specific options. The 'cachedb' module must be +# included in module-config, just before the iterator module. +# cachedb: +# backend: "testframe" +# # secret seed string to calculate hashed keys +# secret-seed: "default" +# +# # For "redis" backend: +# # redis server's IP address or host name +# redis-server-host: 127.0.0.1 +# # redis server's TCP port +# redis-server-port: 6379 +# # timeout (in ms) for communication with the redis server +# redis-timeout: 100 +EOF diff --git a/type/__ungleich_unbound/man.rst b/type/__ungleich_unbound/man.rst new file mode 100644 index 0000000..417b343 --- /dev/null +++ b/type/__ungleich_unbound/man.rst @@ -0,0 +1,74 @@ +cdist-type__ungleich_unbound(7) +=============================== + +NAME +---- +cdist-type__ungleich_unbound - unbound server deployment for ungleich + + +DESCRIPTION +----------- +This unbound (dns resolver and cache) deployment provides DNS64 and fetch +answers from specified upstrean DNS server. This is a singleton type. + +REQUIRED PARAMETERS +------------------- +dns64_prefix + IPv6 prefix used for DNS64. + +forward_addr + DNS servers used to lookup names, can be provided multiple times. It can be + either an IPv4 or IPv6 address but no domain name. + +OPTIONAL PARAMETERS +------------------- +interface + Interface to listen on, can be provided multiple times. Defaults to + '127.0.0.1' and '::1'. + +access_control + Controls which clients are allowed queries to the unbound service (everything + but localhost is refused by default), can be provided multiple times. The + format is described in unbound.conf(5). + +BOOLEAN PARAMETERS +------------------ +disable-ip4 + Do not answer or issue queries over IPv4. Cannot be used alongside the + `--disable-ip6` flag. + +disable-ip6 + Do not answer or issue queries over IPv6. Cannot be used alongside the + `--disable-ip4` flag. + +EXAMPLES +-------- + +.. code-block:: sh + + __ungleich_unbound \ + --interface '::0' \ + --dns64_prefix '2a0a:e5c0:2:10::/96' \ + --forward_addr '2a0a:e5c0:2:1::5' \ + --forward_addr '2a0a:e5c0:2:1::6' \ + --access_control '::0/0 deny' \ + --access_control '2a0a:e5c0::/29 allow' \ + --access_control '2a09:2940::/29 allow' \ + --ip6 + +SEE ALSO +-------- +- `unbound.conf(5) `_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__ungleich_unbound/manifest b/type/__ungleich_unbound/manifest new file mode 100755 index 0000000..525a981 --- /dev/null +++ b/type/__ungleich_unbound/manifest @@ -0,0 +1,79 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + alpine) + __package unbound --state present + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +# Required parameters: +export DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") +export FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") + +# Optional parameters: +if [ -f "$__object/parameter/interface" ]; then + export INTERFACES=$(cat "$__object/parameter/interface") +fi + +if [ -f "$__object/parameter/access_control" ]; then + export ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") +fi + +# Boolean parameters: +if [ -f "$__object/parameter/disable_ip4" ] && \ + [ -f "$__object/parameter/disable_ip6" ]; then + echo "--disable-ip4 and --disable-ip6 cannot be used at the same time." >&2 + exit 1 +fi + +if [ -f "$__object/parameter/disable_ip4" ]; then + export DO_IP4='no' +else + export DO_IP4='yes' +fi + +if [ -f "$__object/parameter/disable_ip6" ]; then + export DO_IP6='no' +else + export DO_IP6='yes' +fi + +# Generate and deploy configuration files. +source_file="$__object/files/unbound.conf" +target_file="/etc/unbound/unbound.conf" + +mkdir -p "$__object/files" +"$__type/files/unbound.conf.sh" > $source_file +require="__package/unbound" __file $target_file \ + --source $source_file \ + --owner root \ + --mode 644 + +# Restart unbound server after reconfiguration. +require="__file/$target_file" __service unbound --action restart diff --git a/type/__ungleich_unbound/parameter/boolean b/type/__ungleich_unbound/parameter/boolean new file mode 100644 index 0000000..a2d5910 --- /dev/null +++ b/type/__ungleich_unbound/parameter/boolean @@ -0,0 +1,2 @@ +disable_ip6 +disable_ip4 diff --git a/type/__ungleich_unbound/parameter/optional_multiple b/type/__ungleich_unbound/parameter/optional_multiple new file mode 100644 index 0000000..ecacd1d --- /dev/null +++ b/type/__ungleich_unbound/parameter/optional_multiple @@ -0,0 +1,2 @@ +access_control +interface diff --git a/type/__ungleich_unbound/parameter/required b/type/__ungleich_unbound/parameter/required new file mode 100644 index 0000000..4ad232f --- /dev/null +++ b/type/__ungleich_unbound/parameter/required @@ -0,0 +1 @@ +dns64_prefix diff --git a/type/__ungleich_unbound/parameter/required_multiple b/type/__ungleich_unbound/parameter/required_multiple new file mode 100644 index 0000000..776abf5 --- /dev/null +++ b/type/__ungleich_unbound/parameter/required_multiple @@ -0,0 +1 @@ +forward_addr diff --git a/type/__ungleich_unbound/singleton b/type/__ungleich_unbound/singleton new file mode 100644 index 0000000..e69de29 From 6b38b248e2227dc29fca104db29f2656c4310202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 14:59:42 +0200 Subject: [PATCH 022/332] Rename __ungleich_unbound into __unbound --- type/{__ungleich_unbound => __unbound}/files/unbound.conf.sh | 0 type/{__ungleich_unbound => __unbound}/man.rst | 2 +- type/{__ungleich_unbound => __unbound}/manifest | 0 type/{__ungleich_unbound => __unbound}/parameter/boolean | 0 .../parameter/optional_multiple | 0 type/{__ungleich_unbound => __unbound}/parameter/required | 0 .../parameter/required_multiple | 0 type/{__ungleich_unbound => __unbound}/singleton | 0 8 files changed, 1 insertion(+), 1 deletion(-) rename type/{__ungleich_unbound => __unbound}/files/unbound.conf.sh (100%) rename type/{__ungleich_unbound => __unbound}/man.rst (98%) rename type/{__ungleich_unbound => __unbound}/manifest (100%) rename type/{__ungleich_unbound => __unbound}/parameter/boolean (100%) rename type/{__ungleich_unbound => __unbound}/parameter/optional_multiple (100%) rename type/{__ungleich_unbound => __unbound}/parameter/required (100%) rename type/{__ungleich_unbound => __unbound}/parameter/required_multiple (100%) rename type/{__ungleich_unbound => __unbound}/singleton (100%) diff --git a/type/__ungleich_unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh similarity index 100% rename from type/__ungleich_unbound/files/unbound.conf.sh rename to type/__unbound/files/unbound.conf.sh diff --git a/type/__ungleich_unbound/man.rst b/type/__unbound/man.rst similarity index 98% rename from type/__ungleich_unbound/man.rst rename to type/__unbound/man.rst index 417b343..6d113b0 100644 --- a/type/__ungleich_unbound/man.rst +++ b/type/__unbound/man.rst @@ -1,4 +1,4 @@ -cdist-type__ungleich_unbound(7) +cdist-type__unbound(7) =============================== NAME diff --git a/type/__ungleich_unbound/manifest b/type/__unbound/manifest similarity index 100% rename from type/__ungleich_unbound/manifest rename to type/__unbound/manifest diff --git a/type/__ungleich_unbound/parameter/boolean b/type/__unbound/parameter/boolean similarity index 100% rename from type/__ungleich_unbound/parameter/boolean rename to type/__unbound/parameter/boolean diff --git a/type/__ungleich_unbound/parameter/optional_multiple b/type/__unbound/parameter/optional_multiple similarity index 100% rename from type/__ungleich_unbound/parameter/optional_multiple rename to type/__unbound/parameter/optional_multiple diff --git a/type/__ungleich_unbound/parameter/required b/type/__unbound/parameter/required similarity index 100% rename from type/__ungleich_unbound/parameter/required rename to type/__unbound/parameter/required diff --git a/type/__ungleich_unbound/parameter/required_multiple b/type/__unbound/parameter/required_multiple similarity index 100% rename from type/__ungleich_unbound/parameter/required_multiple rename to type/__unbound/parameter/required_multiple diff --git a/type/__ungleich_unbound/singleton b/type/__unbound/singleton similarity index 100% rename from type/__ungleich_unbound/singleton rename to type/__unbound/singleton From 3d3a3ef8024a5dc201253bfa5a67b1508510e0d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 15:08:04 +0200 Subject: [PATCH 023/332] Fix shellcheck warnings in newly imported __unbound type --- type/__unbound/files/unbound.conf.sh | 6 +++--- type/__unbound/manifest | 18 +++++++++++------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index e713843..effea65 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -7,7 +7,7 @@ generate_interface() { } generate_access_control() { - echo "$ACCESS_CONTROLS" | while read entry; do + echo "$ACCESS_CONTROLS" | while read -r entry; do echo " access-control: $entry" done } @@ -983,12 +983,12 @@ $(generate_forward_addr) # 1. the keys/certs cannot be produced by unbound. You can use dnscrypt-wrapper # for this: https://github.com/cofyc/dnscrypt-wrapper/blob/master/README.md#usage # 2. dnscrypt channel attaches to an interface. you MUST set interfaces to -# listen on `dnscrypt-port` with the follo0wing snippet: +# listen on 'dnscrypt-port' with the follo0wing snippet: # server: # interface: 0.0.0.0@443 # interface: ::0@443 # -# Finally, `dnscrypt` config has its own section. +# Finally, 'dnscrypt' config has its own section. # dnscrypt: # dnscrypt-enable: yes # dnscrypt-port: 443 diff --git a/type/__unbound/manifest b/type/__unbound/manifest index 525a981..a25cdec 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -33,16 +33,20 @@ case "$os" in esac # Required parameters: -export DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") -export FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") +DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") +export DNS64_PREFIX +FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") +export FORWARD_ADDRS # Optional parameters: if [ -f "$__object/parameter/interface" ]; then - export INTERFACES=$(cat "$__object/parameter/interface") + INTERFACES=$(cat "$__object/parameter/interface") + export INTERFACES fi if [ -f "$__object/parameter/access_control" ]; then - export ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") + ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") + export ACCESS_CONTROLS fi # Boolean parameters: @@ -69,9 +73,9 @@ source_file="$__object/files/unbound.conf" target_file="/etc/unbound/unbound.conf" mkdir -p "$__object/files" -"$__type/files/unbound.conf.sh" > $source_file -require="__package/unbound" __file $target_file \ - --source $source_file \ +"$__type/files/unbound.conf.sh" > "$source_file" +require="__package/unbound" __file "$target_file" \ + --source "$source_file" \ --owner root \ --mode 644 From 89f0dc90ead21d338c271cb283cd10d1ebe57ce1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 3 Jun 2020 15:09:07 +0200 Subject: [PATCH 024/332] Add changelog entry for __unbound type --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2ebad0..a61c82d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ # cdist-contrib changes +* 2020-06-03: New type: __unbound (Timothée Floure) * 2020-04-28: New type: __find_exec (Ander Punnar) From de4508cb0695cf0f7a6561ac7d8b92add1bc313d Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 6 Jun 2020 21:45:40 +0200 Subject: [PATCH 025/332] Mark __dma_auth and __mail_alias as nonparallel Both types modify a single file, so they shouldn't be run at the same time. --- type/__dma_auth/nonparallel | 0 type/__mail_alias/nonparallel | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 type/__dma_auth/nonparallel create mode 100644 type/__mail_alias/nonparallel diff --git a/type/__dma_auth/nonparallel b/type/__dma_auth/nonparallel new file mode 100644 index 0000000..e69de29 diff --git a/type/__mail_alias/nonparallel b/type/__mail_alias/nonparallel new file mode 100644 index 0000000..e69de29 From 7b9ffb4a413693556a6c19258ba3a17d1614613f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:46:18 +0200 Subject: [PATCH 026/332] __unbound: wire remote control configuration --- type/__unbound/files/unbound.conf.sh | 12 ++++++------ type/__unbound/gencode-remote | 8 ++++++++ type/__unbound/man.rst | 6 ++++++ type/__unbound/manifest | 17 +++++++++++++++++ type/__unbound/parameter/boolean | 1 + type/__unbound/parameter/default/rc_interface | 1 + type/__unbound/parameter/optional | 1 + 7 files changed, 40 insertions(+), 6 deletions(-) create mode 100755 type/__unbound/gencode-remote create mode 100644 type/__unbound/parameter/default/rc_interface create mode 100644 type/__unbound/parameter/optional diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index effea65..46d2824 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -857,14 +857,14 @@ python: remote-control: # Enable remote control with unbound-control(8) here. # set up the keys and certificates with unbound-control-setup. - # control-enable: no + control-enable: $RC_ENABLE # what interfaces are listened to for remote control. # give 0.0.0.0 and ::0 to listen to all interfaces. # set to an absolute path to use a unix local name pipe, certificates # are not used for that, so key and cert files need not be present. # control-interface: 127.0.0.1 - # control-interface: ::1 + control-interface: $RC_INTERFACE # port number for remote control operations. # control-port: 8953 @@ -874,16 +874,16 @@ remote-control: # control-use-cert: "yes" # unbound server key file. - # server-key-file: "/unbound_server.key" + server-key-file: "$RC_SERVER_KEY_FILE" # unbound server certificate file. - # server-cert-file: "/unbound_server.pem" + server-cert-file: "$RC_SERVER_CERT_FILE" # unbound-control key file. - # control-key-file: "/unbound_control.key" + control-key-file: "$RC_CONTROL_KEY_FILE" # unbound-control certificate file. - # control-cert-file: "/unbound_control.pem" + control-cert-file: "$RC_CONTROL_CERT_FILE" # Stub zones. # Create entries like below, to make all queries for 'example.com' and diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote new file mode 100755 index 0000000..5d2bb4a --- /dev/null +++ b/type/__unbound/gencode-remote @@ -0,0 +1,8 @@ +#!/bin/sh + +UNBOUND_CERTS_DIR=/etc/unbound + +if [ -f "$__object/parameter/enable_rc" ]; then + echo "unbound-control-setup -d $UNBOUND_CERTS_DIR" + echo "chown unbound:unbound $UNBOUND_CERTS_DIR/*.pem $UNBOUND_CERTS_DIR/*.key" +fi diff --git a/type/__unbound/man.rst b/type/__unbound/man.rst index 6d113b0..679e601 100644 --- a/type/__unbound/man.rst +++ b/type/__unbound/man.rst @@ -31,6 +31,9 @@ access_control but localhost is refused by default), can be provided multiple times. The format is described in unbound.conf(5). +rc_interface + Address or path to socket used for remote control (see `--enable_control`. Defaults to `127.0.0.1`). + BOOLEAN PARAMETERS ------------------ disable-ip4 @@ -41,6 +44,9 @@ disable-ip6 Do not answer or issue queries over IPv6. Cannot be used alongside the `--disable-ip4` flag. +enable_rc + Enable remote control (see `unbound-control(8)`). + EXAMPLES -------- diff --git a/type/__unbound/manifest b/type/__unbound/manifest index a25cdec..d85ad66 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -49,6 +49,11 @@ if [ -f "$__object/parameter/access_control" ]; then export ACCESS_CONTROLS fi +if [ -f "$__object/parameter/rc_interface" ]; then + RC_INTERFACE=$(cat "$__object/parameter/rc_interface") + export RC_INTERFACE +fi + # Boolean parameters: if [ -f "$__object/parameter/disable_ip4" ] && \ [ -f "$__object/parameter/disable_ip6" ]; then @@ -68,6 +73,18 @@ else export DO_IP6='yes' fi +if [ -f "$__object/parameter/enable_rc" ]; then + export RC_ENABLE='yes' +else + export RC_ENABLE='no' +fi + +# Certs for remote control: +export RC_SERVER_KEY_FILE='/etc/unbound/unbound_server.key' +export RC_SERVER_CERT_FILE='/etc/unbound/unbound_server.pem' +export RC_CONTROL_KEY_FILE='/etc/unbound/unbound_control.key' +export RC_CONTROL_CERT_FILE='/etc/unbound/unbound_control.pem' + # Generate and deploy configuration files. source_file="$__object/files/unbound.conf" target_file="/etc/unbound/unbound.conf" diff --git a/type/__unbound/parameter/boolean b/type/__unbound/parameter/boolean index a2d5910..b6e53a1 100644 --- a/type/__unbound/parameter/boolean +++ b/type/__unbound/parameter/boolean @@ -1,2 +1,3 @@ disable_ip6 disable_ip4 +enable_rc diff --git a/type/__unbound/parameter/default/rc_interface b/type/__unbound/parameter/default/rc_interface new file mode 100644 index 0000000..7b9ad53 --- /dev/null +++ b/type/__unbound/parameter/default/rc_interface @@ -0,0 +1 @@ +127.0.0.1 diff --git a/type/__unbound/parameter/optional b/type/__unbound/parameter/optional new file mode 100644 index 0000000..0826b6e --- /dev/null +++ b/type/__unbound/parameter/optional @@ -0,0 +1 @@ +rc_interface From 1e45d85d4d3d171b211e957e099f0e1c1580fe57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:53:56 +0200 Subject: [PATCH 027/332] Add new __unbound_exporter type --- type/__unbound_exporter/files/openrc-service | 12 ++++ type/__unbound_exporter/gencode-remote | 46 ++++++++++++++ type/__unbound_exporter/man.rst | 63 ++++++++++++++++++++ type/__unbound_exporter/manifest | 44 ++++++++++++++ type/__unbound_exporter/parameter/required | 1 + type/__unbound_exporter/singleton | 0 6 files changed, 166 insertions(+) create mode 100644 type/__unbound_exporter/files/openrc-service create mode 100755 type/__unbound_exporter/gencode-remote create mode 100644 type/__unbound_exporter/man.rst create mode 100755 type/__unbound_exporter/manifest create mode 100644 type/__unbound_exporter/parameter/required create mode 100644 type/__unbound_exporter/singleton diff --git a/type/__unbound_exporter/files/openrc-service b/type/__unbound_exporter/files/openrc-service new file mode 100644 index 0000000..6caed5e --- /dev/null +++ b/type/__unbound_exporter/files/openrc-service @@ -0,0 +1,12 @@ +#!/sbin/openrc-run + +name=$RC_SVCNAME +command="/usr/local/bin/unbound_exporter" +command_args="" +command_user="unbound" +command_background="yes" +pidfile="/var/run/$RC_SVCNAME.pid" + +depend() { + need unbound +} diff --git a/type/__unbound_exporter/gencode-remote b/type/__unbound_exporter/gencode-remote new file mode 100755 index 0000000..ef7ef9b --- /dev/null +++ b/type/__unbound_exporter/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +upstream=https://github.com/wish/unbound_exporter/archive +version=$(cat $__object/parameter/version) +release="unbound_exporter-$version" + +cat << EOF +if command -v unbound_exporter +then + # already installed - ignoring. + echo "Nothing to do -" +else + # Initialize working directory + workdir=\$(mktemp -d) + cd \$workdir + + # Download and extract sources for requested release. + curl -L $upstream/v$version.tar.gz --output $release.tar.gz + tar xf $release.tar.gz + + # Build and install binary. + cd $release + go build + install -m755 unbound_exporter /usr/local/bin/ + + # Clean up! + rm -r \$workdir +fi +EOF diff --git a/type/__unbound_exporter/man.rst b/type/__unbound_exporter/man.rst new file mode 100644 index 0000000..934bdd7 --- /dev/null +++ b/type/__unbound_exporter/man.rst @@ -0,0 +1,63 @@ +cdist-type__unbound_exporter(7) +=============================== + +NAME +---- +cdist-type__unbound_exporter - A prometheus exporter for unbound + + +DESCRIPTION +----------- +Simple Prometheus metrics exporter for the Unbound DNS +resolver. It leverages the unbound remote control endpoint +and exposes metrics on port 9167. + + +REQUIRED PARAMETERS +------------------- +version + unbound_exporter release to be used. + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __unbound \ + --interface '::0' \ + --forward_addr '2a0a:e5c0:2:1::5' \ + --forward_addr '2a0a:e5c0:2:1::6' \ + --access_control '::0/0 deny' \ + --access_control '2a0a:e5c0::/29 allow' \ + --access_control '2a09:2940::/29 allow' \ + --disable_ip4 \ + --enable_rc \ + --rc_interface '::1' + + __unbound_exporter --version 0.1.3 + +SEE ALSO +-------- +:strong:`cdist-type__unbound(7)` + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__unbound_exporter/manifest b/type/__unbound_exporter/manifest new file mode 100755 index 0000000..9f71834 --- /dev/null +++ b/type/__unbound_exporter/manifest @@ -0,0 +1,44 @@ +#!/bin/sh -e +# +# 2020 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + alpine) + # Used in gencode-remote. + __package curl + __package tar + __package go + __package libc-dev + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +__file /etc/init.d/unbound_exporter \ + --source "$__type/files/openrc-service" \ + --mode 755 + +require="__file/etc/init.d/unbound_exporter" __service unbound_exporter --action start +require="__file/etc/init.d/unbound_exporter" __start_on_boot unbound_exporter diff --git a/type/__unbound_exporter/parameter/required b/type/__unbound_exporter/parameter/required new file mode 100644 index 0000000..088eda4 --- /dev/null +++ b/type/__unbound_exporter/parameter/required @@ -0,0 +1 @@ +version diff --git a/type/__unbound_exporter/singleton b/type/__unbound_exporter/singleton new file mode 100644 index 0000000..e69de29 From d14011c50b8dbfbddf1adfab9e7823e462ef042c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:55:34 +0200 Subject: [PATCH 028/332] Sync CHANGELOG with __unbound* changes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a61c82d..df26f7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # cdist-contrib changes +* 2020-06-07: New type: __unbound_exporter (Timothée Floure) +* 2020-06-07: Extended type: wire remote control configuration for __unbond (Timothée Floure) * 2020-06-03: New type: __unbound (Timothée Floure) * 2020-04-28: New type: __find_exec (Ander Punnar) From 17f6a4dd471193369e17b0287817fc9cd43895b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:57:51 +0200 Subject: [PATCH 029/332] Only shellcheck .sh files in type/*/files/ --- scripts/run-shellcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-shellcheck.sh b/scripts/run-shellcheck.sh index 769f853..475f426 100755 --- a/scripts/run-shellcheck.sh +++ b/scripts/run-shellcheck.sh @@ -15,7 +15,7 @@ check () { } check -path "*/explorer/*" -check -path "*/files/*" +check -path "*/files/*.sh" check -name manifest check -name gencode-local check -name gencode-remote From bffc38ad52a54d7c80354fac46bf9f556fb4e0c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 08:58:14 +0200 Subject: [PATCH 030/332] __unbound_exporter: make shellcheck happy again --- type/__unbound_exporter/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__unbound_exporter/gencode-remote b/type/__unbound_exporter/gencode-remote index ef7ef9b..fcd85fb 100755 --- a/type/__unbound_exporter/gencode-remote +++ b/type/__unbound_exporter/gencode-remote @@ -18,7 +18,7 @@ # along with cdist. If not, see . upstream=https://github.com/wish/unbound_exporter/archive -version=$(cat $__object/parameter/version) +version=$(cat "$__object/parameter/version") release="unbound_exporter-$version" cat << EOF From 4ff703e6aa74e18b679302e4b02cd2b9fd41fe6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 7 Jun 2020 10:34:35 +0200 Subject: [PATCH 031/332] Fine tuning prometheus exporter for unbound --- type/__unbound/gencode-remote | 8 ++++++++ type/__unbound/manifest | 3 --- type/__unbound_exporter/manifest | 1 + 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote index 5d2bb4a..d12a395 100755 --- a/type/__unbound/gencode-remote +++ b/type/__unbound/gencode-remote @@ -6,3 +6,11 @@ if [ -f "$__object/parameter/enable_rc" ]; then echo "unbound-control-setup -d $UNBOUND_CERTS_DIR" echo "chown unbound:unbound $UNBOUND_CERTS_DIR/*.pem $UNBOUND_CERTS_DIR/*.key" fi + +cat << EOF +if pgrep unbound; then + service unbound reload +else + service unbounb start +fi +EOF diff --git a/type/__unbound/manifest b/type/__unbound/manifest index d85ad66..346f30c 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -95,6 +95,3 @@ require="__package/unbound" __file "$target_file" \ --source "$source_file" \ --owner root \ --mode 644 - -# Restart unbound server after reconfiguration. -require="__file/$target_file" __service unbound --action restart diff --git a/type/__unbound_exporter/manifest b/type/__unbound_exporter/manifest index 9f71834..3602e47 100755 --- a/type/__unbound_exporter/manifest +++ b/type/__unbound_exporter/manifest @@ -26,6 +26,7 @@ case "$os" in # Used in gencode-remote. __package curl __package tar + __package openssl __package go __package libc-dev ;; From 0cd19b3a5dc71fa3fed263fe14eb05aafa58b0aa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Jun 2020 14:44:54 +0200 Subject: [PATCH 032/332] [type/__dma] Use "smarthost" spelling to be consistent with DMA --- type/__dma/gencode-remote | 8 ++++---- type/__dma/man.rst | 14 ++++++-------- type/__dma/parameter/required | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 1b00f04..e2bb405 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -22,9 +22,9 @@ fi # Generate config conf_should=$( - if test -s "${__object}/parameter/smart-host" + if test -s "${__object}/parameter/smarthost" then - printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smart-host")" + printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smarthost")" fi printf 'MAILNAME %s\n' "${mailname}" @@ -85,8 +85,8 @@ conf_should=$( if test -f "${__object}/parameter/null-client" then - test -s "${__object}/parameter/smart-host" || { - echo '--null-client requires a --smart-host to be defined' >&2 + test -s "${__object}/parameter/smarthost" || { + echo '--null-client requires a --smarthost to be defined' >&2 exit 1 } diff --git a/type/__dma/man.rst b/type/__dma/man.rst index a8ba546..cbc1c0c 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -13,15 +13,15 @@ mails from locally installed Mail User Agents (MUA) and deliver the mails to a remote destination. Remote delivery happens over TLS to one or more mailboxes that are local to the -email server configured in the `smart-host` parameter. +mail server configured in the ``smarthost`` parameter. REQUIRED PARAMETERS ------------------- -smart-host - The email server used to send email. +smarthost + The mail server used to send email. It must be configured to act as a relay for the host being configured by - this type so that mail can be sent to users non-local to the smart-host. + this type so that mail can be sent to users non-local to the smarthost. BOOLEAN PARAMETERS @@ -33,7 +33,7 @@ full-bounce not just the headers. null-client Enable to bypass aliases and local delivery, and instead forward all mails - to the defined `--smart-host`. + to the defined ``--smarthost``. send-test-email If present, after setup this type will send an email to root, to allow you to easily test your setup. @@ -78,9 +78,7 @@ EXAMPLES .. code-block:: sh - __dma \ - --smart-host mx1.domain.tld \ - --send-test-email + __dma --smarthost mx1.domain.tld --send-test-email SEE ALSO diff --git a/type/__dma/parameter/required b/type/__dma/parameter/required index 262568f..0753fb6 100644 --- a/type/__dma/parameter/required +++ b/type/__dma/parameter/required @@ -1 +1 @@ -smart-host +smarthost From 45b10f3e098f3e06dccdc2483c36e56527e0b9a1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Jun 2020 14:51:11 +0200 Subject: [PATCH 033/332] [type/__dma] Update parameters to match config names in DMA --- type/__dma/gencode-remote | 6 +++--- type/__dma/parameter/boolean | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index e2bb405..e4760d8 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -78,15 +78,15 @@ conf_should=$( echo 'DEFER' fi - if test -f "${__object}/parameter/full-bounce" + if test -f "${__object}/parameter/fullbounce" then echo 'FULLBOUNCE' fi - if test -f "${__object}/parameter/null-client" + if test -f "${__object}/parameter/nullclient" then test -s "${__object}/parameter/smarthost" || { - echo '--null-client requires a --smarthost to be defined' >&2 + echo '--nullclient requires a --smarthost to be defined' >&2 exit 1 } diff --git a/type/__dma/parameter/boolean b/type/__dma/parameter/boolean index ede7dda..523bb97 100644 --- a/type/__dma/parameter/boolean +++ b/type/__dma/parameter/boolean @@ -1,4 +1,4 @@ defer -full-bounce -null-client -send-test-email +fullbounce +nullclient +send-test-mail From 67b989a717d1c2c817b99220712bab7e78a679e3 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Jun 2020 20:53:01 +0200 Subject: [PATCH 034/332] [type/__dma_auth] Simplify code and add more comments --- type/__dma_auth/explorer/{authusers => state} | 51 ++++++++--- type/__dma_auth/gencode-remote | 84 ++++++++++++------- 2 files changed, 91 insertions(+), 44 deletions(-) rename type/__dma_auth/explorer/{authusers => state} (65%) diff --git a/type/__dma_auth/explorer/authusers b/type/__dma_auth/explorer/state similarity index 65% rename from type/__dma_auth/explorer/authusers rename to type/__dma_auth/explorer/state index c76667b..18e13ce 100755 --- a/type/__dma_auth/explorer/authusers +++ b/type/__dma_auth/explorer/state @@ -31,27 +31,54 @@ else fi awk -F'\n' -v server="${server}" ' +function getvalue(path) { + getline < path + close(path) + return $0 +} + BEGIN { DP = "[: \t]" # copied from dma/conf.c + + parameter_dir = ENVIRON["__object"] "/parameter/" + + host_param = getvalue(parameter_dir "server") + if (!host_param) host_param = ENVIRON["__object_id"] + login_param = getvalue(parameter_dir "login") + passwd_param = getvalue(parameter_dir "password") + + state = "absent" } -# skip comments and empty lines -/^#/ || /^$/ { next } +/^#/ || /^$/ { + # skip comments and empty lines + next +} { + # parse line + login = substr($0, 1, index($0, "|") - 1) + if (!login) { login = $0 } # if no "|" found + host = substr($0, length(login) + 2) + if (match(host, DP)) { + passwd = substr(host, RSTART) host = substr(host, 1, RSTART - 1) - endpos = length(login) + RSTART - } else endpos = length + } else { + passwd = "" + } } -host == server { print endpos, $0 } -' "${auth_conf}" \ -| while read -r pos line - do - printf '%s:%s\n' \ - "$(printf '%s' "$line" | cut -c $((-pos)))" \ - "$(printf '%s' "$line" | cut -c $((pos+2))- | cksum | cut -d' ' -f1)" - done +host == host_param && login == login_param { + if (passwd == passwd_param) + state = "present" + else + state = "different_password" +} + +END { + print state +} +' "${auth_conf}" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index b6b7f63..262a17a 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,6 +18,7 @@ # along with cdist. If not, see . # +state_is=$(cat "${__object}/explorer/state") state_should=$(cat "${__object}/parameter/state") if test -f "${__object}/parameter/server" @@ -28,33 +29,27 @@ else fi login=$(cat "${__object}/parameter/login") +if test "${state_is}" = "${state_should}" +then + # state is as it should + exit 0 +fi + case $state_should in (present) - line_should=$(printf '%s|%s:%s\n' \ - "${login}" "${server}" \ - "$(cksum "${__object}/parameter/password" | cut -d' ' -f1)") - if grep -qxF "${line_should}" "${__object}/explorer/authusers" - then - # correct line already present -> nothing to do - exit 0 - fi - test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1; } mode=1 - if test -s "${__object}/explorer/authusers" + if test "${state_is}" = 'absent' then - printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" - else printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + else + printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" fi ;; (absent) - # no matching logins present -> nothing to do - test -s "${__object}/explorer/authusers" || exit 0 - mode=0 printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" @@ -67,16 +62,14 @@ esac auth_conf=$(cat "${__object}/explorer/auth_conf") -if test -z "${auth_conf}" -then +test -n "${auth_conf}" || { echo 'Cannot determine path of dma auth.conf' >&2 exit 1 -fi +} + cat < drop all lines for this host + next + } + } } +# leave other lines alone { print } END { if (mode && !written) { - printf "%s|%s:%s\n", ENVIRON["login"], ENVIRON["server"], getpw() + # append line if no match to replace was found + print_should() } } ' <"${auth_conf}" >"${auth_conf}.tmp" \ From 96fcccf5294e1ed381096b731207674807ce6222 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 9 Jun 2020 21:39:28 +0200 Subject: [PATCH 035/332] [type/__mail_alias] Improve documentation --- type/__mail_alias/explorer/aliases | 43 ++++++++++++------ type/__mail_alias/explorer/aliases_file | 2 +- type/__mail_alias/gencode-remote | 59 ++++++++++++++++--------- 3 files changed, 69 insertions(+), 35 deletions(-) diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index 66940d5..0a8da94 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -32,26 +32,41 @@ function print_aliases(aliases, matches) { } /^#/ { - # comment - select = 0; cont = 0; next -} - -{ - cont = ($0 ~ /\\$/) - if (cont) sub(/[ \t]*\\$/, "", $0) -} - -/^[[:blank:]]/ || cont { - # continuation line - if (select) print_aliases($0) + # comment line (ignore) + select = 0; cont = 0 # comments terminate alias lists and continuations next } -$1 == ENVIRON["__object_id"] { +/^[ \t]/ || cont { + # continuation line (either the previous line ended in a backslash or the + # line starts with whitespace) + + if (select) + print_aliases($0) +} + +{ + # detect if the line is a line to be continued (ends with a backslash) + cont = ($0 ~ /\\$/) + + # if it is, we drop the backslash from the line and skip to next line + # (the contents have been printed above if they should) + if (cont) { + sub(/[ \t]*\\$/, "", $0) + next + } +} + +$1 == ENVIRON["__object_id"] && !select { + # "target" user -> print alias list + # (only if !select; because of whitespacecontinuation lines) select = 1 print_aliases($2) next } -{ select = 0 } +{ + # other user + select = 0 +} ' "${aliases_file}" diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file index f7c4596..2710792 100755 --- a/type/__mail_alias/explorer/aliases_file +++ b/type/__mail_alias/explorer/aliases_file @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer tries to find the correct aliases file. +# This explorer finds the aliases file to modify. found() { echo "$*"; exit 0; } diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 7778536..f4cbf46 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -29,7 +29,13 @@ in exit 0 fi - echo "set aliases" >>"$__messages_out" + if test -s "${__object}/explorer/aliases" + then + echo "update aliases" >>"$__messages_out" + else + echo "add aliases" >>"$__messages_out" + fi + mode=1 ;; (absent) @@ -37,6 +43,7 @@ in test -s "${__object}/explorer/aliases" || exit 0 echo "delete aliases" >>"$__messages_out" + mode=0 ;; (*) @@ -46,11 +53,10 @@ esac aliases_file=$(cat "${__object}/explorer/aliases_file") -if test -z "${aliases_file}" -then +test -n "${aliases_file}" || { echo 'Could not determine aliases file path.' >&2 exit 1 -fi +} # "export" variables to remote printf 'mode=%u\n' "${mode}" @@ -58,16 +64,18 @@ printf "aliases_file='%s'\n" "${aliases_file}" cat <<'EOF' test -f "${aliases_file}" || touch "${aliases_file}" -awk -F ':[ \t]*' -v mode="${mode}" ' -function sepafter(f, default, _) { + +awk -F ':[ \t]*' -v mode=$mode ' +function sepafter(f, default, _) { + # finds the separator between field $f and $(f+1) _ = substr($0, length($f) + 1, index(substr($0, length($f)+1), $(f+1)) - 1) - if (_) return _ - else return default + return _ ? _ : default } function write_aliases() { if (aliases_written) return + # print aliases line printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") while ((getline < aliases_should_file) > 0) { if (aliases_written) printf ", " @@ -83,36 +91,45 @@ BEGIN { } /^#/ { - # comment - select = 0; cont = 0 + # comment line (leave alone) + select = 0; cont = 0 # comments terminate alias lists and continuations print next } -{ - cont = ($0 ~ /\\$/) - if (cont) sub(/[ \t]*\\$/, "", $0) -} - /^[ \t]/ || cont { - # continuation line + # continuation line (either the previous line ended in a backslash or the + # line starts with whitespace) + + # if in the alias list of the "target" user, we drop the line as it has been + # rewritten previously if (select) next } +{ + # detect if the line is a line to be continued (ends with a backslash) + cont = ($0 ~ /\\$/) + # if it is, we drop the backslash from the line. + if (cont) sub(/[ \t]*\\$/, "", $0) +} + $1 == ENVIRON["__object_id"] { - in_list = 1 + # "target" user -> rewrite aliases list + select = 1 if (mode) write_aliases() next } { - in_list = 0 + # other user + select = 0 print } END { - # if the last line as an alias definition, the separator will be reused - if (mode && !aliases_written) write_aliases() + # if the last line was an alias, the separator will be reused (looks better) + if (mode && !aliases_written) + write_aliases() } ' <"${aliases_file}" >"${aliases_file}.tmp" || { echo 'Generating new aliases file failed!' >&2 @@ -121,9 +138,11 @@ END { if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" then + # aliases file was modified, replace and run `newaliases` mv "${aliases_file}.tmp" "${aliases_file}" newaliases else + # no modifications were made, delete the temp file. rm "${aliases_file}.tmp" fi EOF From ca9e011d50ff2296fa0e4db74d99ecadbcee680f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 15:09:51 +0200 Subject: [PATCH 036/332] [type/__dma_auth] Fix off-by-one error --- type/__dma_auth/explorer/state | 2 +- type/__dma_auth/gencode-remote | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/type/__dma_auth/explorer/state b/type/__dma_auth/explorer/state index 18e13ce..0e6f7be 100755 --- a/type/__dma_auth/explorer/state +++ b/type/__dma_auth/explorer/state @@ -64,7 +64,7 @@ BEGIN { host = substr($0, length(login) + 2) if (match(host, DP)) { - passwd = substr(host, RSTART) + passwd = substr(host, RSTART + 1) host = substr(host, 1, RSTART - 1) } else { passwd = "" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 262a17a..c49779f 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -99,7 +99,10 @@ BEGIN { } # skip comments and empty lines -/^#/ || /^$/ { print; next } +/^#/ || /^$/ { + print + next +} { # parse line @@ -110,7 +113,7 @@ BEGIN { host = substr($0, length(login) + 2) if (match(host, DP)) { - passwd = substr(host, RSTART) + passwd = substr(host, RSTART + 1) host = substr(host, 1, RSTART - 1) } else { passwd = "" From 5b8ae33b4e0bad18f8b76b3cb326884dc16b9550 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 15:21:35 +0200 Subject: [PATCH 037/332] [type/__dma_auth] Improve documentation and handle duplicate lines better The state explorer gained a new value "multiple" (it is not used anywhere, just informative). The code will only write a "should" line once and drop duplicate lines. --- type/__dma_auth/explorer/state | 22 ++++++++++++++++------ type/__dma_auth/gencode-remote | 10 +++++++--- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/type/__dma_auth/explorer/state b/type/__dma_auth/explorer/state index 0e6f7be..668b50f 100755 --- a/type/__dma_auth/explorer/state +++ b/type/__dma_auth/explorer/state @@ -17,8 +17,13 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer looks for lines matching the server parameter in dma's auth.conf -# and reports the login and server fields (password is cksummed) +# This explorer looks for a line matching the login and server parameters +# in dma's auth.conf and reports: +# present: a line matching login + host + password exists +# absent: no line matching login + host exists +# different_password: a line exists but with a different pasword +# multiple: multiple lines matching login + host exist +# (should never happen) auth_conf=$("${__type_explorer}/auth_conf") test -r "${auth_conf}" || exit 0 @@ -32,6 +37,7 @@ fi awk -F'\n' -v server="${server}" ' function getvalue(path) { + # Reads the first line of the file located at path and returns it. getline < path close(path) return $0 @@ -42,6 +48,7 @@ BEGIN { parameter_dir = ENVIRON["__object"] "/parameter/" + # Read the parameters of this object host_param = getvalue(parameter_dir "server") if (!host_param) host_param = ENVIRON["__object_id"] login_param = getvalue(parameter_dir "login") @@ -72,10 +79,13 @@ BEGIN { } host == host_param && login == login_param { - if (passwd == passwd_param) - state = "present" - else - state = "different_password" + # a match… + if (state == "absent") { + state = ((passwd == passwd_param) ? "present" : "different_password") + } else { + # report "multiple" to that the type can remove the duplicates. + state = "multiple" + } } END { diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index c49779f..46d9f31 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -78,6 +78,7 @@ test -f "${auth_conf}" || touch "${auth_conf}" awk -F '\n' -v mode=$mode ' function getvalue(path) { + # Reads the first line of the file located at path and returns it. getline < path close(path) return $0 @@ -124,11 +125,12 @@ host == host_param { if (mode) { # state_should == present if (login == login_param && !written) { - # replace line if host and login match + # replace line if host and login match (but only if no line has + # been written already -> no duplicates) print_should() written = 1 - next } + next } else { # state_should == absent if (!login_param || login == login_param) { @@ -139,7 +141,9 @@ host == host_param { } # leave other lines alone -{ print } +{ + print +} END { if (mode && !written) { From 193b1780dee58be94010e06bf3f3e296f7b4b283 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 15:38:34 +0200 Subject: [PATCH 038/332] Improve error message when invalid --state is used. --- type/__dma_auth/gencode-remote | 3 ++- type/__mail_alias/gencode-remote | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 46d9f31..c57e5cc 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -55,7 +55,8 @@ in printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" ;; (*) - printf 'Invalid --state: %s' "${state_should}" >&2 + printf 'Invalid --state: %s.\n' "${state_should}" >&2 + printf 'Acceptable values are: present, absent.\n' >&2 exit 1 ;; esac diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index f4cbf46..22ae89b 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -47,7 +47,8 @@ in mode=0 ;; (*) - printf 'Invalid --state given: %s\n' "$state_should" >&2 + printf 'Invalid --state: %s.\n' "$state_should" >&2 + printf 'Acceptable values are: present, absent.\n' >&2 exit 1 esac From 551348509717b4b44394a59bd11034e6363d5fbd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 17:01:22 +0200 Subject: [PATCH 039/332] [type/__dma] Improve documentation --- type/__dma/explorer/conf | 8 +++- type/__dma/gencode-remote | 96 ++++++++++++++++++++++++++------------- type/__dma/man.rst | 7 ++- 3 files changed, 75 insertions(+), 36 deletions(-) diff --git a/type/__dma/explorer/conf b/type/__dma/explorer/conf index 129e3c3..b4d6d26 100755 --- a/type/__dma/explorer/conf +++ b/type/__dma/explorer/conf @@ -17,8 +17,12 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer looks for lines matching the server parameter in dma's auth.conf -# and reports the login and server fields (password is cksummed) +# This explorer returns a sorted list of "active" (= non-commented) lines +# in the dma.conf file. +# "Trailing" line comments are stripped off. +# +# NOTE: This explorer assumes that the sort(1) utility supports the non-POXIX +# -s (stable sort) option. CONF_PATH=/etc/dma # set in Makefile dma_conf="${CONF_PATH:?}/dma.conf" diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index e4760d8..01537bf 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -20,7 +20,7 @@ else fi -# Generate config +# Generate "should" values for config conf_should=$( if test -s "${__object}/parameter/smarthost" then @@ -60,7 +60,7 @@ conf_should=$( if test -s "${__object}/parameter/port" then printf 'PORT %u\n' "$(cat "${__object}/parameter/port")" - elif test "${default_smtp_port}" -ne 25 + elif test "${default_smtp_port}" -ne 25 # DMA uses port 25 by default then printf 'PORT %u\n' "${default_smtp_port}" fi @@ -93,6 +93,7 @@ conf_should=$( echo 'NULLCLIENT' fi ) +# Sort conf_should to compare against "conf_is" conf_should=$(echo "$conf_should" | sort -s -k 1,1) config_updated=false @@ -100,24 +101,55 @@ if ! echo "$conf_should" | cmp -s "${__object}/explorer/conf" - then # config needs to be updated echo "dma_conf='${CONF_PATH:?}/dma.conf'" + + # The following AWK script will output the new config file to be stored on + # disk. To do so it reads the current dma.conf file and the config options + # that should be set (from stdin). + # Note that the path to the current dma.conf is passed to AWK twice, because + # the new file cannot be generated in one pass. + + # The logic tries to place options at a sensible location, that is: + # a) if the option is already used in the config file: + # group all similar options (e.g. MASQUERADE) at one place in the order + # they are listed in stdin. + # b) if it is a new option and a "default comment" (e.g. "#PORT 25") exists: + # place options grouped directly after the comment (the comment is left + # alone) + # c) otherwise: + # options are grouped by word (the first word in the line) and appended + # at the end of the file. + cat <<'EOF' awk -F '\n' ' -function comment_line(line) { return match(line, /^[ \t]*#+[ \t]*/) } -function empty_line(line) { return match(line, /^[ \t]*$/) } -function is_word(s) { return s ~ /^[A-Z_]+$/ } +function comment_line(line) { + # returns the position in line at which the comment's text starts + # (0 if the line is not a comment) + match(line, /^[ \t]*\#+[ \t]*/) + return RSTART ? (RLENGTH + 1) : 0 +} +function empty_line(line) { return line ~ /^[ \t]*$/ } +function is_word(s) { return s ~ /^[A-Z_]+$/ } # "looks like a plausible word" function first(line, sep) { + # returns the part of the line until sep is found + # (or the whole line if sep is not found) if (!sep) sep = SUBSEP return index(line, sep) ? substr(line, 1, index(line, sep) - 1) : line } function rest(line, sep) { + # returns the part of the line after the first occurrence of sep is found. + # (or nothing if sep is not found) if (!sep) sep = SUBSEP if (index(line, sep)) return substr(line, index(line, sep) + 1) } function conf_pop(word, value) { + # returns the next value for the config `word` and delete it from the list. + # if value is set, this function will only return value if it is the first + # option in the list, otherwise it returns 0. + if (!(word in conf)) return 0 if (!value) { if (index(conf[word], SUBSEP)) # more than one element? @@ -137,12 +169,14 @@ function conf_pop(word, value) { } function print_conf(word, value) { + # print a config line with the given parameters printf "%s", word if (value) printf " %s", value printf "\n" } function print_confs(word, value) { + # print config lines for all values stored in conf[word]. if (!(word in conf)) return if (conf[word]) { while (value = conf_pop(word)) @@ -154,6 +188,7 @@ function print_confs(word, value) { } BEGIN { + # read the "should" state into the `conf` array. while (getline < "/dev/stdin") { word = first($0, " ") if ((word in conf)) @@ -163,11 +198,12 @@ BEGIN { } } -# first pass, gather information +# first pass, gather information about where which information is stored in the +# current config file. This information will be used in the second pass. NR == FNR { if (comment_line($0)) { # comment line - word = first(substr($0, RLENGTH + 1), " ") + word = first(substr($0, comment_line($0) + 1), " ") if (is_word(word)) last_occ["#" word] = FNR } else { word = first($0, " ") @@ -175,19 +211,22 @@ NR == FNR { } } +# before second pass prepare hashes containing location information to be used +# in the second pass. NR > FNR && FNR == 1 { - # before second pass prepare hashes - + # First we drop the locations of commented-out options if a non-commented + # option is available. If a non-commented option is available, we will + # append new config options there to have them all at one place. for (k in last_occ) if (k ~ /^\#/ && (substr(k, 2) in last_occ)) delete last_occ[k] - for (k in last_occ) { - line_map[last_occ[k]] = k - } + # Reverse the option => line mapping. The line_map allows for easier lookups + # in the second pass. + for (k in last_occ) line_map[last_occ[k]] = k } -# second pass, output new config +# second pass, generate and output new config NR > FNR { if (comment_line($0) || empty_line($0)) { # comment or empty line @@ -195,21 +234,24 @@ NR > FNR { if ((FNR in line_map)) { if (line_map[FNR] ~ /^\#/) { - # the "matching" comment line is here + # This line contains a commented config option. If the conf hash + # contains options to be set, we output them here because this + # option is not used in the current config. k = substr(line_map[FNR], 2) if ((k in conf)) print_confs(k) } if (("INSECURE" in conf) && line_map[FNR] ~ /^\#?SECURE$/) { - # INSECURE goes where SECURE comment is + # INSECURE goes where SECURE comment is. print_confs("INSECURE") } } } else { - sub(/[ \t]*\#.*$/, "", $0) # ignore comments word = first($0, " ") + value = rest($0, " ") + sub(/[ \t]*\#.*$/, "", value) # ignore comments in value - if ((word in conf) && rest($0, " ") == first(conf[word])) { + if ((word in conf) && value == first(conf[word])) { # keep config options we want conf_pop(word) print @@ -223,12 +265,13 @@ NR > FNR { } END { - # print rest of config options + # print rest of config options ( for (word in conf) print_confs(word) } ' "${dma_conf}" "${dma_conf}" <<'EOF' >"${dma_conf}.tmp" \ && mv "${dma_conf}.tmp" "${dma_conf}" EOF + # Pass in "conf_should" via stdin echo "${conf_should}" echo 'EOF' @@ -239,20 +282,9 @@ fi if test -f "${__object}/parameter/send-test-email" then - modified=false - - if grep -q '^__mail_alias/root:' "${__messages_in}" - then - modified=true - elif grep -q '^__dma_auth/' "${__messages_in}" - then - modified=true - elif $config_updated - then - modified=true - fi - - if $modified + if grep -q '^__mail_alias/root:' "${__messages_in}" \ + || grep -q '^__dma_auth/' "${__messages_in}" \ + || $config_updated then cat <<-EOF sendmail root < Date: Thu, 11 Jun 2020 18:07:28 +0200 Subject: [PATCH 040/332] [type/__dma] Use EQS to split config lines --- type/__dma/gencode-remote | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 01537bf..cb2cdbe 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -130,19 +130,20 @@ function comment_line(line) { function empty_line(line) { return line ~ /^[ \t]*$/ } function is_word(s) { return s ~ /^[A-Z_]+$/ } # "looks like a plausible word" -function first(line, sep) { +function first(line, sep_re) { # returns the part of the line until sep is found # (or the whole line if sep is not found) - if (!sep) sep = SUBSEP - return index(line, sep) ? substr(line, 1, index(line, sep) - 1) : line + if (!sep_re) sep_re = "[" SUBSEP "]" + match(line, sep_re) + return RSTART ? substr(line, 1, RSTART - 1) : line } -function rest(line, sep) { +function rest(line, sep_re) { # returns the part of the line after the first occurrence of sep is found. # (or nothing if sep is not found) - if (!sep) sep = SUBSEP - if (index(line, sep)) - return substr(line, index(line, sep) + 1) + if (!sep_re) sep_re = "[" SUBSEP "]" + if (match(line, sep_re)) + return substr(line, RSTART + RLENGTH + 1) } function conf_pop(word, value) { @@ -188,13 +189,15 @@ function print_confs(word, value) { } BEGIN { + EQS = /[ \t]/ # copied from dma/conf.c + # read the "should" state into the `conf` array. while (getline < "/dev/stdin") { - word = first($0, " ") + word = first($0, EQS) if ((word in conf)) - conf[word] = conf[word] SUBSEP rest($0, " ") + conf[word] = conf[word] SUBSEP rest($0, EQS) else - conf[word] = rest($0, " ") + conf[word] = rest($0, EQS) } } @@ -203,10 +206,10 @@ BEGIN { NR == FNR { if (comment_line($0)) { # comment line - word = first(substr($0, comment_line($0) + 1), " ") + word = first(substr($0, comment_line($0) + 1), /[ ]/) if (is_word(word)) last_occ["#" word] = FNR } else { - word = first($0, " ") + word = first($0, EQS) if (is_word(word)) last_occ[word] = FNR } } @@ -247,8 +250,8 @@ NR > FNR { } } } else { - word = first($0, " ") - value = rest($0, " ") + word = first($0, EQS) + value = rest($0, EQS) sub(/[ \t]*\#.*$/, "", value) # ignore comments in value if ((word in conf) && value == first(conf[word])) { From 27102340de8bfde2fa03ebf8dfca7157dbe8ea9f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 20:50:42 +0200 Subject: [PATCH 041/332] [type/__mail_alias] Add bug notice about commas --- type/__mail_alias/man.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst index d6c7873..3782ffb 100644 --- a/type/__mail_alias/man.rst +++ b/type/__mail_alias/man.rst @@ -44,6 +44,15 @@ EXAMPLES # Disable redirection of mail for joe __mail_alias joe --state absent + +BUGS +---- +- Quoted strings are not parsed by this type. As a result, email addresses + containing ``,`` (commas) are treated incorrectly (they are treated as two + addresses/aliases.) + Make sure that email addresses do not contain commas. + + SEE ALSO -------- :strong:`aliases`\ (5) From c777a2b1c27ca81f5296c3b4395fa22db74478e1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 11 Jun 2020 21:58:47 +0200 Subject: [PATCH 042/332] [type/__mail_alias] Some fixes in continuation line processing --- type/__mail_alias/explorer/aliases | 25 ++++++++++++++----------- type/__mail_alias/gencode-remote | 25 +++++++++++++------------ 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index 0a8da94..4fffd3b 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -17,13 +17,15 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# Find aliases for a given name and print the aliases line separated +# Find aliases for a given user name and print the aliases (each one on a +# separate line) aliases_file=$("${__type_explorer}/aliases_file") test -r "${aliases_file}" || exit 0 awk -F ':[ \t]*' ' function print_aliases(aliases, matches) { + # prints comma-separated aliases (one per line) split(aliases, matches, /,[ \t]*/) for (i in matches) { gsub(/^[ \t]*|[ \t]*$/, "", matches[i]) @@ -37,15 +39,11 @@ function print_aliases(aliases, matches) { next } -/^[ \t]/ || cont { - # continuation line (either the previous line ended in a backslash or the - # line starts with whitespace) - - if (select) - print_aliases($0) -} - { + # is this line a continuation line? + # (the prev. line ended in a backslash or the line starts with whitespace) + is_cont = /^[ \t]/ || cont + # detect if the line is a line to be continued (ends with a backslash) cont = ($0 ~ /\\$/) @@ -57,9 +55,14 @@ function print_aliases(aliases, matches) { } } -$1 == ENVIRON["__object_id"] && !select { +is_cont { + # if in the alias list of the "target" user, we also print these aliases. + if (select) print_aliases($0) + next +} + +$1 == ENVIRON["__object_id"] { # "target" user -> print alias list - # (only if !select; because of whitespacecontinuation lines) select = 1 print_aliases($2) next diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 22ae89b..cc5fc42 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -69,7 +69,7 @@ test -f "${aliases_file}" || touch "${aliases_file}" awk -F ':[ \t]*' -v mode=$mode ' function sepafter(f, default, _) { # finds the separator between field $f and $(f+1) - _ = substr($0, length($f) + 1, index(substr($0, length($f)+1), $(f+1)) - 1) + _ = substr($0, length($f)+1, index(substr($0, length($f)+1), $(f+1))-1) return _ ? _ : default } @@ -91,29 +91,30 @@ BEGIN { aliases_should_file = (ENVIRON["__object"] "/parameter/alias") } -/^#/ { +/^[ \t]*\#/ { # comment line (leave alone) select = 0; cont = 0 # comments terminate alias lists and continuations print next } -/^[ \t]/ || cont { - # continuation line (either the previous line ended in a backslash or the - # line starts with whitespace) - - # if in the alias list of the "target" user, we drop the line as it has been - # rewritten previously - if (select) next -} - { + # is this line a continuation line? + # (the prev. line ended in a backslash or the line starts with whitespace) + is_cont = /^[ \t]/ || cont + # detect if the line is a line to be continued (ends with a backslash) cont = ($0 ~ /\\$/) # if it is, we drop the backslash from the line. if (cont) sub(/[ \t]*\\$/, "", $0) } +is_cont { + # we ignore the line as it has been rewritten previously or is not + # interesting + next +} + $1 == ENVIRON["__object_id"] { # "target" user -> rewrite aliases list select = 1 @@ -139,7 +140,7 @@ END { if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" then - # aliases file was modified, replace and run `newaliases` + # aliases file was modified, replace and run `newaliases`. mv "${aliases_file}.tmp" "${aliases_file}" newaliases else From 0f81b89f709e26a98a369d6e8b0797eb3ca85909 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 22 Jun 2020 13:29:28 +0200 Subject: [PATCH 043/332] [type/__dma] Make --smarthost optional --- type/__dma/parameter/optional | 1 + type/__dma/parameter/required | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 type/__dma/parameter/required diff --git a/type/__dma/parameter/optional b/type/__dma/parameter/optional index 3f6cb5d..615c189 100644 --- a/type/__dma/parameter/optional +++ b/type/__dma/parameter/optional @@ -1,3 +1,4 @@ mailname port security +smarthost diff --git a/type/__dma/parameter/required b/type/__dma/parameter/required deleted file mode 100644 index 0753fb6..0000000 --- a/type/__dma/parameter/required +++ /dev/null @@ -1 +0,0 @@ -smarthost From 27b832f2127fc0b475a6390cacb9219b9516328f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 22 Jun 2020 14:02:13 +0200 Subject: [PATCH 044/332] [type/__dma] Add support for Alpine Linux requires the testing repository, currently. --- type/__dma/gencode-remote | 8 ++++---- type/__dma/manifest | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index cb2cdbe..a6aca0d 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -122,7 +122,7 @@ then cat <<'EOF' awk -F '\n' ' function comment_line(line) { - # returns the position in line at which the comment's text starts + # returns the position in line at which the comment'\''s text starts # (0 if the line is not a comment) match(line, /^[ \t]*\#+[ \t]*/) return RSTART ? (RLENGTH + 1) : 0 @@ -143,7 +143,7 @@ function rest(line, sep_re) { # (or nothing if sep is not found) if (!sep_re) sep_re = "[" SUBSEP "]" if (match(line, sep_re)) - return substr(line, RSTART + RLENGTH + 1) + return substr(line, RSTART + RLENGTH) } function conf_pop(word, value) { @@ -189,7 +189,7 @@ function print_confs(word, value) { } BEGIN { - EQS = /[ \t]/ # copied from dma/conf.c + EQS = "[ \t]" # copied from dma/conf.c # read the "should" state into the `conf` array. while (getline < "/dev/stdin") { @@ -206,7 +206,7 @@ BEGIN { NR == FNR { if (comment_line($0)) { # comment line - word = first(substr($0, comment_line($0) + 1), /[ ]/) + word = first(substr($0, comment_line($0)), " ") if (is_word(word)) last_occ["#" word] = FNR } else { word = first($0, EQS) diff --git a/type/__dma/manifest b/type/__dma/manifest index 7abd7c8..75e42d7 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -5,6 +5,10 @@ os=$(cat "${__global}/explorer/os") # Install DMA case $os in + (alpine) + __package dma --state present + export require='__package/dma' + ;; (debian|devuan|ubuntu) __package dma --state present export require='__package/dma' From aa605cada45d5a91da5c4153fa822d95a8214d80 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 22 Jun 2020 14:02:42 +0200 Subject: [PATCH 045/332] [type/__mail_aliases] Add support for Alpine Linux Alpine's DMA package has a typo and installs "newailases" instead of "newaliases". I adjusted the code-remote to only run newaliases if it is available. Otherwise, tough luck, user gotta either fix his system or run manually. --- type/__mail_alias/explorer/aliases_file | 4 ++-- type/__mail_alias/gencode-remote | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file index 2710792..90bccde 100755 --- a/type/__mail_alias/explorer/aliases_file +++ b/type/__mail_alias/explorer/aliases_file @@ -28,7 +28,7 @@ check_file() { fi } -case $("$__explorer/os") +case $("${__explorer}/os") in (freebsd|openbsd|solaris) check_file /etc/mail/aliases @@ -36,7 +36,7 @@ in # default found /etc/mail/aliases ;; - (debian|devuan|ubuntu) + (alpine|debian|devuan|ubuntu) check_file /etc/aliases # default diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index cc5fc42..3eaad75 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -142,7 +142,9 @@ if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" then # aliases file was modified, replace and run `newaliases`. mv "${aliases_file}.tmp" "${aliases_file}" - newaliases + + # run newaliases if present + command -v newaliases >/dev/null 2>&1 && newaliases || true else # no modifications were made, delete the temp file. rm "${aliases_file}.tmp" From f6a36a60c0f92ae212c2e1274cba64ffdc0c856c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 25 Jun 2020 11:15:23 +0200 Subject: [PATCH 046/332] Import __matrix_riot from ungleich's dot-cdist --- type/__matrix_riot/files/config.json.sh | 90 +++++++++++++++ type/__matrix_riot/gencode-remote | 73 +++++++++++++ type/__matrix_riot/man.rst | 87 +++++++++++++++ type/__matrix_riot/manifest | 103 ++++++++++++++++++ type/__matrix_riot/parameter/boolean | 1 + type/__matrix_riot/parameter/default/brand | 1 + .../parameter/default/cookie_policy_url | 1 + .../parameter/default/default_country_code | 1 + .../parameter/default/default_server_name | 1 + .../parameter/default/default_server_url | 1 + type/__matrix_riot/parameter/default/owner | 1 + .../parameter/default/privacy_policy_url | 1 + .../parameter/default/room_directory_servers | 1 + type/__matrix_riot/parameter/optional | 13 +++ .../__matrix_riot/parameter/optional_multiple | 1 + type/__matrix_riot/parameter/required | 2 + 16 files changed, 378 insertions(+) create mode 100755 type/__matrix_riot/files/config.json.sh create mode 100755 type/__matrix_riot/gencode-remote create mode 100644 type/__matrix_riot/man.rst create mode 100755 type/__matrix_riot/manifest create mode 100644 type/__matrix_riot/parameter/boolean create mode 100644 type/__matrix_riot/parameter/default/brand create mode 100644 type/__matrix_riot/parameter/default/cookie_policy_url create mode 100644 type/__matrix_riot/parameter/default/default_country_code create mode 100644 type/__matrix_riot/parameter/default/default_server_name create mode 100644 type/__matrix_riot/parameter/default/default_server_url create mode 100644 type/__matrix_riot/parameter/default/owner create mode 100644 type/__matrix_riot/parameter/default/privacy_policy_url create mode 100644 type/__matrix_riot/parameter/default/room_directory_servers create mode 100644 type/__matrix_riot/parameter/optional create mode 100644 type/__matrix_riot/parameter/optional_multiple create mode 100644 type/__matrix_riot/parameter/required diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_riot/files/config.json.sh new file mode 100755 index 0000000..e7fca72 --- /dev/null +++ b/type/__matrix_riot/files/config.json.sh @@ -0,0 +1,90 @@ +#!/bin/sh +# +# Upstream configuration guide/documentation: +# https://github.com/vector-im/riot-web/blob/develop/docs/config.md + +generate_embedded_pages () { + if [ $EMBED_HOMEPAGE ]; then + cat << EOF + "embeddedPages": { + "homeUrl": "home.html" + }, +EOF + fi +} + +generate_jitsi_config () { + if [ "$JITSI_DOMAIN" != "" ]; then + cat << EOF + "jitsi": { + "preferredDomain": "$JITSI_DOMAIN" + }, +EOF + fi +} + +generate_branding () { + echo '"branding": {' + + if [ "$BRANDING_AUTH_HEADER_LOGO_URL" != "" ]; then + cat << EOF + "authHeaderLogoUrl": "$BRANDING_AUTH_HEADER_LOGO_URL", +EOF + fi + + if [ "$BRANDING_AUTH_FOOTER_LINKS" != "" ]; then + cat << EOF + "authFooterLinks": "$BRANDING_AUTH_FOOTER_LINKS", +EOF + fi + + cat << EOF + "welcomeBackgroundUrl": "themes/riot/img/backgrounds/valley.jpg" +EOF + echo '},' +} + +cat << EOF +{ + "default_server_config": { + "m.homeserver": { + "base_url": "$DEFAULT_SERVER_URL", + "server_name": "$DEFAULT_SERVER_NAME" + }, + "m.identity_server": { + "base_url": "https://vector.im" + } + }, + "brand": "$BRAND", + $(generate_branding) + "defaultCountryCode": "$DEFAULT_COUNTRY_CODE", + "integrations_ui_url": "https://scalar.vector.im/", + "integrations_rest_url": "https://scalar.vector.im/api", + "integrations_widgets_urls": [ + "https://scalar.vector.im/_matrix/integrations/v1", + "https://scalar.vector.im/api", + "https://scalar-staging.vector.im/_matrix/integrations/v1", + "https://scalar-staging.vector.im/api", + "https://scalar-staging.riot.im/scalar/api" + ], + "bug_report_endpoint_url": "https://riot.im/bugreports/submit", + "roomDirectory": { + "servers": [ + $ROOM_DIRECTORY_SERVERS + ] + }, + "disable_custom_urls": "$DISABLE_CUSTOM_URLS", + $(generate_embedded_pages) + $(generate_jitsi_config) + "terms_and_conditions_links": [ + { + "url": "$PRIVACY_POLICY_URL", + "text": "Privacy Policy" + }, + { + "url": "$COOKIE_POLICY_URL", + "text": "Cookie Policy" + } + ] +} +EOF diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_riot/gencode-remote new file mode 100755 index 0000000..97c4777 --- /dev/null +++ b/type/__matrix_riot/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh -e +# +# 2019 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +VERSION=$(cat "$__object/parameter/version") +INSTALL_DIR=$(cat "$__object/parameter/install_dir") +OWNER=$(cat "$__object/parameter/owner") + +src="riot-v$VERSION" +archive="$src.tar.gz" +config='config.json' +homepage='home.html' +welcomepage='welcome.html' +welcomelogo='welcome-logo.svg' +url="https://github.com/vector-im/riot-web/releases/download/v$VERSION/$archive" + +# tar and curl are installed by the __matrix-riot manifest. mktemp is usually +# provided by coreutils and assumed installed. +cat << EOF +set -e + +# Ensure that coreutils is installed. +if [ ! -x \$(which mktemp) ]; then + echo "mktemp is not available on the remote host." >&2 + exit 1 +fi + +# Create temporary working directory. +tmpdir=\$(mktemp -d) +custom_files_dir="\$tmpdir/custom_files" +cd \$tmpdir + +# Download and extract sources. +curl -L '$url' > $archive +tar xf $archive + +# Backup files deployed by __matrix_riot. +mkdir -p \$custom_files_dir +for file in $INSTALL_DIR/cdist/*; do + cp "\$file" "\$custom_files_dir" +done + +# Deploy sources and restore configuration. +rm -r '$INSTALL_DIR' +mv '$src' '$INSTALL_DIR' + +for file in \$custom_files_dir/*; do + cp "\$file" '$INSTALL_DIR' +done + +# Chown deployed files to requested owner. +chown -R '$OWNER' '$INSTALL_DIR' + +# Remove temporary working directory. +cd / +rm -r \$tmpdir +EOF diff --git a/type/__matrix_riot/man.rst b/type/__matrix_riot/man.rst new file mode 100644 index 0000000..2fe8eae --- /dev/null +++ b/type/__matrix_riot/man.rst @@ -0,0 +1,87 @@ +cdist-type__matrix_riot(7) +====================== + +NAME +---- +cdist-type__matrix_riot - Install and configure Riot, a web Matrix client. + + +DESCRIPTION +----------- +This type install and configure the Riot web client. + + +REQUIRED PARAMETERS +------------------- +install_dir + Root directory of Riot's static files. + +version + Release of Riot to install. + +OPTIONAL PARAMETERS +------------------- +default_server_name + Name of matrix homeserver to connect to, defaults to 'matrix.org'. + +default_server_url + URL of matrix homeserver to connect to, defaults to 'https://matrix-client.matrix.org'. + +owner + Owner of the deployed files, passed to `chown`. Defaults to 'root'. + +brand + Web UI branding, defaults to 'Riot'. + +default_country_code + ISO 3166 alpha2 country code to use when showing country selectors, such as + phone number inputs. Defaults to GB. + +privacy_policy_url + Defaults to 'https://riot.im/privacy'. + +cookie_policy_url + Defaults to 'https://matrix.org/docs/guides/riot_im_cookie_policy'. + +jitsi_domain + Domain name of preferred Jitsi instance (default is jitsi.riot.im). This is + used whenever a user clicks on the voice/video call buttons. + +homepage + Path to custom homepage, displayed once logged in. + +welcomepage + Path to custom welcome (= login) page. + +custom_asset + Serve a file a the top-level directory (e.g. /my-custom-logo.svg). Can be specified multiple times. + +BOOLEAN PARAMETERS +------------------- +disable_custom_urls + Disallow the user to change the default homeserver when signing up or logging in. + +EXAMPLES +-------- + +.. code-block:: sh + + __matrix_riot my-riot --install_dir /var/www/riot-web --version 1.5.6 + + +SEE ALSO +-------- +- `cdist-type__matrix_synapse(7) `_ + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2019 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__matrix_riot/manifest b/type/__matrix_riot/manifest new file mode 100755 index 0000000..a843f4b --- /dev/null +++ b/type/__matrix_riot/manifest @@ -0,0 +1,103 @@ +#!/bin/sh -e +# +# 2019 Timothée Floure (timothee.floure@ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +INSTALL_DIR=$(cat "$__object/parameter/install_dir") + +export DEFAULT_SERVER_NAME=$(cat "$__object/parameter/default_server_name") +export DEFAULT_SERVER_URL=$(cat "$__object/parameter/default_server_url") +export BRAND=$(cat "$__object/parameter/brand") +export DEFAULT_COUNTRY_CODE=$(cat "$__object/parameter/default_country_code") +export ROOM_DIRECTORY_SERVERS=$(cat "$__object/parameter/room_directory_servers") +export PRIVACY_POLICY_URL=$(cat "$__object/parameter/privacy_policy_url") +export COOKIE_POLICY_URL=$(cat "$__object/parameter/cookie_policy_url") + +if [ -f "$__object/parameter/jitsi_domain" ]; then + export JITSI_DOMAIN=$(cat "$__object/parameter/jitsi_domain") +fi + +if [ -f "$__object/parameter/branding_auth_header_logo_url" ]; then + export BRANDING_AUTH_HEADER_LOGO_URL=$(cat "$__object/parameter/branding_auth_header_logo_url") +fi + +if [ -f "$__object/parameter/branding_auth_footer_links" ]; then + export BRANDING_AUTH_FOOTER_LINKS=$(cat "$__object/parameter/branding_auth_footer_links") +fi + +if [ -f "$__object/parameter/homepage" ]; then + export EMBED_HOMEPAGE=1 + homepage=$(cat "$__object/parameter/homepage") +fi + +if [ -f "$__object/parameter/welcomepage" ]; then + export EMBED_WELCOMEPAGE=1 + welcomepage=$(cat "$__object/parameter/welcomepage") +fi + +if [ -f "$__object/parameter/custom_asset" ]; then + for file in $(cat "$__object/parameter/custom_asset"); do + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename $file)" \ + --source "$file" \ + --mode 0664 \ + --state present + done +fi + +if [ -f "$__object/parameter/disable_custom_urls" ]; then + export DISABLE_CUSTOM_URLS='true' +else + export DISABLE_CUSTOM_URLS='false' +fi + +# Owner of the uploaded files. +owner=$(cat "$__object/parameter/owner") + +# Ensure that curl and tar are installed, as they will be required by the +# gencode-remote script. +__package curl --state present +__package tar --state present + +# Generate and deploy configuration file. +mkdir -p "$__object/files" +"$__type/files/config.json.sh" > "$__object/files/config.json" + +# Install the config.json configuration file. The application's sources are +# downloaded and deployed by gencode-remote. +__directory "$INSTALL_DIR/cdist" \ + --owner "$owner" --mode 0755 --parents \ + --state present + +require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/config.json" \ + --source "$__object/files/config.json" \ + --mode 0664 \ + --state present + +if [ $EMBED_HOMEPAGE ]; then + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/home.html" \ + --source "$homepage" \ + --mode 0664 \ + --state present +fi + +if [ $EMBED_WELCOMEPAGE ]; then + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/welcome.html" \ + --source "$welcomepage" \ + --mode 0664 \ + --state present +fi diff --git a/type/__matrix_riot/parameter/boolean b/type/__matrix_riot/parameter/boolean new file mode 100644 index 0000000..4d77768 --- /dev/null +++ b/type/__matrix_riot/parameter/boolean @@ -0,0 +1 @@ +disable_custom_urls diff --git a/type/__matrix_riot/parameter/default/brand b/type/__matrix_riot/parameter/default/brand new file mode 100644 index 0000000..e8095bb --- /dev/null +++ b/type/__matrix_riot/parameter/default/brand @@ -0,0 +1 @@ +Riot diff --git a/type/__matrix_riot/parameter/default/cookie_policy_url b/type/__matrix_riot/parameter/default/cookie_policy_url new file mode 100644 index 0000000..04e9c2b --- /dev/null +++ b/type/__matrix_riot/parameter/default/cookie_policy_url @@ -0,0 +1 @@ +https://matrix.org/docs/guides/riot_im_cookie_policy diff --git a/type/__matrix_riot/parameter/default/default_country_code b/type/__matrix_riot/parameter/default/default_country_code new file mode 100644 index 0000000..30ac4a3 --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_country_code @@ -0,0 +1 @@ +GB diff --git a/type/__matrix_riot/parameter/default/default_server_name b/type/__matrix_riot/parameter/default/default_server_name new file mode 100644 index 0000000..5528ffd --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_server_name @@ -0,0 +1 @@ +matrix.org diff --git a/type/__matrix_riot/parameter/default/default_server_url b/type/__matrix_riot/parameter/default/default_server_url new file mode 100644 index 0000000..2cb9227 --- /dev/null +++ b/type/__matrix_riot/parameter/default/default_server_url @@ -0,0 +1 @@ +https://matrix-client.matrix.org diff --git a/type/__matrix_riot/parameter/default/owner b/type/__matrix_riot/parameter/default/owner new file mode 100644 index 0000000..d8649da --- /dev/null +++ b/type/__matrix_riot/parameter/default/owner @@ -0,0 +1 @@ +root diff --git a/type/__matrix_riot/parameter/default/privacy_policy_url b/type/__matrix_riot/parameter/default/privacy_policy_url new file mode 100644 index 0000000..4cdd12c --- /dev/null +++ b/type/__matrix_riot/parameter/default/privacy_policy_url @@ -0,0 +1 @@ +https://riot.im/privacy diff --git a/type/__matrix_riot/parameter/default/room_directory_servers b/type/__matrix_riot/parameter/default/room_directory_servers new file mode 100644 index 0000000..4ea73ad --- /dev/null +++ b/type/__matrix_riot/parameter/default/room_directory_servers @@ -0,0 +1 @@ +"matrix.org" diff --git a/type/__matrix_riot/parameter/optional b/type/__matrix_riot/parameter/optional new file mode 100644 index 0000000..21a2faf --- /dev/null +++ b/type/__matrix_riot/parameter/optional @@ -0,0 +1,13 @@ +default_server_url +default_server_name +brand +default_country_code +privacy_policy_url +cookie_policy_url +room_directory_servers +owner +homepage +welcomepage +jitsi_domain +branding_auth_header_logo_url +branding_auth_footer_links diff --git a/type/__matrix_riot/parameter/optional_multiple b/type/__matrix_riot/parameter/optional_multiple new file mode 100644 index 0000000..4c2ca54 --- /dev/null +++ b/type/__matrix_riot/parameter/optional_multiple @@ -0,0 +1 @@ +custom_asset diff --git a/type/__matrix_riot/parameter/required b/type/__matrix_riot/parameter/required new file mode 100644 index 0000000..a76477e --- /dev/null +++ b/type/__matrix_riot/parameter/required @@ -0,0 +1,2 @@ +version +install_dir From 43c59985d0aba80ae6b41a56143e3247423c23a5 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 25 Jun 2020 18:07:51 +0200 Subject: [PATCH 047/332] [type/__mail_alias] Fallback to /etc/aliases instead of /etc/mail/aliases --- type/__mail_alias/explorer/aliases_file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file index 90bccde..a59bb99 100755 --- a/type/__mail_alias/explorer/aliases_file +++ b/type/__mail_alias/explorer/aliases_file @@ -47,6 +47,6 @@ in check_file /etc/aliases # default - found /etc/mail/aliases + found /etc/aliases ;; esac From 331461219387b733d7978171aa0947954ecedbe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 11 Jul 2020 09:21:37 +0200 Subject: [PATCH 048/332] __matrix_riot: improve friendship with shellcheck --- type/__matrix_riot/files/config.json.sh | 2 +- type/__matrix_riot/gencode-remote | 4 ---- type/__matrix_riot/manifest | 9 ++++++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_riot/files/config.json.sh index e7fca72..6cd1648 100755 --- a/type/__matrix_riot/files/config.json.sh +++ b/type/__matrix_riot/files/config.json.sh @@ -4,7 +4,7 @@ # https://github.com/vector-im/riot-web/blob/develop/docs/config.md generate_embedded_pages () { - if [ $EMBED_HOMEPAGE ]; then + if [ "$EMBED_HOMEPAGE" != "" ]; then cat << EOF "embeddedPages": { "homeUrl": "home.html" diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_riot/gencode-remote index 97c4777..dbc886c 100755 --- a/type/__matrix_riot/gencode-remote +++ b/type/__matrix_riot/gencode-remote @@ -24,10 +24,6 @@ OWNER=$(cat "$__object/parameter/owner") src="riot-v$VERSION" archive="$src.tar.gz" -config='config.json' -homepage='home.html' -welcomepage='welcome.html' -welcomelogo='welcome-logo.svg' url="https://github.com/vector-im/riot-web/releases/download/v$VERSION/$archive" # tar and curl are installed by the __matrix-riot manifest. mktemp is usually diff --git a/type/__matrix_riot/manifest b/type/__matrix_riot/manifest index a843f4b..544bd96 100755 --- a/type/__matrix_riot/manifest +++ b/type/__matrix_riot/manifest @@ -16,7 +16,10 @@ # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -# + +# Ignore "Declare and assign separately to avoid masking return values. [SC2155]" +# => not relevant for the type arguments. +# shellcheck disable=SC2155 INSTALL_DIR=$(cat "$__object/parameter/install_dir") @@ -51,8 +54,8 @@ if [ -f "$__object/parameter/welcomepage" ]; then fi if [ -f "$__object/parameter/custom_asset" ]; then - for file in $(cat "$__object/parameter/custom_asset"); do - require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename $file)" \ + "$__object/parameter/custom_asset" | while IFS= read -r file; do + require="__directory/$INSTALL_DIR/cdist" __file "$INSTALL_DIR/cdist/$(basename "$file")" \ --source "$file" \ --mode 0664 \ --state present From 70047d10b1878e46a53c26a2bc8c50779c89f0ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 20 Jul 2020 17:20:33 +0200 Subject: [PATCH 049/332] Initial __netbox type --- type/__netbox/files/configuration.py.sh | 258 ++++++++++++++++++++++++ type/__netbox/files/gunicorn.py | 16 ++ type/__netbox/files/netbox-rq.service | 21 ++ type/__netbox/files/netbox.service | 22 ++ type/__netbox/gencode-remote | 58 ++++++ type/__netbox/manifest | 47 +++++ type/__netbox/parameter/required | 5 + type/__netbox/singleton | 0 8 files changed, 427 insertions(+) create mode 100755 type/__netbox/files/configuration.py.sh create mode 100644 type/__netbox/files/gunicorn.py create mode 100644 type/__netbox/files/netbox-rq.service create mode 100644 type/__netbox/files/netbox.service create mode 100755 type/__netbox/gencode-remote create mode 100644 type/__netbox/manifest create mode 100644 type/__netbox/parameter/required create mode 100644 type/__netbox/singleton diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh new file mode 100755 index 0000000..b7e1597 --- /dev/null +++ b/type/__netbox/files/configuration.py.sh @@ -0,0 +1,258 @@ +#!/bin/sh + +cat << EOF +######################### +# # +# Required settings # +# # +######################### + +# This is a list of valid fully-qualified domain names (FQDNs) for the NetBox server. NetBox will not permit write +# access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. +# +# Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] +ALLOWED_HOSTS = [ '$ALLOWED_HOST' ] + +# PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: +# https://docs.djangoproject.com/en/stable/ref/settings/#databases +DATABASE = { + 'NAME': '$DATABASE_NAME', # Database name + 'USER': '$DATABASE_USER', # PostgreSQL username + 'PASSWORD': '$DATABASE_PASSWORD', # PostgreSQL password + 'HOST': 'localhost', # Database server + 'PORT': '', # Database port (leave blank for default) + 'CONN_MAX_AGE': 300, # Max database connection age +} + +# Redis database settings. Redis is used for caching and for queuing background tasks such as webhook events. A separate +# configuration exists for each. Full connection details are required in both sections, and it is strongly recommended +# to use two separate database IDs. +REDIS = { + 'tasks': { + 'HOST': 'localhost', + 'PORT': 6379, + # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 0, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + }, + 'caching': { + 'HOST': 'localhost', + 'PORT': 6379, + # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel + # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], + # 'SENTINEL_SERVICE': 'netbox', + 'PASSWORD': '', + 'DATABASE': 1, + 'DEFAULT_TIMEOUT': 300, + 'SSL': False, + } +} + +# This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. +# For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and +# symbols. NetBox will not run without this defined. For more information, see +# https://docs.djangoproject.com/en/stable/ref/settings/#std:setting-SECRET_KEY +SECRET_KEY = '$SECRET_KEY' + + +######################### +# # +# Optional settings # +# # +######################### + +# Specify one or more name and email address tuples representing NetBox administrators. These people will be notified of +# application errors (assuming correct email settings are provided). +ADMINS = [ + # ['John Doe', 'jdoe@example.com'], +] + +# URL schemes that are allowed within links in NetBox +ALLOWED_URL_SCHEMES = ( + 'file', 'ftp', 'ftps', 'http', 'https', 'irc', 'mailto', 'sftp', 'ssh', 'tel', 'telnet', 'tftp', 'vnc', 'xmpp', +) + +# Optionally display a persistent banner at the top and/or bottom of every page. HTML is allowed. To display the same +# content in both banners, define BANNER_TOP and set BANNER_BOTTOM = BANNER_TOP. +BANNER_TOP = '' +BANNER_BOTTOM = '' + +# Text to include on the login page above the login form. HTML is allowed. +BANNER_LOGIN = '' + +# Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: +# BASE_PATH = 'netbox/' +BASE_PATH = '' + +# Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) +CACHE_TIMEOUT = 900 + +# Maximum number of days to retain logged changes. Set to 0 to retain changes indefinitely. (Default: 90) +CHANGELOG_RETENTION = 90 + +# API Cross-Origin Resource Sharing (CORS) settings. If CORS_ORIGIN_ALLOW_ALL is set to True, all origins will be +# allowed. Otherwise, define a list of allowed origins using either CORS_ORIGIN_WHITELIST or +# CORS_ORIGIN_REGEX_WHITELIST. For more information, see https://github.com/ottoyiu/django-cors-headers +CORS_ORIGIN_ALLOW_ALL = False +CORS_ORIGIN_WHITELIST = [ + # 'https://hostname.example.com', +] +CORS_ORIGIN_REGEX_WHITELIST = [ + # r'^(https?://)?(\w+\.)?example\.com$', +] + +# Set to True to enable server debugging. WARNING: Debugging introduces a substantial performance penalty and may reveal +# sensitive information about your installation. Only enable debugging while performing testing. Never enable debugging +# on a production system. +DEBUG = False + +# Email settings +EMAIL = { + 'SERVER': 'localhost', + 'PORT': 25, + 'USERNAME': '', + 'PASSWORD': '', + 'USE_SSL': False, + 'USE_TLS': False, + 'TIMEOUT': 10, # seconds + 'FROM_EMAIL': '', +} + +# Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table +# (all prefixes and IP addresses not assigned to a VRF), set ENFORCE_GLOBAL_UNIQUE to True. +ENFORCE_GLOBAL_UNIQUE = False + +# Exempt certain models from the enforcement of view permissions. Models listed here will be viewable by all users and +# by anonymous users. List models in the form \`.\`. Add '*' to this list to exempt all models. +EXEMPT_VIEW_PERMISSIONS = [ + # 'dcim.site', + # 'dcim.region', + # 'ipam.prefix', +] + +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +# HTTP_PROXIES = { +# 'http': 'http://10.10.1.10:3128', +# 'https': 'http://10.10.1.10:1080', +# } + +# IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing +# NetBox from an internal IP. +INTERNAL_IPS = ('127.0.0.1', '::1') + +# Enable custom logging. Please see the Django documentation for detailed guidance on configuring custom logs: +# https://docs.djangoproject.com/en/stable/topics/logging/ +LOGGING = {} + +# Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users +# are permitted to access most data in NetBox (excluding secrets) but not make any changes. +LOGIN_REQUIRED = False + +# The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to +# re-authenticate. (Default: 1209600 [14 days]) +LOGIN_TIMEOUT = None + +# Setting this to True will display a "maintenance mode" banner at the top of every page. +MAINTENANCE_MODE = False + +# An API consumer can request an arbitrary number of objects =by appending the "limit" parameter to the URL (e.g. +# "?limit=1000"). This setting defines the maximum limit. Setting it to 0 or None will allow an API consumer to request +# all objects by specifying "?limit=0". +MAX_PAGE_SIZE = 1000 + +# The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that +# the default value of this setting is derived from the installed location. +# MEDIA_ROOT = '/opt/netbox/netbox/media' + +# By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the +# class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: +# STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' +# STORAGE_CONFIG = { +# 'AWS_ACCESS_KEY_ID': 'Key ID', +# 'AWS_SECRET_ACCESS_KEY': 'Secret', +# 'AWS_STORAGE_BUCKET_NAME': 'netbox', +# 'AWS_S3_REGION_NAME': 'eu-west-1', +# } + +# Expose Prometheus monitoring metrics at the HTTP endpoint '/metrics' +METRICS_ENABLED = False + +# Credentials that NetBox will uses to authenticate to devices when connecting via NAPALM. +NAPALM_USERNAME = '' +NAPALM_PASSWORD = '' + +# NAPALM timeout (in seconds). (Default: 30) +NAPALM_TIMEOUT = 30 + +# NAPALM optional arguments (see http://napalm.readthedocs.io/en/latest/support/#optional-arguments). Arguments must +# be provided as a dictionary. +NAPALM_ARGS = {} + +# Determine how many objects to display per page within a list. (Default: 50) +PAGINATE_COUNT = 50 + +# Enable installed plugins. Add the name of each plugin to the list. +PLUGINS = [] + +# Plugins configuration settings. These settings are used by various plugins that the user may have installed. +# Each key in the dictionary is the name of an installed plugin and its value is a dictionary of settings. +# PLUGINS_CONFIG = { +# 'my_plugin': { +# 'foo': 'bar', +# 'buzz': 'bazz' +# } +# } + +# When determining the primary IP address for a device, IPv6 is preferred over IPv4 by default. Set this to True to +# prefer IPv4 instead. +PREFER_IPV4 = False + +# Rack elevation size defaults, in pixels. For best results, the ratio of width to height should be roughly 10:1. +RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = 22 +RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 + +# Remote authentication support +REMOTE_AUTH_ENABLED = False +REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' +REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' +REMOTE_AUTH_AUTO_CREATE_USER = True +REMOTE_AUTH_DEFAULT_GROUPS = [] +REMOTE_AUTH_DEFAULT_PERMISSIONS = [] + +# This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. +RELEASE_CHECK_TIMEOUT = 24 * 3600 + +# This repository is used to check whether there is a new release of NetBox available. Set to None to disable the +# version check or use the URL below to check for release in the official NetBox repository. +RELEASE_CHECK_URL = None +# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +# The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +# REPORTS_ROOT = '/opt/netbox/netbox/reports' + +# The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of +# this setting is derived from the installed location. +# SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' + +# By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use +# local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only +# database access.) Note that the user as which NetBox runs must have read and write permissions to this path. +SESSION_FILE_PATH = None + +# Time zone (default: UTC) +TIME_ZONE = 'UTC' + +# Date/time formatting. See the following link for supported formats: +# https://docs.djangoproject.com/en/stable/ref/templates/builtins/#date +DATE_FORMAT = 'N j, Y' +SHORT_DATE_FORMAT = 'Y-m-d' +TIME_FORMAT = 'g:i a' +SHORT_TIME_FORMAT = 'H:i:s' +DATETIME_FORMAT = 'N j, Y g:i a' +SHORT_DATETIME_FORMAT = 'Y-m-d H:i' +EOF diff --git a/type/__netbox/files/gunicorn.py b/type/__netbox/files/gunicorn.py new file mode 100644 index 0000000..c582677 --- /dev/null +++ b/type/__netbox/files/gunicorn.py @@ -0,0 +1,16 @@ +# The IP address (typically localhost) and port that the Netbox WSGI process should listen on +bind = '127.0.0.1:8001' + +# Number of gunicorn workers to spawn. This should typically be 2n+1, where +# n is the number of CPU cores present. +workers = 3 + +# Number of threads per worker process +threads = 3 + +# Timeout (in seconds) for a request to complete +timeout = 120 + +# The maximum number of requests a worker can handle before being respawned +max_requests = 5000 +max_requests_jitter = 500 diff --git a/type/__netbox/files/netbox-rq.service b/type/__netbox/files/netbox-rq.service new file mode 100644 index 0000000..77d7091 --- /dev/null +++ b/type/__netbox/files/netbox-rq.service @@ -0,0 +1,21 @@ +[Unit] +Description=NetBox Request Queue Worker +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service new file mode 100644 index 0000000..18eb045 --- /dev/null +++ b/type/__netbox/files/netbox.service @@ -0,0 +1,22 @@ +[Unit] +Description=NetBox WSGI Service +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +PIDFile=/var/tmp/netbox.pid +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote new file mode 100755 index 0000000..a52b110 --- /dev/null +++ b/type/__netbox/gencode-remote @@ -0,0 +1,58 @@ +#!/bin/sh + +VERSION=$(cat "$__object/parameter/version") + +src="netbox-$VERSION" +archive="v$VERSION.tar.gz" +url="https://github.com/netbox-community/netbox/archive/$archive" +install_dir=/opt/netbox/netbox + +cat << EOF +set -e + +# Ensure that coreutils is installed. +if [ ! -x \$(which mktemp) ]; then + echo "mktemp is not available on the remote host." >&2 + exit 1 +fi + +# Create temporary working directory. +tmpdir=\$(mktemp -d) +cd "\$tmpdir" + +# Download and extract sources. +curl -L '$url' > '$archive' +tar xf '$archive' + +# Save cdist-upload configuration file. +mkdir -p "\$tmpdir" +cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" + +# Deploy sources and restore configuration. +rm -r '$install_dir' +cp -r '$src/netbox' '$install_dir' +cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' + +# Setup & enter python virtualenv. +virtualenv /opt/netbox/venv + +# Install python dependencies. +/opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" + +# Set final permissions. +chown -R netbox /opt/netbox + +# Run database migrations. +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate + +# Generate static assets. +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collectstatic --no-input + +# Remove temporary working directory. +cd / +rm -r "\$tmpdir" + +# Restart services. +service netbox restart +service netbox-rq restart +EOF diff --git a/type/__netbox/manifest b/type/__netbox/manifest new file mode 100644 index 0000000..64a0a55 --- /dev/null +++ b/type/__netbox/manifest @@ -0,0 +1,47 @@ +#!/bin/sh + +os=$(cat "$__global/explorer/os") + +case "$os" in + debian|ubuntu) + # Install netbox dependencies. + for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv; do + __package $pkg + done + + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +export DATABASE_NAME=$(cat "$__object/parameter/database") +export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") +export ALLOWED_HOST=$(cat "$__object/parameter/host") +export SECRET_KEY=$(cat "$__object/parameter/secret-key") + +# Create system user used to run netbox. +__user netbox --system --home /opt/netbox --create-home + +# Generate and upload netbox configuration. +mkdir -p "$__object/files" +"$__type/files/configuration.py.sh" > "$__object/files/configuration.py" + +require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents +require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ + --source "$__object/files/configuration.py" + +# Upload systemd units and gunicorn configuration. +for unit in netbox netbox-rq; do + __file /etc/systemd/system/$unit.service \ + --mode 644 --source "$__type/files/$unit.service" \ + --onchange 'systemctl daemon-reload' +done + +# Python worker configuration. +require="__user/netbox" __file /opt/netbox/gunicorn.py \ + --mode 644 --source "$__type/files/gunicorn.py" diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required new file mode 100644 index 0000000..388048a --- /dev/null +++ b/type/__netbox/parameter/required @@ -0,0 +1,5 @@ +version +database +database-password +secret-key +host diff --git a/type/__netbox/singleton b/type/__netbox/singleton new file mode 100644 index 0000000..e69de29 From ab2c826cc3347cc3ff2842480a20dce185cc89d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jul 2020 07:46:45 +0200 Subject: [PATCH 050/332] __netbox: add minimal LDAP support --- type/__netbox/files/ldap_config.py.sh | 27 +++++++++++++++++++++++++ type/__netbox/gencode-remote | 8 ++++++++ type/__netbox/manifest | 29 +++++++++++++++++++++++++++ type/__netbox/parameter/optional | 4 ++++ 4 files changed, 68 insertions(+) create mode 100755 type/__netbox/files/ldap_config.py.sh create mode 100644 type/__netbox/parameter/optional diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh new file mode 100755 index 0000000..88af4b9 --- /dev/null +++ b/type/__netbox/files/ldap_config.py.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +cat << EOF +############################## +# LDAP-backed authentication # +############################## + +import ldap +from django_auth_ldap.config import LDAPSearch + +# Server URI +AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" + +# Set the DN and password for the NetBox service account. +AUTH_LDAP_BIND_DN = "$LDAP_BIND_DN" +AUTH_LDAP_BIND_PASSWORD = "$LDAP_BIND_PASSWORD" + +# If a user's DN is producible from their username, we don't need to search. +AUTH_LDAP_USER_DN_TEMPLATE = "$LDAP_USER_DN_TEMPLATE" + +# You can map user attributes to Django attributes as so. +AUTH_LDAP_USER_ATTR_MAP = { + "first_name": "givenName", + "last_name": "sn", + "email": "mail" +} +EOF diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index a52b110..e3157a9 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -27,18 +27,26 @@ tar xf '$archive' # Save cdist-upload configuration file. mkdir -p "\$tmpdir" cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" +cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' +cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. virtualenv /opt/netbox/venv # Install python dependencies. /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" +EOF +if [ -f "$__object/parameter/ldap-server" ]; then + echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" +fi + +cat << EOF # Set final permissions. chown -R netbox /opt/netbox diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 64a0a55..2976686 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -10,6 +10,11 @@ case "$os" in __package $pkg done + if [ -f "$__object/parameter/ldap-server" ]; then + for pkg in libldap2-dev libsasl2-dev libssl-dev; do + __package $pkg + done + fi ;; *) printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 @@ -23,18 +28,42 @@ export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") export ALLOWED_HOST=$(cat "$__object/parameter/host") export SECRET_KEY=$(cat "$__object/parameter/secret-key") +if [ -f "$__object/parameter/ldap-server" ]; then + export LDAP_SERVER=$(cat "$__object/parameter/ldap-server") +fi + +if [ -f "$__object/parameter/ldap-bind-dn" ]; then + export LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") +fi + +if [ -f "$__object/parameter/ldap-bind-password" ]; then + export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") +fi + +if [ -f "$__object/parameter/ldap-user-dn-template" ]; then + export LDAP_USER_DN_TEMPLATE=$(cat "$__object/parameter/ldap-user-dn-template") +fi + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home # Generate and upload netbox configuration. mkdir -p "$__object/files" "$__type/files/configuration.py.sh" > "$__object/files/configuration.py" +"$__type/files/ldap_config.py.sh" > "$__object/files/ldap_config.py" require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents require="__directory/opt/netbox/netbox/cdist " __file \ /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ --source "$__object/files/configuration.py" +if [ -f "$__object/parameter/ldap-server" ]; then + require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + --source "$__object/files/ldap_config.py" +fi + + # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do __file /etc/systemd/system/$unit.service \ diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional new file mode 100644 index 0000000..c946d73 --- /dev/null +++ b/type/__netbox/parameter/optional @@ -0,0 +1,4 @@ +ldap-server +ldap-bind-dn +ldap-bind-password +ldap-user-dn-template From ef7a214dae108351e1e2ae66a51966a639ee90d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jul 2020 08:30:09 +0200 Subject: [PATCH 051/332] __netbox: support basic LDAP group mapping --- type/__netbox/files/ldap_config.py.sh | 40 +++++++++++++++++++++++++-- type/__netbox/manifest | 16 +++++++++-- type/__netbox/parameter/optional | 5 +++- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 88af4b9..901d6f5 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -6,7 +6,7 @@ cat << EOF ############################## import ldap -from django_auth_ldap.config import LDAPSearch +from django_auth_ldap.config import LDAPSearch, PosixGroupType # Server URI AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" @@ -15,8 +15,10 @@ AUTH_LDAP_SERVER_URI = "$LDAP_SERVER" AUTH_LDAP_BIND_DN = "$LDAP_BIND_DN" AUTH_LDAP_BIND_PASSWORD = "$LDAP_BIND_PASSWORD" -# If a user's DN is producible from their username, we don't need to search. -AUTH_LDAP_USER_DN_TEMPLATE = "$LDAP_USER_DN_TEMPLATE" +# Search for user entry. +AUTH_LDAP_USER_SEARCH = LDAPSearch("$LDAP_USER_BASE", + ldap.SCOPE_SUBTREE, + "(uid=%(user)s)") # You can map user attributes to Django attributes as so. AUTH_LDAP_USER_ATTR_MAP = { @@ -25,3 +27,35 @@ AUTH_LDAP_USER_ATTR_MAP = { "email": "mail" } EOF + +if [ "$LDAP_GROUP_BASE" != "" ]; then + cat << EOF + +# This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group +# hierarchy. +AUTH_LDAP_GROUP_SEARCH = LDAPSearch("$LDAP_GROUP_BASE", ldap.SCOPE_SUBTREE, + "(objectClass=posixGroup)") +AUTH_LDAP_GROUP_TYPE = PosixGroupType() + +# Mirror LDAP group assignments. +AUTH_LDAP_MIRROR_GROUPS = True +EOF + + if [ "$LDAP_REQUIRE_GROUP" != "" ]; then + cat << EOF + +# Define a group required to login. +AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" +EOF + fi + + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + cat << EOF + +# Define special user types using groups. Exercise great caution when assigning superuser status. +AUTH_LDAP_USER_FLAGS_BY_GROUP = { + "is_superuser": "$LDAP_SUPERUSER_GROUP", +} +EOF + fi +fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2976686..32c85e3 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -40,8 +40,20 @@ if [ -f "$__object/parameter/ldap-bind-password" ]; then export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") fi -if [ -f "$__object/parameter/ldap-user-dn-template" ]; then - export LDAP_USER_DN_TEMPLATE=$(cat "$__object/parameter/ldap-user-dn-template") +if [ -f "$__object/parameter/ldap-user-base" ]; then + export LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") +fi + +if [ -f "$__object/parameter/ldap-group-base" ]; then + export LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") +fi + +if [ -f "$__object/parameter/ldap-require-group" ]; then + export LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") +fi + +if [ -f "$__object/parameter/ldap-superuser-group" ]; then + export LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") fi # Create system user used to run netbox. diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index c946d73..290c98b 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,4 +1,7 @@ ldap-server ldap-bind-dn ldap-bind-password -ldap-user-dn-template +ldap-user-base +ldap-group-base +ldap-require-group +ldap-superuser-group From e2f01bbb3b3c297cd17400437eb8bd13f4697a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 22 Jul 2020 17:14:06 +0200 Subject: [PATCH 052/332] __matrix_riot: rename to __matrix_element --- .../files/config.json.sh | 2 +- .../gencode-remote | 2 +- .../man.rst | 20 +++++++++---------- .../manifest | 0 .../parameter/boolean | 0 type/__matrix_element/parameter/default/brand | 1 + .../parameter/default/cookie_policy_url | 0 .../parameter/default/default_country_code | 0 .../parameter/default/default_server_name | 0 .../parameter/default/default_server_url | 0 .../parameter/default/owner | 0 .../parameter/default/privacy_policy_url | 1 + .../parameter/default/room_directory_servers | 0 .../parameter/optional | 0 .../parameter/optional_multiple | 0 .../parameter/required | 0 type/__matrix_riot/parameter/default/brand | 1 - .../parameter/default/privacy_policy_url | 1 - 18 files changed, 14 insertions(+), 14 deletions(-) rename type/{__matrix_riot => __matrix_element}/files/config.json.sh (96%) rename type/{__matrix_riot => __matrix_element}/gencode-remote (97%) rename type/{__matrix_riot => __matrix_element}/man.rst (73%) rename type/{__matrix_riot => __matrix_element}/manifest (100%) rename type/{__matrix_riot => __matrix_element}/parameter/boolean (100%) create mode 100644 type/__matrix_element/parameter/default/brand rename type/{__matrix_riot => __matrix_element}/parameter/default/cookie_policy_url (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_country_code (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_server_name (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/default_server_url (100%) rename type/{__matrix_riot => __matrix_element}/parameter/default/owner (100%) create mode 100644 type/__matrix_element/parameter/default/privacy_policy_url rename type/{__matrix_riot => __matrix_element}/parameter/default/room_directory_servers (100%) rename type/{__matrix_riot => __matrix_element}/parameter/optional (100%) rename type/{__matrix_riot => __matrix_element}/parameter/optional_multiple (100%) rename type/{__matrix_riot => __matrix_element}/parameter/required (100%) delete mode 100644 type/__matrix_riot/parameter/default/brand delete mode 100644 type/__matrix_riot/parameter/default/privacy_policy_url diff --git a/type/__matrix_riot/files/config.json.sh b/type/__matrix_element/files/config.json.sh similarity index 96% rename from type/__matrix_riot/files/config.json.sh rename to type/__matrix_element/files/config.json.sh index 6cd1648..9791f38 100755 --- a/type/__matrix_riot/files/config.json.sh +++ b/type/__matrix_element/files/config.json.sh @@ -39,7 +39,7 @@ EOF fi cat << EOF - "welcomeBackgroundUrl": "themes/riot/img/backgrounds/valley.jpg" + "welcomeBackgroundUrl": "themes/element/img/backgrounds/lake.jpg" EOF echo '},' } diff --git a/type/__matrix_riot/gencode-remote b/type/__matrix_element/gencode-remote similarity index 97% rename from type/__matrix_riot/gencode-remote rename to type/__matrix_element/gencode-remote index dbc886c..e643976 100755 --- a/type/__matrix_riot/gencode-remote +++ b/type/__matrix_element/gencode-remote @@ -46,7 +46,7 @@ cd \$tmpdir curl -L '$url' > $archive tar xf $archive -# Backup files deployed by __matrix_riot. +# Backup files deployed by __matrix_element. mkdir -p \$custom_files_dir for file in $INSTALL_DIR/cdist/*; do cp "\$file" "\$custom_files_dir" diff --git a/type/__matrix_riot/man.rst b/type/__matrix_element/man.rst similarity index 73% rename from type/__matrix_riot/man.rst rename to type/__matrix_element/man.rst index 2fe8eae..c3e0d5a 100644 --- a/type/__matrix_riot/man.rst +++ b/type/__matrix_element/man.rst @@ -1,23 +1,23 @@ -cdist-type__matrix_riot(7) +cdist-type__matrix_element(7) ====================== NAME ---- -cdist-type__matrix_riot - Install and configure Riot, a web Matrix client. +cdist-type__matrix_element - Install and configure Element, a web Matrix client. DESCRIPTION ----------- -This type install and configure the Riot web client. +This type install and configure the Element web client. REQUIRED PARAMETERS ------------------- install_dir - Root directory of Riot's static files. + Root directory of Element's static files. version - Release of Riot to install. + Release of Element to install. OPTIONAL PARAMETERS ------------------- @@ -31,20 +31,20 @@ owner Owner of the deployed files, passed to `chown`. Defaults to 'root'. brand - Web UI branding, defaults to 'Riot'. + Web UI branding, defaults to 'Element'. default_country_code ISO 3166 alpha2 country code to use when showing country selectors, such as phone number inputs. Defaults to GB. privacy_policy_url - Defaults to 'https://riot.im/privacy'. + Defaults to 'https://element.io/privacy'. cookie_policy_url - Defaults to 'https://matrix.org/docs/guides/riot_im_cookie_policy'. + Defaults to 'https://matrix.org/docs/guides/element_im_cookie_policy'. jitsi_domain - Domain name of preferred Jitsi instance (default is jitsi.riot.im). This is + Domain name of preferred Jitsi instance (default is jitsi.element.im). This is used whenever a user clicks on the voice/video call buttons. homepage @@ -66,7 +66,7 @@ EXAMPLES .. code-block:: sh - __matrix_riot my-riot --install_dir /var/www/riot-web --version 1.5.6 + __matrix_element my-element --install_dir /var/www/element-web --version 1.5.6 SEE ALSO diff --git a/type/__matrix_riot/manifest b/type/__matrix_element/manifest similarity index 100% rename from type/__matrix_riot/manifest rename to type/__matrix_element/manifest diff --git a/type/__matrix_riot/parameter/boolean b/type/__matrix_element/parameter/boolean similarity index 100% rename from type/__matrix_riot/parameter/boolean rename to type/__matrix_element/parameter/boolean diff --git a/type/__matrix_element/parameter/default/brand b/type/__matrix_element/parameter/default/brand new file mode 100644 index 0000000..907f907 --- /dev/null +++ b/type/__matrix_element/parameter/default/brand @@ -0,0 +1 @@ +Element diff --git a/type/__matrix_riot/parameter/default/cookie_policy_url b/type/__matrix_element/parameter/default/cookie_policy_url similarity index 100% rename from type/__matrix_riot/parameter/default/cookie_policy_url rename to type/__matrix_element/parameter/default/cookie_policy_url diff --git a/type/__matrix_riot/parameter/default/default_country_code b/type/__matrix_element/parameter/default/default_country_code similarity index 100% rename from type/__matrix_riot/parameter/default/default_country_code rename to type/__matrix_element/parameter/default/default_country_code diff --git a/type/__matrix_riot/parameter/default/default_server_name b/type/__matrix_element/parameter/default/default_server_name similarity index 100% rename from type/__matrix_riot/parameter/default/default_server_name rename to type/__matrix_element/parameter/default/default_server_name diff --git a/type/__matrix_riot/parameter/default/default_server_url b/type/__matrix_element/parameter/default/default_server_url similarity index 100% rename from type/__matrix_riot/parameter/default/default_server_url rename to type/__matrix_element/parameter/default/default_server_url diff --git a/type/__matrix_riot/parameter/default/owner b/type/__matrix_element/parameter/default/owner similarity index 100% rename from type/__matrix_riot/parameter/default/owner rename to type/__matrix_element/parameter/default/owner diff --git a/type/__matrix_element/parameter/default/privacy_policy_url b/type/__matrix_element/parameter/default/privacy_policy_url new file mode 100644 index 0000000..37fa4bc --- /dev/null +++ b/type/__matrix_element/parameter/default/privacy_policy_url @@ -0,0 +1 @@ +https://element.io/privacy diff --git a/type/__matrix_riot/parameter/default/room_directory_servers b/type/__matrix_element/parameter/default/room_directory_servers similarity index 100% rename from type/__matrix_riot/parameter/default/room_directory_servers rename to type/__matrix_element/parameter/default/room_directory_servers diff --git a/type/__matrix_riot/parameter/optional b/type/__matrix_element/parameter/optional similarity index 100% rename from type/__matrix_riot/parameter/optional rename to type/__matrix_element/parameter/optional diff --git a/type/__matrix_riot/parameter/optional_multiple b/type/__matrix_element/parameter/optional_multiple similarity index 100% rename from type/__matrix_riot/parameter/optional_multiple rename to type/__matrix_element/parameter/optional_multiple diff --git a/type/__matrix_riot/parameter/required b/type/__matrix_element/parameter/required similarity index 100% rename from type/__matrix_riot/parameter/required rename to type/__matrix_element/parameter/required diff --git a/type/__matrix_riot/parameter/default/brand b/type/__matrix_riot/parameter/default/brand deleted file mode 100644 index e8095bb..0000000 --- a/type/__matrix_riot/parameter/default/brand +++ /dev/null @@ -1 +0,0 @@ -Riot diff --git a/type/__matrix_riot/parameter/default/privacy_policy_url b/type/__matrix_riot/parameter/default/privacy_policy_url deleted file mode 100644 index 4cdd12c..0000000 --- a/type/__matrix_riot/parameter/default/privacy_policy_url +++ /dev/null @@ -1 +0,0 @@ -https://riot.im/privacy From 5960356e7b0e540ed49af17ec05a0ba608dfaae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 10 Jul 2020 10:41:20 +0200 Subject: [PATCH 053/332] __unbound: add --local_data flag --- type/__unbound/files/unbound.conf.sh | 9 +++++++++ type/__unbound/man.rst | 4 ++++ type/__unbound/manifest | 5 +++++ type/__unbound/parameter/optional_multiple | 1 + 4 files changed, 19 insertions(+) diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index 46d2824..2f7ecff 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -18,6 +18,14 @@ generate_forward_addr() { done } +generate_local_data() { + IFS=' +' + for entry in $LOCAL_DATA; do + echo " local-data: \"$entry\"" + done +} + cat << EOF # # THIS FILE HAS BEEN GENERATED BY CDIST, DO NOT EDIT BY HAND. @@ -720,6 +728,7 @@ $(generate_access_control) # (this makes example.com, www.example.com, etc, all go to 192.0.2.3) # local-zone: "example.com" redirect # local-data: "example.com A 192.0.2.3" +$(generate_local_data) # # Shorthand to make PTR records, "IPv4 name" or "IPv6 name". # You can also add PTR records using local-data directly, but then diff --git a/type/__unbound/man.rst b/type/__unbound/man.rst index 679e601..316d011 100644 --- a/type/__unbound/man.rst +++ b/type/__unbound/man.rst @@ -34,6 +34,10 @@ access_control rc_interface Address or path to socket used for remote control (see `--enable_control`. Defaults to `127.0.0.1`). +local_data + Configure local data, which is served in reply to queries for it. Can be + specified multiple times. + BOOLEAN PARAMETERS ------------------ disable-ip4 diff --git a/type/__unbound/manifest b/type/__unbound/manifest index 346f30c..5d6b50f 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -54,6 +54,11 @@ if [ -f "$__object/parameter/rc_interface" ]; then export RC_INTERFACE fi +if [ -f "$__object/parameter/local_data" ]; then + LOCAL_DATA=$(cat "$__object/parameter/local_data") + export LOCAL_DATA +fi + # Boolean parameters: if [ -f "$__object/parameter/disable_ip4" ] && \ [ -f "$__object/parameter/disable_ip6" ]; then diff --git a/type/__unbound/parameter/optional_multiple b/type/__unbound/parameter/optional_multiple index ecacd1d..3fe7eca 100644 --- a/type/__unbound/parameter/optional_multiple +++ b/type/__unbound/parameter/optional_multiple @@ -1,2 +1,3 @@ access_control +local_data interface From 7eecc99c73f2304c5f4837dd9324a5d925256782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 27 Jul 2020 14:53:45 +0200 Subject: [PATCH 054/332] __unbound: fix typo in gencode-remote / service start --- type/__unbound/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote index d12a395..ba6d92b 100755 --- a/type/__unbound/gencode-remote +++ b/type/__unbound/gencode-remote @@ -11,6 +11,6 @@ cat << EOF if pgrep unbound; then service unbound reload else - service unbounb start + service unbound start fi EOF From f73bf49e4b510b426dce4c04d1d63e08455f1ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jul 2020 14:23:30 +0200 Subject: [PATCH 055/332] Remove manual / outdated CHANGELOG --- CHANGELOG.md | 6 ------ README.md | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index df26f7e..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,6 +0,0 @@ -# cdist-contrib changes - -* 2020-06-07: New type: __unbound_exporter (Timothée Floure) -* 2020-06-07: Extended type: wire remote control configuration for __unbond (Timothée Floure) -* 2020-06-03: New type: __unbound (Timothée Floure) -* 2020-04-28: New type: __find_exec (Ander Punnar) diff --git a/README.md b/README.md index ef4b2c3..4231de2 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ tool with community-maitained types which are either too specific to fit/be maintained in cdist itself or were not accepted in code cdist but could still be useful. -This project does not have releases and is continously updated: see -`CHANGELOG.md` for details. +This project does not have releases and is continously updated: see git history +for change log. ## Using cdist-contrib From bf5f85068d8614f97e190e5115e34f5442ba7924 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 7 Aug 2020 10:39:38 +0200 Subject: [PATCH 056/332] Add a type to create a borg repository. --- type/__borg_repo/gencode-remote | 36 ++++++++++++++++ type/__borg_repo/man.rst | 43 +++++++++++++++++++ type/__borg_repo/manifest | 14 ++++++ type/__borg_repo/parameter/boolean | 1 + type/__borg_repo/parameter/default/encryption | 1 + type/__borg_repo/parameter/optional | 1 + type/__borg_repo/parameter/required | 1 + 7 files changed, 97 insertions(+) create mode 100644 type/__borg_repo/gencode-remote create mode 100644 type/__borg_repo/man.rst create mode 100644 type/__borg_repo/manifest create mode 100644 type/__borg_repo/parameter/boolean create mode 100644 type/__borg_repo/parameter/default/encryption create mode 100644 type/__borg_repo/parameter/optional create mode 100644 type/__borg_repo/parameter/required diff --git a/type/__borg_repo/gencode-remote b/type/__borg_repo/gencode-remote new file mode 100644 index 0000000..542bf5f --- /dev/null +++ b/type/__borg_repo/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh + +passphrase= +appendonly= + +case "$(cat "${__object:?}/parameter/encryption")" in + none) + enc=none + ;; + repokey) + enc=repokey + if [ -f "${__object:?}/parameter/passphrase" ]; + then + passphrase="$(cat "${__object:?}/parameter/passphrase")" + else + echo "__borg_repo cannot use repokey encryption with no passphrase. Aborting." >&2; + exit 1; + fi + ;; + *) + echo "$enc is not a known encryption mode for __borg_repo. Aborting." >&2 + exit 1; +esac + +if [ -f "${__object:?}/parameter/append-only" ]; +then + appendonly='--append-only' +fi + +cat <<- EOF + if ! borg check --repository-only 1>&2 2>/dev/null "/${__object_id:?}"; + then + BORG_NEW_PASSPHRASE=$passphrase borg init -e ${enc:?} $appendonly /${__object_id:?} + fi +EOF + diff --git a/type/__borg_repo/man.rst b/type/__borg_repo/man.rst new file mode 100644 index 0000000..38ab0c9 --- /dev/null +++ b/type/__borg_repo/man.rst @@ -0,0 +1,43 @@ +cdist-type__borg_repo(7) +======================== + +NAME +---- +cdist-type__borg_repo - Configure a borg repository on host + + +DESCRIPTION +----------- + +Initializes a borg repository at the location specified in the +`${__object_id}`. Nothing is done if the repository already exists. + +Currently, only `none` and `repokey` are supported as encryption modes; +`repokey` requires the `passphrase` argument to be given. The default is +`none`. + +REQUIRED PARAMETERS +------------------- +encryption + The encryption to use. + +OPTIONAL PARAMETERS +------------------- +passphrase + The passphrase to encrypt the keyfile with. + +BOOLEAN PARAMETERS +------------------ +append-only + If the repository is append-only + +AUTHORS +------- +Joachim Desroches + +COPYING +------- +Copyright \(C) 2020 Joachim Desroches. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__borg_repo/manifest b/type/__borg_repo/manifest new file mode 100644 index 0000000..fe18c9c --- /dev/null +++ b/type/__borg_repo/manifest @@ -0,0 +1,14 @@ +#!/bin/sh + +os="$(cat "${__global:?}"/explorer/os)" + +case "$os" in + "alpine") + borg_package=borgbackup + ;; + *) + echo "__borg_repo is not yet implemented for os $os. Aborting." >&2; + exit 1; +esac + +__package "$borg_package" diff --git a/type/__borg_repo/parameter/boolean b/type/__borg_repo/parameter/boolean new file mode 100644 index 0000000..f8ee7c6 --- /dev/null +++ b/type/__borg_repo/parameter/boolean @@ -0,0 +1 @@ +append-only diff --git a/type/__borg_repo/parameter/default/encryption b/type/__borg_repo/parameter/default/encryption new file mode 100644 index 0000000..621e94f --- /dev/null +++ b/type/__borg_repo/parameter/default/encryption @@ -0,0 +1 @@ +none diff --git a/type/__borg_repo/parameter/optional b/type/__borg_repo/parameter/optional new file mode 100644 index 0000000..f63b25b --- /dev/null +++ b/type/__borg_repo/parameter/optional @@ -0,0 +1 @@ +passphrase diff --git a/type/__borg_repo/parameter/required b/type/__borg_repo/parameter/required new file mode 100644 index 0000000..a5465f8 --- /dev/null +++ b/type/__borg_repo/parameter/required @@ -0,0 +1 @@ +encryption From 47584d6da64b86a266d950a61879ccb82268d2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 08:58:29 +0200 Subject: [PATCH 057/332] __netbox: use __systemd_unit, do not recreate existing directory --- type/__netbox/gencode-remote | 1 - type/__netbox/manifest | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index e3157a9..122df96 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -25,7 +25,6 @@ curl -L '$url' > '$archive' tar xf '$archive' # Save cdist-upload configuration file. -mkdir -p "\$tmpdir" cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 32c85e3..2503bb4 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -78,9 +78,9 @@ fi # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do - __file /etc/systemd/system/$unit.service \ - --mode 644 --source "$__type/files/$unit.service" \ - --onchange 'systemctl daemon-reload' + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled done # Python worker configuration. From 240b8f1615d292ddca081304ce930792e330fc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 09:01:05 +0200 Subject: [PATCH 058/332] __netbox: make shellcheck happy --- type/__netbox/manifest | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2503bb4..fc9192c 100644 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -23,37 +23,48 @@ case "$os" in ;; esac -export DATABASE_NAME=$(cat "$__object/parameter/database") -export DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") -export ALLOWED_HOST=$(cat "$__object/parameter/host") -export SECRET_KEY=$(cat "$__object/parameter/secret-key") +DATABASE_NAME=$(cat "$__object/parameter/database") +export DATABASE_NAME +DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") +export DATABASE_PASSWORD +ALLOWED_HOST=$(cat "$__object/parameter/host") +export ALLOWED_HOST +SECRET_KEY=$(cat "$__object/parameter/secret-key") +export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then - export LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + export LDAP_SERVER fi if [ -f "$__object/parameter/ldap-bind-dn" ]; then - export LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + export LDAP_BIND_DN fi if [ -f "$__object/parameter/ldap-bind-password" ]; then - export LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + export LDAP_BIND_PASSWORD fi if [ -f "$__object/parameter/ldap-user-base" ]; then - export LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + export LDAP_USER_BASE fi if [ -f "$__object/parameter/ldap-group-base" ]; then - export LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + export LDAP_GROUP_BASE fi if [ -f "$__object/parameter/ldap-require-group" ]; then - export LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + export LDAP_REQUIRE_GROUP fi if [ -f "$__object/parameter/ldap-superuser-group" ]; then - export LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + export LDAP_SUPERUSER_GROUP fi # Create system user used to run netbox. From dfd9f108b1ba11b91c3e198fde60f86e5bf34006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 12 Aug 2020 09:15:18 +0200 Subject: [PATCH 059/332] __netbox: add minimal manpage --- type/__netbox/man.rst | 94 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 type/__netbox/man.rst diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst new file mode 100644 index 0000000..25aeb26 --- /dev/null +++ b/type/__netbox/man.rst @@ -0,0 +1,94 @@ +cdist-type__netbox(7) +===================== + +NAME +---- +cdist-type__netbox - Install and configure NetBox + + +DESCRIPTION +----------- +This (singleton) type installs and configures a NetBox instance, a web +application to help manage and document computer networks + + +REQUIRED PARAMETERS +------------------- +version + NetBox version to be installed. + +database + PostgreSQL database name. + +database-password + PostgreSQL database password. + +secret-key + Random secret key of at least 50 alphanumeric characters. This key must be + unique to this installation and must not be shared outside the local + system. + +host + Hostname (domain or IP address) on which the application is served. + +OPTIONAL PARAMETERS +------------------- +ldap-server + LDAP server URI. Enables LDAP-backed authentication if specified. + +ldap-bind-dn + DN for the NetBox service account. Required for LDAP authentication. + +ldap-bind-password + Password for the NetBox service account. Required for LDAP authentication. + +ldap-user-base + Base used for searching user entries. Required for LDAP authentication. + +ldap-group-base + Base used for searching group entries. + +ldap-require-group + Group required to login. + +ldap-superuser-group + Make members of this groups superusers. + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __netbox --version 2.8.7 --database netbox \ + --database-password "secretsecretsecret" \ + --secret-key "secretsecretsecret" \ + --host "${__target_host:?}" \ + --ldap-server "ldaps://ldap.domain.tld" \ + --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ + --ldap-bind-password "secretsecretsecret" \ + --ldap-user-base "ou=users,dc=domain,dc=tld" \ + --ldap-group-base "ou=groups,dc=domain,dc=tld" \ + --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ + --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + + +SEE ALSO +-------- +- `NetBox documentation `_ + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2020 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. From efab6ac23ec52e8d5dad8cda9df9b050d66a23f1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 16 Aug 2020 15:18:47 +0200 Subject: [PATCH 060/332] __netbox: add version memorisation This saves the installed version and checks if a new version must be installed. If no update required, it updates the configuration files if required. --- type/__netbox/explorer/version | 5 ++++ type/__netbox/gencode-remote | 42 ++++++++++++++++++++++++++++------ type/__netbox/man.rst | 10 ++++++++ type/__netbox/manifest | 0 4 files changed, 50 insertions(+), 7 deletions(-) create mode 100755 type/__netbox/explorer/version mode change 100644 => 100755 type/__netbox/manifest diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version new file mode 100755 index 0000000..a9ed590 --- /dev/null +++ b/type/__netbox/explorer/version @@ -0,0 +1,5 @@ +#!/bin/sh + +# output version if exist +version_path="/opt/netbox/netbox/cdist/version" +if [ -f "$version_path" ]; then cat "$version_path"; fi diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 122df96..48463f7 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -1,5 +1,8 @@ #!/bin/sh +echo "set -e" + +old_version="$(cat "$__object/explorer/version")" VERSION=$(cat "$__object/parameter/version") src="netbox-$VERSION" @@ -7,9 +10,8 @@ archive="v$VERSION.tar.gz" url="https://github.com/netbox-community/netbox/archive/$archive" install_dir=/opt/netbox/netbox -cat << EOF -set -e - +if [ "$VERSION" != "$old_version" ]; then + cat << EOF # Ensure that coreutils is installed. if [ ! -x \$(which mktemp) ]; then echo "mktemp is not available on the remote host." >&2 @@ -41,11 +43,11 @@ virtualenv /opt/netbox/venv /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" EOF -if [ -f "$__object/parameter/ldap-server" ]; then - echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" -fi + if [ -f "$__object/parameter/ldap-server" ]; then + echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + fi -cat << EOF + cat << EOF # Set final permissions. chown -R netbox /opt/netbox @@ -59,7 +61,33 @@ sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collect cd / rm -r "\$tmpdir" +# Save version after successful installation +printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' +EOF + + # meta + printf "%s %s\n" "installed" "$VERSION" >> "$__messages_out" + changes=yes + +# check if configuration changed +elif grep -q "^__file/opt/netbox/" "$__messages_in"; then + cat <> "$__messages_out" + changes=yes +fi + + +# check for changes +if [ "$changes" = "yes" ]; then + cat < Date: Sun, 16 Aug 2020 15:31:38 +0200 Subject: [PATCH 061/332] __netbox: add manpage explanation of the type --- type/__netbox/man.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 84f1681..e41b099 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -9,7 +9,11 @@ cdist-type__netbox - Install and configure NetBox DESCRIPTION ----------- This (singleton) type installs and configures a NetBox instance, a web -application to help manage and document computer networks +application to help manage and document computer networks. + +It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. +Netbox will be run via `gnuicorn` as WSGI service. It setup systemd unit files +for the services `netbox` and `netbox-rq`. REQUIRED PARAMETERS From 0281b2c8046832d067dfa3be0da8702ef590faf4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 20 Aug 2020 17:53:25 +0200 Subject: [PATCH 062/332] __netbox: no ldap config if no ldap parameters The ldap config file will be empty (only comments) if no parameter name set matches `^ldap-*`. It still generate a ldap config if one or more ldap parameters are set, but not all basic parameters. --- type/__netbox/files/ldap_config.py.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 901d6f5..3e7aaa6 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,5 +1,19 @@ #!/bin/sh +# no configuration if there are no ldap parameters +if ls -1qA "$__object/parameter"/ldap-* | grep -q .; then + # skip + cat << EOF +############################## +# LDAP-backed authentication # +############################## + +# no options set +EOF + exit 0 +fi + + cat << EOF ############################## # LDAP-backed authentication # From a05cb214e17303068649aab3749099e4a06ceef1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 20 Aug 2020 19:37:09 +0200 Subject: [PATCH 063/332] __netbox: fix shellcheck by changing parameter checks The working `ls | grep .` like solution was dropped based on the shellcheck warning. Rather a solution with `find(1)` is used. As alternative, it could be simply checked if the glob succeeds or not by test all given arguments if there are existing files. It was dropped in question if it's really better than the `find(1)` solution. --- type/__netbox/files/ldap_config.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index 3e7aaa6..bc494fb 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if ls -1qA "$__object/parameter"/ldap-* | grep -q .; then +if [ "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then # skip cat << EOF ############################## From e6e6dbcd0029639539ce7fe2d8a6fb7d753ad727 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 21 Aug 2020 23:50:24 +0200 Subject: [PATCH 064/332] __netbox: added some netbox configuration options Those who might be useful are added .. --- type/__netbox/files/configuration.py.sh | 64 ++++++++++++------ type/__netbox/man.rst | 65 ++++++++++++++++++- type/__netbox/manifest | 65 +++++++++++++++++++ type/__netbox/parameter/boolean | 4 ++ type/__netbox/parameter/default/basepath | 1 + .../parameter/default/redis-dbid-offset | 1 + type/__netbox/parameter/default/redis-host | 1 + .../__netbox/parameter/default/redis-password | 1 + type/__netbox/parameter/default/redis-port | 1 + .../parameter/default/smtp-from-email | 1 + type/__netbox/parameter/default/smtp-host | 1 + type/__netbox/parameter/default/smtp-password | 1 + type/__netbox/parameter/default/smtp-port | 1 + type/__netbox/parameter/default/smtp-user | 1 + type/__netbox/parameter/optional | 12 ++++ 15 files changed, 200 insertions(+), 20 deletions(-) create mode 100644 type/__netbox/parameter/boolean create mode 100644 type/__netbox/parameter/default/basepath create mode 100644 type/__netbox/parameter/default/redis-dbid-offset create mode 100644 type/__netbox/parameter/default/redis-host create mode 100644 type/__netbox/parameter/default/redis-password create mode 100644 type/__netbox/parameter/default/redis-port create mode 100644 type/__netbox/parameter/default/smtp-from-email create mode 100644 type/__netbox/parameter/default/smtp-host create mode 100644 type/__netbox/parameter/default/smtp-password create mode 100644 type/__netbox/parameter/default/smtp-port create mode 100644 type/__netbox/parameter/default/smtp-user diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index b7e1597..e621845 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -29,26 +29,26 @@ DATABASE = { # to use two separate database IDs. REDIS = { 'tasks': { - 'HOST': 'localhost', - 'PORT': 6379, + 'HOST': '$REDIS_HOST', + 'PORT': $REDIS_PORT, # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], # 'SENTINEL_SERVICE': 'netbox', - 'PASSWORD': '', - 'DATABASE': 0, + 'PASSWORD': '$REDIS_PASSWORD', + 'DATABASE': $((REDIS_DBID_OFFSET + 0)), 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'SSL': $REDIS_SSL, }, 'caching': { - 'HOST': 'localhost', - 'PORT': 6379, + 'HOST': '$REDIS_HOST', + 'PORT': $REDIS_PORT, # Comment out \`HOST\` and \`PORT\` lines and uncomment the following if using Redis Sentinel # 'SENTINELS': [('mysentinel.redis.example.com', 6379)], # 'SENTINEL_SERVICE': 'netbox', - 'PASSWORD': '', - 'DATABASE': 1, + 'PASSWORD': '$REDIS_PASSWORD', + 'DATABASE': $((REDIS_DBID_OFFSET + 1)), 'DEFAULT_TIMEOUT': 300, - 'SSL': False, + 'SSL': $REDIS_SSL, } } @@ -86,7 +86,7 @@ BANNER_LOGIN = '' # Base URL path if accessing NetBox within a directory. For example, if installed at http://example.com/netbox/, set: # BASE_PATH = 'netbox/' -BASE_PATH = '' +BASE_PATH = '$BASEPATH' # Cache timeout in seconds. Set to 0 to dissable caching. Defaults to 900 (15 minutes) CACHE_TIMEOUT = 900 @@ -112,14 +112,14 @@ DEBUG = False # Email settings EMAIL = { - 'SERVER': 'localhost', - 'PORT': 25, - 'USERNAME': '', - 'PASSWORD': '', - 'USE_SSL': False, - 'USE_TLS': False, + 'SERVER': '$SMTP_HOST', + 'PORT': $SMTP_PORT, + 'USERNAME': '$SMTP_USER', + 'PASSWORD': '$SMTP_PASSWORD', + 'USE_SSL': $SMTP_USE_SSL, + 'USE_TLS': $SMTP_USE_TLS, 'TIMEOUT': 10, # seconds - 'FROM_EMAIL': '', + 'FROM_EMAIL': '$SMTP_FROM_EMAIL', } # Enforcement of unique IP space can be toggled on a per-VRF basis. To enforce unique IP space within the global table @@ -134,12 +134,38 @@ EXEMPT_VIEW_PERMISSIONS = [ # 'ipam.prefix', ] +EOF + +if [ "$HTTP_PROXY" != "" ] || [ "$HTTPS_PROXY" != "" ]; then + cat << EOF +# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). +HTTP_PROXIES = { +EOF + if [ "$HTTP_PROXY" != "" ]; then + cat << EOF + 'http': '$HTTP_PROXY', +EOF + fi + if [ "$HTTPS_PROXY" != "" ]; then + cat << EOF + 'https': '$HTTPS_PROXY', +EOF + fi + cat << EOF +} +EOF + +else + cat << EOF # HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). # HTTP_PROXIES = { # 'http': 'http://10.10.1.10:3128', # 'https': 'http://10.10.1.10:1080', # } +EOF +fi +cat << EOF # IP addresses recognized as internal to the system. The debugging toolbar will be available only to clients accessing # NetBox from an internal IP. INTERNAL_IPS = ('127.0.0.1', '::1') @@ -150,7 +176,7 @@ LOGGING = {} # Setting this to True will permit only authenticated users to access any part of NetBox. By default, anonymous users # are permitted to access most data in NetBox (excluding secrets) but not make any changes. -LOGIN_REQUIRED = False +LOGIN_REQUIRED = $LOGIN_REQUIRED # The length of time (in seconds) for which a user will remain logged into the web UI before being prompted to # re-authenticate. (Default: 1209600 [14 days]) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index e41b099..3e607b5 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -58,9 +58,62 @@ ldap-require-group ldap-superuser-group Make members of this groups superusers. +redis-host + Redis database hostname. Defaults to ``localhost``. + +redis-port + Redis database port. Defaults to ``6379``. + +redis-password + Redis password. Defaults to empty password. + +redis-dbid-offset + Offset to set the redis database id's. The `tasks` database id is `offset + 0` + and `caching` is `offset + 1`. The offset defaults to ``0``. + +smtp-host + Host of the SMTP email server. Defaults to ``localhost``. + +smtp-port + Port of the SMTP email server. Defaults to ``25``. + +smtp-user + Username to access the SMTP email server. Defaults to empty. + +smtp-password + Password to access the SMTP email server. Defaults to empty. + +smtp-from-email + Email from which NetBox will be sent of. Defaults to empty. + +basepath + Base URL path if accessing netbox within a directory instead of directly the + webroot ``/``. For example, if installed at https://example.com/netbox/, set + the value ``netbox/``. + +http-proxy +https-proxy + Proxy which will be used with any HTTP request like webhooks. + BOOLEAN PARAMETERS ------------------ -None. +redis-ssl + Enables a secure TLS/SSL connection to the redis database. By default, ssl + is disabled. + +smtp-use-tls + Uses TLS to connect to the SMTP email server. `See documentation + ` + SEE ALSO -------- - `NetBox documentation `_ diff --git a/type/__netbox/manifest b/type/__netbox/manifest index fc9192c..1f7f8ed 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -23,6 +23,7 @@ case "$os" in ;; esac + DATABASE_NAME=$(cat "$__object/parameter/database") export DATABASE_NAME DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") @@ -67,6 +68,70 @@ if [ -f "$__object/parameter/ldap-superuser-group" ]; then export LDAP_SUPERUSER_GROUP fi +# have default values +REDIS_HOST="$(cat "$__object/parameter/redis-host")" +export REDIS_HOST +REDIS_PORT="$(cat "$__object/parameter/redis-port")" +export REDIS_PORT +REDIS_PASSWORD="$(cat "$__object/parameter/redis-password")" +export REDIS_PASSWORD +REDIS_DBID_OFFSET="$(cat "$__object/parameter/redis-dbid-offset")" +export REDIS_DBID_OFFSET +if [ -f "$__object/parameter/redis-ssl" ]; then + REDIS_SSL="True" +else + REDIS_SSL="False" +fi +export REDIS_SSL + +SMTP_HOST="$(cat "$__object/parameter/smtp-host")" +export SMTP_HOST +SMTP_PORT="$(cat "$__object/parameter/smtp-port")" +export SMTP_PORT +SMTP_USER="$(cat "$__object/parameter/smtp-user")" +export SMTP_USER +SMTP_PASSWORD="$(cat "$__object/parameter/smtp-password")" +export SMTP_PASSWORD +SMTP_FROM_EMAIL="$(cat "$__object/parameter/smtp-from-email")" +export SMTP_FROM_EMAIL + +if [ -f "$__object/parameter/smtp-use-ssl" ]; then + SMTP_USE_SSL="True" +else + SMTP_USE_SSL="False" +fi +export SMTP_USE_SSL +if [ -f "$__object/parameter/smtp-use-tls" ]; then + if [ "$SMTP_USE_SSL" = "True" ]; then + echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" + exit 2 + fi + SMTP_USE_TLS="True" +else + SMTP_USE_TLS="False" +fi +export SMTP_USE_TLS + +BASEPATH="$(cat "$__object/parameter/basepath")" +export BASEPATH + +if [ -f "$__object/parameter/http-proxy" ]; then + HTTP_PROXY=$(cat "$__object/parameter/http-proxy") + export HTTP_PROXY +fi +if [ -f "$__object/parameter/https-proxy" ]; then + HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") + export HTTPS_PROXY +fi + +if [ -f "$__object/parameter/login-required" ]; then + LOGIN_REQUIRED="True" +else + LOGIN_REQUIRED="False" +fi +export LOGIN_REQUIRED + + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/boolean b/type/__netbox/parameter/boolean new file mode 100644 index 0000000..a94808c --- /dev/null +++ b/type/__netbox/parameter/boolean @@ -0,0 +1,4 @@ +redis-ssl +smtp-use-ssl +smtp-use-tls +login-required diff --git a/type/__netbox/parameter/default/basepath b/type/__netbox/parameter/default/basepath new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/basepath @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/redis-dbid-offset b/type/__netbox/parameter/default/redis-dbid-offset new file mode 100644 index 0000000..573541a --- /dev/null +++ b/type/__netbox/parameter/default/redis-dbid-offset @@ -0,0 +1 @@ +0 diff --git a/type/__netbox/parameter/default/redis-host b/type/__netbox/parameter/default/redis-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/redis-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/redis-password b/type/__netbox/parameter/default/redis-password new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/redis-password @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/redis-port b/type/__netbox/parameter/default/redis-port new file mode 100644 index 0000000..a8c4b8e --- /dev/null +++ b/type/__netbox/parameter/default/redis-port @@ -0,0 +1 @@ +6379 diff --git a/type/__netbox/parameter/default/smtp-from-email b/type/__netbox/parameter/default/smtp-from-email new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-from-email @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/smtp-host b/type/__netbox/parameter/default/smtp-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/smtp-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/smtp-password b/type/__netbox/parameter/default/smtp-password new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-password @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/default/smtp-port b/type/__netbox/parameter/default/smtp-port new file mode 100644 index 0000000..7273c0f --- /dev/null +++ b/type/__netbox/parameter/default/smtp-port @@ -0,0 +1 @@ +25 diff --git a/type/__netbox/parameter/default/smtp-user b/type/__netbox/parameter/default/smtp-user new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/smtp-user @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 290c98b..159c813 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -5,3 +5,15 @@ ldap-user-base ldap-group-base ldap-require-group ldap-superuser-group +redis-host +redis-port +redis-password +redis-dbid-offset +smtp-host +smtp-port +smtp-user +smtp-password +smtp-from-email +basepath +http-proxy +https-proxy From 855843ea34ff896ccc23e08f5e84b6f0bdfb49b5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 09:19:12 +0200 Subject: [PATCH 065/332] __netbox: small code and config handling improvements fixed codestyle and detection if config files need to be copied. --- type/__netbox/gencode-remote | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 48463f7..8533f4a 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -66,26 +66,29 @@ printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' EOF # meta - printf "%s %s\n" "installed" "$VERSION" >> "$__messages_out" + printf "installed %s\n" "$VERSION" >> "$__messages_out" changes=yes # check if configuration changed elif grep -q "^__file/opt/netbox/" "$__messages_in"; then - cat <> "$__messages_out" + printf "configuration\n" >> "$__messages_out" changes=yes fi # check for changes if [ "$changes" = "yes" ]; then - cat < Date: Sat, 22 Aug 2020 14:40:07 +0200 Subject: [PATCH 066/332] __netbox: note + link to find NetBox versions --- type/__netbox/man.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 3e607b5..d91c6e8 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -19,7 +19,9 @@ for the services `netbox` and `netbox-rq`. REQUIRED PARAMETERS ------------------- version - NetBox version to be installed. + NetBox version to be installed. You can find the correct and newest version + on GitHub at the NetBox project page under + "`Releases `_". database PostgreSQL database name. From 987393942b9acc10e743c60b264f08dc203877bb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 15:53:24 +0200 Subject: [PATCH 067/332] __netbox: add more database parameters --- type/__netbox/files/configuration.py.sh | 4 ++-- type/__netbox/man.rst | 9 +++++++++ type/__netbox/manifest | 7 +++++++ type/__netbox/parameter/default/database-host | 1 + type/__netbox/parameter/default/database-port | 1 + type/__netbox/parameter/optional | 2 ++ type/__netbox/parameter/required | 1 + 7 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 type/__netbox/parameter/default/database-host create mode 100644 type/__netbox/parameter/default/database-port diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index e621845..65b00f5 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -19,8 +19,8 @@ DATABASE = { 'NAME': '$DATABASE_NAME', # Database name 'USER': '$DATABASE_USER', # PostgreSQL username 'PASSWORD': '$DATABASE_PASSWORD', # PostgreSQL password - 'HOST': 'localhost', # Database server - 'PORT': '', # Database port (leave blank for default) + 'HOST': '$DATABASE_HOST', # Database server + 'PORT': '$DATABASE_PORT', # Database port (leave blank for default) 'CONN_MAX_AGE': 300, # Max database connection age } diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index d91c6e8..a09b13a 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -26,6 +26,9 @@ version database PostgreSQL database name. +database-user + PostgreSQL database user. + database-password PostgreSQL database password. @@ -39,6 +42,12 @@ host OPTIONAL PARAMETERS ------------------- +database-host + PostgreSQL database hostname. Defaults to ``localhost``. + +database-port + PostgreSQL database port. Defaults to empty (uses the default port). + ldap-server LDAP server URI. Enables LDAP-backed authentication if specified. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 1f7f8ed..45c8afc 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -26,8 +26,15 @@ esac DATABASE_NAME=$(cat "$__object/parameter/database") export DATABASE_NAME +DATABASE_USER="$(cat "$__object/parameter/database-user")" +export DATABASE_USER DATABASE_PASSWORD=$(cat "$__object/parameter/database-password") export DATABASE_PASSWORD +DATABASE_HOST="$(cat "$__object/parameter/database-host")" +export DATABASE_HOST +DATABASE_PORT="$(cat "$__object/parameter/database-port")" +export DATABASE_PORT + ALLOWED_HOST=$(cat "$__object/parameter/host") export ALLOWED_HOST SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox/parameter/default/database-host b/type/__netbox/parameter/default/database-host new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/type/__netbox/parameter/default/database-host @@ -0,0 +1 @@ +localhost diff --git a/type/__netbox/parameter/default/database-port b/type/__netbox/parameter/default/database-port new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/type/__netbox/parameter/default/database-port @@ -0,0 +1 @@ + diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 159c813..d97f961 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,3 +1,5 @@ +database-host +database-port ldap-server ldap-bind-dn ldap-bind-password diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 388048a..84e8e4b 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -1,5 +1,6 @@ version database +database-user database-password secret-key host From 9fb3c7f52ef2a27e7d60d850d1918083c90d84c5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 16:00:15 +0200 Subject: [PATCH 068/332] __netbox: fix manpage indents --- type/__netbox/man.rst | 67 ++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index a09b13a..2e43bbd 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -43,88 +43,89 @@ host OPTIONAL PARAMETERS ------------------- database-host - PostgreSQL database hostname. Defaults to ``localhost``. + PostgreSQL database hostname. Defaults to ``localhost``. database-port - PostgreSQL database port. Defaults to empty (uses the default port). + PostgreSQL database port. Defaults to empty (uses the default port). ldap-server - LDAP server URI. Enables LDAP-backed authentication if specified. + LDAP server URI. Enables LDAP-backed authentication if specified. ldap-bind-dn - DN for the NetBox service account. Required for LDAP authentication. + DN for the NetBox service account. Required for LDAP authentication. ldap-bind-password - Password for the NetBox service account. Required for LDAP authentication. + Password for the NetBox service account. Required for LDAP authentication. ldap-user-base - Base used for searching user entries. Required for LDAP authentication. + Base used for searching user entries. Required for LDAP authentication. ldap-group-base - Base used for searching group entries. + Base used for searching group entries. ldap-require-group - Group required to login. + Group required to login. ldap-superuser-group - Make members of this groups superusers. + Make members of this groups superusers. redis-host - Redis database hostname. Defaults to ``localhost``. + Redis database hostname. Defaults to ``localhost``. redis-port - Redis database port. Defaults to ``6379``. + Redis database port. Defaults to ``6379``. redis-password - Redis password. Defaults to empty password. + Redis password. Defaults to empty password. redis-dbid-offset - Offset to set the redis database id's. The `tasks` database id is `offset + 0` - and `caching` is `offset + 1`. The offset defaults to ``0``. + Offset to set the redis database id's. The `tasks` database id is + `offset + 0` and `caching` is `offset + 1`. The offset defaults + to ``0``. smtp-host - Host of the SMTP email server. Defaults to ``localhost``. + Host of the SMTP email server. Defaults to ``localhost``. smtp-port - Port of the SMTP email server. Defaults to ``25``. + Port of the SMTP email server. Defaults to ``25``. smtp-user - Username to access the SMTP email server. Defaults to empty. + Username to access the SMTP email server. Defaults to empty. smtp-password - Password to access the SMTP email server. Defaults to empty. + Password to access the SMTP email server. Defaults to empty. smtp-from-email - Email from which NetBox will be sent of. Defaults to empty. + Email from which NetBox will be sent of. Defaults to empty. basepath - Base URL path if accessing netbox within a directory instead of directly the - webroot ``/``. For example, if installed at https://example.com/netbox/, set - the value ``netbox/``. + Base URL path if accessing netbox within a directory instead of directly the + webroot ``/``. For example, if installed at https://example.com/netbox/, set + the value ``netbox/``. http-proxy https-proxy - Proxy which will be used with any HTTP request like webhooks. + Proxy which will be used with any HTTP request like webhooks. BOOLEAN PARAMETERS ------------------ redis-ssl - Enables a secure TLS/SSL connection to the redis database. By default, ssl - is disabled. + Enables a secure TLS/SSL connection to the redis database. By default, ssl + is disabled. smtp-use-tls - Uses TLS to connect to the SMTP email server. `See documentation - Date: Sat, 22 Aug 2020 20:55:35 +0200 Subject: [PATCH 069/332] __netbox: auto-install sudo as dependency There are may be systems does not have `sudo` preinstalled like debian-minimal. Else, `su` could be used as alternative. --- type/__netbox/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 45c8afc..3033c4d 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -6,7 +6,7 @@ case "$os" in debian|ubuntu) # Install netbox dependencies. for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv; do + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do __package $pkg done From ae7ffac6bbf17e64b6ed79768f6db944395672dd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:29:30 +0200 Subject: [PATCH 070/332] __netbox: force virtualenv to be python3 Set a flag to be sure it is python3. Else, you may get an ugly error if python2 is not installed (and netbox requires python3.6 at minimum). --- type/__netbox/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 8533f4a..9359252 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -37,7 +37,8 @@ cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. -virtualenv /opt/netbox/venv +# forcing python3 to be sure (till python4 gets released ..) +virtualenv -p python3 /opt/netbox/venv # Install python dependencies. /opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" From ce47cec2e7883723d8b11efda1f97090ac95cd8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:50:54 +0200 Subject: [PATCH 071/332] __netbox: silence curl progress bar .. but still showing errors if any. --- type/__netbox/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 9359252..e0eb09d 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -23,7 +23,7 @@ tmpdir=\$(mktemp -d) cd "\$tmpdir" # Download and extract sources. -curl -L '$url' > '$archive' +curl -sS -L '$url' > '$archive' tar xf '$archive' # Save cdist-upload configuration file. From 4a04a2c4728ef0625bc49a039ece3240058159f0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 21:58:44 +0200 Subject: [PATCH 072/332] __netbox: repalace tabs with whitespaces Doing `s/\t/ /g` to have all times the same indents. --- type/__netbox/files/ldap_config.py.sh | 14 ++--- type/__netbox/man.rst | 20 +++--- type/__netbox/manifest | 88 +++++++++++++-------------- 3 files changed, 61 insertions(+), 61 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index bc494fb..cf4ed92 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -43,7 +43,7 @@ AUTH_LDAP_USER_ATTR_MAP = { EOF if [ "$LDAP_GROUP_BASE" != "" ]; then - cat << EOF + cat << EOF # This search ought to return all groups to which the user belongs. django_auth_ldap uses this to determine group # hierarchy. @@ -55,21 +55,21 @@ AUTH_LDAP_GROUP_TYPE = PosixGroupType() AUTH_LDAP_MIRROR_GROUPS = True EOF - if [ "$LDAP_REQUIRE_GROUP" != "" ]; then - cat << EOF + if [ "$LDAP_REQUIRE_GROUP" != "" ]; then + cat << EOF # Define a group required to login. AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" EOF - fi + fi - if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then - cat << EOF + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + cat << EOF # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { "is_superuser": "$LDAP_SUPERUSER_GROUP", } EOF - fi + fi fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 2e43bbd..4e63ccc 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -144,16 +144,16 @@ EXAMPLES .. code-block:: sh __netbox --version 2.8.7 --database netbox \ - --database-password "secretsecretsecret" \ - --secret-key "secretsecretsecret" \ - --host "${__target_host:?}" \ - --ldap-server "ldaps://ldap.domain.tld" \ - --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ - --ldap-bind-password "secretsecretsecret" \ - --ldap-user-base "ou=users,dc=domain,dc=tld" \ - --ldap-group-base "ou=groups,dc=domain,dc=tld" \ - --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ - --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + --database-password "secretsecretsecret" \ + --secret-key "secretsecretsecret" \ + --host "${__target_host:?}" \ + --ldap-server "ldaps://ldap.domain.tld" \ + --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ + --ldap-bind-password "secretsecretsecret" \ + --ldap-user-base "ou=users,dc=domain,dc=tld" \ + --ldap-group-base "ou=groups,dc=domain,dc=tld" \ + --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ + --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" NOTES diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 3033c4d..21a9bc6 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -3,24 +3,24 @@ os=$(cat "$__global/explorer/os") case "$os" in - debian|ubuntu) - # Install netbox dependencies. - for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do - __package $pkg - done + debian|ubuntu) + # Install netbox dependencies. + for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do + __package $pkg + done - if [ -f "$__object/parameter/ldap-server" ]; then - for pkg in libldap2-dev libsasl2-dev libssl-dev; do - __package $pkg - done - fi - ;; - *) - printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; + if [ -f "$__object/parameter/ldap-server" ]; then + for pkg in libldap2-dev libsasl2-dev libssl-dev; do + __package $pkg + done + fi + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; esac @@ -41,38 +41,38 @@ SECRET_KEY=$(cat "$__object/parameter/secret-key") export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then - LDAP_SERVER=$(cat "$__object/parameter/ldap-server") - export LDAP_SERVER + LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + export LDAP_SERVER fi if [ -f "$__object/parameter/ldap-bind-dn" ]; then - LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") - export LDAP_BIND_DN + LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + export LDAP_BIND_DN fi if [ -f "$__object/parameter/ldap-bind-password" ]; then - LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") - export LDAP_BIND_PASSWORD + LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + export LDAP_BIND_PASSWORD fi if [ -f "$__object/parameter/ldap-user-base" ]; then - LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") - export LDAP_USER_BASE + LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + export LDAP_USER_BASE fi if [ -f "$__object/parameter/ldap-group-base" ]; then - LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") - export LDAP_GROUP_BASE + LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") + export LDAP_GROUP_BASE fi if [ -f "$__object/parameter/ldap-require-group" ]; then - LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") - export LDAP_REQUIRE_GROUP + LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") + export LDAP_REQUIRE_GROUP fi if [ -f "$__object/parameter/ldap-superuser-group" ]; then - LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") - export LDAP_SUPERUSER_GROUP + LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") + export LDAP_SUPERUSER_GROUP fi # have default values @@ -123,12 +123,12 @@ BASEPATH="$(cat "$__object/parameter/basepath")" export BASEPATH if [ -f "$__object/parameter/http-proxy" ]; then - HTTP_PROXY=$(cat "$__object/parameter/http-proxy") - export HTTP_PROXY + HTTP_PROXY=$(cat "$__object/parameter/http-proxy") + export HTTP_PROXY fi if [ -f "$__object/parameter/https-proxy" ]; then - HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") - export HTTPS_PROXY + HTTPS_PROXY=$(cat "$__object/parameter/https-proxy") + export HTTPS_PROXY fi if [ -f "$__object/parameter/login-required" ]; then @@ -149,23 +149,23 @@ mkdir -p "$__object/files" require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ - --source "$__object/files/configuration.py" + /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ + --source "$__object/files/configuration.py" if [ -f "$__object/parameter/ldap-server" ]; then - require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ - --source "$__object/files/ldap_config.py" + require="__directory/opt/netbox/netbox/cdist " __file \ + /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + --source "$__object/files/ldap_config.py" fi # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled done # Python worker configuration. require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" + --mode 644 --source "$__type/files/gunicorn.py" From 0b261070d362fed393c9e021654917bb525f8b7a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:04:33 +0200 Subject: [PATCH 073/332] __netbox: man note regards creating a superuser --- type/__netbox/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 4e63ccc..91313be 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -166,6 +166,11 @@ you are welcome to contribute! - `Possible optional settings ` +If you not setup ldap authentification, you may be interested into how to +`setting up a super user +` +directly on the machine to be able to access and use NetBox. + SEE ALSO -------- - `NetBox documentation `_ From 584e0bbda85669de99e907784c17a4d176aae996 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:23:36 +0200 Subject: [PATCH 074/332] __netbox: fix netbox version save The cdist directory needs to be created to write there. This is done directly after the new tarball is installed. --- type/__netbox/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index e0eb09d..d2ec5f9 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -33,6 +33,8 @@ cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' +mkdir '$install_dir/cdist' + cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' From 88ddebd0404a5b0d4adf597ec6b42c61c7240ea5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 22 Aug 2020 22:47:15 +0200 Subject: [PATCH 075/332] __netbox: fix deprecated configuration option Fixes error message `/opt/netbox/netbox/netbox/settings.py:141: UserWarning: REMOTE_AUTH_DEFAULT_PERMISSIONS should be a dictionary. Backward compatibility will be removed in v2.10.` --- type/__netbox/files/configuration.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 65b00f5..8679437 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -247,7 +247,7 @@ REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] -REMOTE_AUTH_DEFAULT_PERMISSIONS = [] +REMOTE_AUTH_DEFAULT_PERMISSIONS = {} # This determines how often the GitHub API is called to check the latest release of NetBox. Must be at least 1 hour. RELEASE_CHECK_TIMEOUT = 24 * 3600 From f5c8c26db6436777430a31c410c1c442fce184c7 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 11:36:35 +0200 Subject: [PATCH 076/332] __netbox: new config parameters for data paths .. to move them to an other partition. --- type/__netbox/files/configuration.py.sh | 36 ++++++++++++++++--------- type/__netbox/man.rst | 33 +++++++++++++++++++++++ type/__netbox/manifest | 19 +++++++++++++ type/__netbox/parameter/optional | 4 +++ 4 files changed, 80 insertions(+), 12 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 8679437..af19153 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -154,15 +154,6 @@ EOF cat << EOF } EOF - -else - cat << EOF -# HTTP proxies NetBox should use when sending outbound HTTP requests (e.g. for webhooks). -# HTTP_PROXIES = { -# 'http': 'http://10.10.1.10:3128', -# 'https': 'http://10.10.1.10:1080', -# } -EOF fi cat << EOF @@ -190,10 +181,18 @@ MAINTENANCE_MODE = False # all objects by specifying "?limit=0". MAX_PAGE_SIZE = 1000 +EOF + +if [ "$MEDIA_ROOT" != "" ]; then + cat << EOF # The file path where uploaded media such as image attachments are stored. A trailing slash is not needed. Note that # the default value of this setting is derived from the installed location. -# MEDIA_ROOT = '/opt/netbox/netbox/media' +MEDIA_ROOT = '$MEDIA_ROOT' +EOF +fi + +cat << EOF # By default uploaded media is stored on the local filesystem. Using Django-storages is also supported. Provide the # class path of the storage driver in STORAGE_BACKEND and any configuration options in STORAGE_CONFIG. For example: # STORAGE_BACKEND = 'storages.backends.s3boto3.S3Boto3Storage' @@ -257,14 +256,27 @@ RELEASE_CHECK_TIMEOUT = 24 * 3600 RELEASE_CHECK_URL = None # RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' +EOF + +if [ "$REPORTS_ROOT" != "" ]; then + cat << EOF # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -# REPORTS_ROOT = '/opt/netbox/netbox/reports' +REPORTS_ROOT = '$REPORTS_ROOT' +EOF +fi + +if [ "$SCRIPTS_ROOT" != "" ]; then + cat << EOF # The file path where custom scripts will be stored. A trailing slash is not needed. Note that the default value of # this setting is derived from the installed location. -# SCRIPTS_ROOT = '/opt/netbox/netbox/scripts' +SCRIPTS_ROOT = '$SCRIPTS_ROOT' +EOF +fi + +cat << EOF # By default, NetBox will store session data in the database. Alternatively, a file path can be specified here to use # local file storage instead. (This can be useful for enabling authentication on a standby instance with read-only # database access.) Note that the user as which NetBox runs must have read and write permissions to this path. diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 91313be..c910d5c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -107,6 +107,39 @@ http-proxy https-proxy Proxy which will be used with any HTTP request like webhooks. +data-root + This parameter set's the media, reports and scripts root to subdirectories + of the given directory. Values can be overwritten by special parameters like + `--media-root` for example. Use this option if you want to store persistant + data of netbox on an other partition. A trailing shlash is not needed. + + The data directories have following predefined sub-directory names: + + media root: + ``$data_root/media`` + reports root: + ``$data_root/reports`` + scripts root: + ``$data_root/scripts`` + +media-root + The file path to where media files (like image attachments) are stored. + Change this path if you require to store data on an other partiotion. + A trailing slash is not needed. By default, it will be stored into the + installation directory (``/opt/netbox/netbox/netbox/media``). + +reports-root + The file path of where custom reports are kept. Change this path if you + require to store data on an other partition. A trailing slash is not + needed. By default, it will be stored into the installation directory + (``/opt/netbox/netbox/netbox/reports``). + +scripts-root + The file path of where custom scripts are kept. Change this path if you + require to store data on an other partition. A trailing slash is not + needed. By default, it will be stored into the installation directory + (``/opt/netbox/netbox/netbox/scripts``). + BOOLEAN PARAMETERS ------------------ redis-ssl diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 21a9bc6..133b527 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -138,6 +138,25 @@ else fi export LOGIN_REQUIRED +if [ -f "$__object/parameter/data-root" ]; then + data_root="$(cat "$__object/parameter/data-root")" + MEDIA_ROOT="$data_root/media" + REPORTS_ROOT="$data_root/reports" + SCRIPTS_ROOT="$data_root/scripts" +fi +if [ -f "$__object/parameter/media-root" ]; then + MEDIA_ROOT="$(cat "$__object/parameter/media-root")" +fi +export MEDIA_ROOT +if [ -f "$__object/parameter/reports-root" ]; then + REPORTS_ROOT="$(cat "$__object/parameter/reports-root")" +fi +export REPORTS_ROOT +if [ -f "$__object/parameter/scripts-root" ]; then + SCRIPTS_ROOT="$(cat "$__object/parameter/scripts-root")" +fi +export SCRIPTS_ROOT + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index d97f961..21fea5f 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -19,3 +19,7 @@ smtp-from-email basepath http-proxy https-proxy +data-root +media-root +reports-root +scripts-root From 49bb527dea8752fe14155516581bd7ee9049143a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 16:52:02 +0200 Subject: [PATCH 077/332] __netbox: autogen secretkey and store at remote The secret key is generated if it is not set via parameter and the explorer does not return any. It will be saved in the netbox home directory to easily read the key for the config generation. --- type/__netbox/explorer/secretkey | 8 ++++++++ type/__netbox/explorer/version | 2 +- type/__netbox/man.rst | 16 +++++++++++----- type/__netbox/manifest | 19 ++++++++++++++++++- type/__netbox/parameter/optional | 1 + type/__netbox/parameter/required | 1 - 6 files changed, 39 insertions(+), 8 deletions(-) create mode 100755 type/__netbox/explorer/secretkey diff --git a/type/__netbox/explorer/secretkey b/type/__netbox/explorer/secretkey new file mode 100755 index 0000000..9f0331c --- /dev/null +++ b/type/__netbox/explorer/secretkey @@ -0,0 +1,8 @@ +#!/bin/sh -e + +# Explorer will output the key if he exists. + +secretkey="/opt/netbox/.secretkey" +if [ -f "$secretkey" ]; then + cat "$secretkey" +fi diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version index a9ed590..5231e16 100755 --- a/type/__netbox/explorer/version +++ b/type/__netbox/explorer/version @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # output version if exist version_path="/opt/netbox/netbox/cdist/version" diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index c910d5c..77b47d0 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -32,16 +32,22 @@ database-user database-password PostgreSQL database password. -secret-key - Random secret key of at least 50 alphanumeric characters. This key must be - unique to this installation and must not be shared outside the local - system. - host Hostname (domain or IP address) on which the application is served. OPTIONAL PARAMETERS ------------------- +secret-key + Random secret key of at least 50 alphanumeric characters and symbols. This + key must be unique to this installation and must not be shared outside the + local system. If no secret key is given, the type generates an own 50 chars + long key and saves it on the remote host to remember it for the next run. + + The secret, random string is used to assist in the creation new + cryptographic hashes for passwords and HTTP cookies. It is not directly + used for hasing user passwords or for encrpted storage. It can be changed + at any time, but will invalidate all existing sessions. + database-host PostgreSQL database hostname. Defaults to ``localhost``. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 133b527..cd47d38 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -37,7 +37,18 @@ export DATABASE_PORT ALLOWED_HOST=$(cat "$__object/parameter/host") export ALLOWED_HOST -SECRET_KEY=$(cat "$__object/parameter/secret-key") + +if [ -f "$__object/parameter/secret-key" ]; then + SECRET_KEY=$(cat "$__object/parameter/secret-key") +elif [ -s "$__object/explorer/secretkey" ]; then + # Can be done over netbox/generate_secret_key.py too, but it's to + # complicated with the variable setup (can't generated right now!). + # Generates a 50-character long key (without ' cause of python quotes) + SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" +else + # take the key that is already used + SECRET_KEY="$(cat "$__object/explorer/secretkey")" +fi export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then @@ -177,6 +188,12 @@ if [ -f "$__object/parameter/ldap-server" ]; then --source "$__object/files/ldap_config.py" fi +# save secret +require="__user/netbox" __file /opt/netbox/.secretkey --mode 400 \ + --owner netbox --source - << SECRET +$SECRET_KEY +SECRET + # Upload systemd units and gunicorn configuration. for unit in netbox netbox-rq; do diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 21fea5f..47722e6 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -1,3 +1,4 @@ +secret-key database-host database-port ldap-server diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 84e8e4b..3d03e5a 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -2,5 +2,4 @@ version database database-user database-password -secret-key host From 4e90c93eef881a228272167f2e328de9a16bf892 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 17:15:32 +0200 Subject: [PATCH 078/332] __netbox: updated copyright Added me to the copyright after there are some more changes :-) --- type/__netbox/man.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 77b47d0..10485a1 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -217,11 +217,13 @@ SEE ALSO AUTHORS ------- Timothée Floure +Matthias Stecher COPYING ------- -Copyright \(C) 2020 Timothée Floure. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2020 Timothée Floure. +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. From 863c95059e7d5b10607b97b88aa49b402c0561d3 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 23 Aug 2020 19:48:56 +0200 Subject: [PATCH 079/332] __netbox: fix REMOTE_AUTH_BACKEND module not found Updated with the backend from the current upstream configuration. --- type/__netbox/files/configuration.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index af19153..0b8304d 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -242,7 +242,7 @@ RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 # Remote authentication support REMOTE_AUTH_ENABLED = False -REMOTE_AUTH_BACKEND = 'utilities.auth_backends.RemoteUserBackend' +REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] From 95fd7d872ed1c649742008604d4becc88be010a4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 24 Aug 2020 18:57:27 +0200 Subject: [PATCH 080/332] __netbox: fix ldap config short-circuit Forgot to negotate if statement .. works now :/ --- type/__netbox/files/ldap_config.py.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index cf4ed92..c31f46a 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if [ "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then +if [ -z "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then # skip cat << EOF ############################## From 7d694342fffe1dd2bd850ac38c5fa7580da8fedb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 24 Aug 2020 19:31:49 +0200 Subject: [PATCH 081/332] __netbox: new parameter for automatic release checks --- type/__netbox/files/configuration.py.sh | 14 ++++++++++++-- type/__netbox/man.rst | 8 ++++++-- type/__netbox/manifest | 5 +++++ type/__netbox/parameter/boolean | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 0b8304d..21104b4 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -253,11 +253,21 @@ RELEASE_CHECK_TIMEOUT = 24 * 3600 # This repository is used to check whether there is a new release of NetBox available. Set to None to disable the # version check or use the URL below to check for release in the official NetBox repository. -RELEASE_CHECK_URL = None -# RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' EOF +if [ "$UPDATE_CHECK" != "" ]; then + cat << EOF +RELEASE_CHECK_URL = None + +EOF +else + cat << EOF +RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' + +EOF +fi + if [ "$REPORTS_ROOT" != "" ]; then cat << EOF # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 10485a1..761494a 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -163,8 +163,12 @@ smtp-use-ssl for more information. login-required - Sets if a login is required to access all sites. By default, anounymous users - can see most data (excluding secrets) but not make any changes. + Sets if a login is required to access all sites. By default, anonymous + users can see most data (excluding secrets) but not make any changes. + +update-notify + Enables the NetBox version check for new upstream updates. It checks every + 24 hours for new releases and notify the admin users in the gui if any. MESSAGES -------- diff --git a/type/__netbox/manifest b/type/__netbox/manifest index cd47d38..62da0a2 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -168,6 +168,11 @@ if [ -f "$__object/parameter/scripts-root" ]; then fi export SCRIPTS_ROOT +if [ -f "$__object/parameter/update-notify" ]; then + UPDATE_CHECK="yes" + export UPDATE_CHECK +fi + # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home diff --git a/type/__netbox/parameter/boolean b/type/__netbox/parameter/boolean index a94808c..d568037 100644 --- a/type/__netbox/parameter/boolean +++ b/type/__netbox/parameter/boolean @@ -2,3 +2,4 @@ redis-ssl smtp-use-ssl smtp-use-tls login-required +update-notify From 8ff7a4616dcedac333b5e93d994cbff2887ffbe9 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 25 Aug 2020 19:30:32 +0200 Subject: [PATCH 082/332] __netbox: allow multiple hostnames --- type/__netbox/files/configuration.py.sh | 2 +- type/__netbox/man.rst | 2 ++ type/__netbox/manifest | 8 ++++++-- type/__netbox/parameter/required | 1 - type/__netbox/parameter/required_multiple | 1 + 5 files changed, 10 insertions(+), 4 deletions(-) create mode 100644 type/__netbox/parameter/required_multiple diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 21104b4..4b7be11 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -11,7 +11,7 @@ cat << EOF # access to the server via any other hostnames. The first FQDN in the list will be treated as the preferred name. # # Example: ALLOWED_HOSTS = ['netbox.example.com', 'netbox.internal.local'] -ALLOWED_HOSTS = [ '$ALLOWED_HOST' ] +ALLOWED_HOSTS = [$ALLOWED_HOSTS ] # PostgreSQL database configuration. See the Django documentation for a complete list of available parameters: # https://docs.djangoproject.com/en/stable/ref/settings/#databases diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 761494a..2af2bef 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -34,6 +34,7 @@ database-password host Hostname (domain or IP address) on which the application is served. + Multiple hostnames are possible; given as multiple arguments. OPTIONAL PARAMETERS ------------------- @@ -190,6 +191,7 @@ EXAMPLES --database-password "secretsecretsecret" \ --secret-key "secretsecretsecret" \ --host "${__target_host:?}" \ + --host "cool-netbox.xyz" \ --ldap-server "ldaps://ldap.domain.tld" \ --ldap-bind-dn "uid=netbox,ou=services,dc=domain,dc=tld" \ --ldap-bind-password "secretsecretsecret" \ diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 62da0a2..2f5cf4e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -35,8 +35,12 @@ export DATABASE_HOST DATABASE_PORT="$(cat "$__object/parameter/database-port")" export DATABASE_PORT -ALLOWED_HOST=$(cat "$__object/parameter/host") -export ALLOWED_HOST +# list of hosts +ALLOWED_HOSTS="" +while read -r hostname; do + ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," +done < "$__object/parameter/host" +export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox/parameter/required b/type/__netbox/parameter/required index 3d03e5a..02fca9f 100644 --- a/type/__netbox/parameter/required +++ b/type/__netbox/parameter/required @@ -2,4 +2,3 @@ version database database-user database-password -host diff --git a/type/__netbox/parameter/required_multiple b/type/__netbox/parameter/required_multiple new file mode 100644 index 0000000..c70dc2d --- /dev/null +++ b/type/__netbox/parameter/required_multiple @@ -0,0 +1 @@ +host From eaf639790b00bdb5d7d9197939d0e6fa7963f3a2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 17:39:19 +0200 Subject: [PATCH 083/332] __netbox: remove ldap pip3 package if not required --- type/__netbox/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d2ec5f9..abe7ffa 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -48,6 +48,8 @@ EOF if [ -f "$__object/parameter/ldap-server" ]; then echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + else + echo "/opt/netbox/venv/bin/pip3 uninstall django-auth-ldap" fi cat << EOF From 49d39eaee50bc848840cb4fda91e2ed0cb4a14f1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 26 Aug 2020 18:01:52 +0200 Subject: [PATCH 084/332] [type/__mail_alias] Fix error with some AWK implementations Some AWK implementations seem to have a problem with parameters named default. awk: cmd. line:2: function sepafter(f, default, _) { awk: cmd. line:2: ^ syntax error awk: cmd. line:5: return _ ? _ : default awk: cmd. line:5: ^ syntax error In addition the temp file is removed if an error occurs. --- type/__mail_alias/gencode-remote | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 3eaad75..9f4af1b 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -67,10 +67,10 @@ cat <<'EOF' test -f "${aliases_file}" || touch "${aliases_file}" awk -F ':[ \t]*' -v mode=$mode ' -function sepafter(f, default, _) { +function sepafter(f, def, _) { # finds the separator between field $f and $(f+1) _ = substr($0, length($f)+1, index(substr($0, length($f)+1), $(f+1))-1) - return _ ? _ : default + return _ ? _ : def } function write_aliases() { @@ -134,6 +134,7 @@ END { write_aliases() } ' <"${aliases_file}" >"${aliases_file}.tmp" || { + rm -f "${aliases_file}.tmp" echo 'Generating new aliases file failed!' >&2 exit 1 } From e0a1b4f66343415b002c6d1c86114da3c9d402f8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 20:01:23 +0200 Subject: [PATCH 085/332] __netbox_gunicorn: extract gunicorn setup from __netbox Because someone *want* to use something other than just gunicorn, it was extracted to a own type. Because gunicorn is a bit deep in the netbox installation process, it's a bit harder to isolate it. `__netbox_uwsgi` will come, too. --- type/__netbox/gencode-remote | 8 +- type/__netbox/manifest | 14 +--- type/__netbox_gunicorn/explorer/installed | 4 + .../explorer/should_installed | 3 + .../files/gunicorn.py.sh} | 19 ++++- .../files/netbox.service | 3 +- type/__netbox_gunicorn/gencode-remote | 36 +++++++++ type/__netbox_gunicorn/man.rst | 75 +++++++++++++++++++ type/__netbox_gunicorn/manifest | 20 +++++ .../parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + type/__netbox_gunicorn/singleton | 0 12 files changed, 168 insertions(+), 16 deletions(-) create mode 100755 type/__netbox_gunicorn/explorer/installed create mode 100755 type/__netbox_gunicorn/explorer/should_installed rename type/{__netbox/files/gunicorn.py => __netbox_gunicorn/files/gunicorn.py.sh} (56%) mode change 100644 => 100755 rename type/{__netbox => __netbox_gunicorn}/files/netbox.service (88%) create mode 100755 type/__netbox_gunicorn/gencode-remote create mode 100644 type/__netbox_gunicorn/man.rst create mode 100755 type/__netbox_gunicorn/manifest create mode 100644 type/__netbox_gunicorn/parameter/default/bind-to create mode 100644 type/__netbox_gunicorn/parameter/optional_multiple create mode 100644 type/__netbox_gunicorn/singleton diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index abe7ffa..af4d8c8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -35,6 +35,7 @@ rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' mkdir '$install_dir/cdist' +cp '$src/requirements.txt' /opt/netbox/ # backup dependency info cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' @@ -43,7 +44,9 @@ cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Install python dependencies. -/opt/netbox/venv/bin/pip3 install -r "\$tmpdir/$src/requirements.txt" +# avoid gunicorn, because it will be done in an other type +grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ + | xargs /opt/netbox/venv/bin/pip3 install EOF if [ -f "$__object/parameter/ldap-server" ]; then @@ -94,8 +97,7 @@ fi # check for changes if [ "$changes" = "yes" ]; then cat << EOF -# Restart services. -service netbox restart +# Restart service. service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 2f5cf4e..f0da2a3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,13 +204,7 @@ $SECRET_KEY SECRET -# Upload systemd units and gunicorn configuration. -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done - -# Python worker configuration. -require="__user/netbox" __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__type/files/gunicorn.py" +# Upload systemd unit for worker service +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled diff --git a/type/__netbox_gunicorn/explorer/installed b/type/__netbox_gunicorn/explorer/installed new file mode 100755 index 0000000..c6f5d87 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show gunicorn | awk '/Version:/{print $2}' diff --git a/type/__netbox_gunicorn/explorer/should_installed b/type/__netbox_gunicorn/explorer/should_installed new file mode 100755 index 0000000..073be92 --- /dev/null +++ b/type/__netbox_gunicorn/explorer/should_installed @@ -0,0 +1,3 @@ +#!/bin/sh -e + +awk -v FS="==" '$1 ~ /gunicorn/{print $2}' /opt/netbox/requirements.txt diff --git a/type/__netbox/files/gunicorn.py b/type/__netbox_gunicorn/files/gunicorn.py.sh old mode 100644 new mode 100755 similarity index 56% rename from type/__netbox/files/gunicorn.py rename to type/__netbox_gunicorn/files/gunicorn.py.sh index c582677..862ba06 --- a/type/__netbox/files/gunicorn.py +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -1,9 +1,23 @@ +#!/bin/sh -e + +# Generates gunicorn config +# see https://docs.gunicorn.org/en/stable/settings.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = '127.0.0.1:8001' +bind = [$HOST ] # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. -workers = 3 +workers = $(( 2*cores + 1 )) # Number of threads per worker process threads = 3 @@ -14,3 +28,4 @@ timeout = 120 # The maximum number of requests a worker can handle before being respawned max_requests = 5000 max_requests_jitter = 500 +EOF diff --git a/type/__netbox/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service similarity index 88% rename from type/__netbox/files/netbox.service rename to type/__netbox_gunicorn/files/netbox.service index 18eb045..475ddc4 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,5 +1,5 @@ [Unit] -Description=NetBox WSGI Service +Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ After=network-online.target Wants=network-online.target @@ -19,4 +19,5 @@ RestartSec=30 PrivateTmp=true [Install] +Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote new file mode 100755 index 0000000..6958976 --- /dev/null +++ b/type/__netbox_gunicorn/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh -e + +curr_installed="$(cat "$__object/explorer/installed")" +should_installed="$(cat "$__object/explorer/should_installed")" + +# gunicorn version change +if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" + + do_restart=yes + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" +fi + +# configuration changes +if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configuration\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no message cause it should be obvious +fi + + +# restart gunicorn +if [ "$do_restart"]; then + cat << EOF +# Restart service +service gunicorn-netbox restart +EOF + echo restarted >> "$__messages_out" +fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst new file mode 100644 index 0000000..38ac4a5 --- /dev/null +++ b/type/__netbox_gunicorn/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_gunicorn - run netbox with gunicorn + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The hosts the gunicorn socket should be bind to. Formats are `IP`, + `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple + times. Defaults to ``127.0.0.1:8001``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +updated $old to $new + The version of the gunicorn software was updated from `$old` to `$new`. + +configuration + Configuration for gunicorn changed. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_gunicorn + + # with arguments + __netbox $args + require="__netbox" __netbox_gunicorn \ + --bind-to 0.0.0.0:8001 \ + --bind-to 1.2.3.4:5678 + + +SEE ALSO +-------- +:strong:`__netbox`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest new file mode 100755 index 0000000..dab8181 --- /dev/null +++ b/type/__netbox_gunicorn/manifest @@ -0,0 +1,20 @@ +#!/bin/sh -e + +HOST="" +while read -r host; do + HOST="$HOST '$host'," +done < "$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + +# gunicorn config file +__file /opt/netbox/gunicorn.py \ + --mode 644 --source "$__object/files/gunicorn.py" + +# install service file +__systemd_unit gunicorn-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_gunicorn/parameter/default/bind-to b/type/__netbox_gunicorn/parameter/default/bind-to new file mode 100644 index 0000000..f4c980e --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:8001 diff --git a/type/__netbox_gunicorn/parameter/optional_multiple b/type/__netbox_gunicorn/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to diff --git a/type/__netbox_gunicorn/singleton b/type/__netbox_gunicorn/singleton new file mode 100644 index 0000000..e69de29 From bbce0030aba387232f48a6c4b676c2b82cba28ed Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 26 Aug 2020 21:07:31 +0200 Subject: [PATCH 086/332] __netbox: enable ldap usage via configuration.py The REMOTE_AUTH_BACKEND must be set to use LDAP. It now exports USE_LDAP to generally say if LDAP is being used in the configuration or not. --- type/__netbox/files/configuration.py.sh | 14 ++++++++++++++ type/__netbox/files/ldap_config.py.sh | 2 +- type/__netbox/manifest | 12 ++++++------ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 4b7be11..ea73133 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -240,9 +240,23 @@ PREFER_IPV4 = False RACK_ELEVATION_DEFAULT_UNIT_HEIGHT = 22 RACK_ELEVATION_DEFAULT_UNIT_WIDTH = 220 +EOF + +if [ "$USE_LDAP" ]; then + cat << EOF +# Remote authentication support with ldap +REMOTE_AUTH_ENABLED = True +REMOTE_AUTH_BACKEND = 'netbox.authentication.LDAPBackend' +EOF +else + cat << EOF # Remote authentication support REMOTE_AUTH_ENABLED = False REMOTE_AUTH_BACKEND = 'netbox.authentication.RemoteUserBackend' +EOF +fi + +cat << EOF REMOTE_AUTH_HEADER = 'HTTP_REMOTE_USER' REMOTE_AUTH_AUTO_CREATE_USER = True REMOTE_AUTH_DEFAULT_GROUPS = [] diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index c31f46a..a49d800 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -1,7 +1,7 @@ #!/bin/sh # no configuration if there are no ldap parameters -if [ -z "$(find "$__object/parameter/" -type f -name 'ldap-*' -print)" ]; then +if [ -z "$USE_LDAP" ]; then # skip cat << EOF ############################## diff --git a/type/__netbox/manifest b/type/__netbox/manifest index f0da2a3..079ea48 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -57,38 +57,38 @@ export SECRET_KEY if [ -f "$__object/parameter/ldap-server" ]; then LDAP_SERVER=$(cat "$__object/parameter/ldap-server") + USE_LDAP=yes export LDAP_SERVER fi - if [ -f "$__object/parameter/ldap-bind-dn" ]; then LDAP_BIND_DN=$(cat "$__object/parameter/ldap-bind-dn") + USE_LDAP=yes export LDAP_BIND_DN fi - if [ -f "$__object/parameter/ldap-bind-password" ]; then LDAP_BIND_PASSWORD=$(cat "$__object/parameter/ldap-bind-password") + USE_LDAP=yes export LDAP_BIND_PASSWORD fi - if [ -f "$__object/parameter/ldap-user-base" ]; then LDAP_USER_BASE=$(cat "$__object/parameter/ldap-user-base") + USE_LDAP=yes export LDAP_USER_BASE fi - if [ -f "$__object/parameter/ldap-group-base" ]; then LDAP_GROUP_BASE=$(cat "$__object/parameter/ldap-group-base") export LDAP_GROUP_BASE fi - if [ -f "$__object/parameter/ldap-require-group" ]; then LDAP_REQUIRE_GROUP=$(cat "$__object/parameter/ldap-require-group") export LDAP_REQUIRE_GROUP fi - if [ -f "$__object/parameter/ldap-superuser-group" ]; then LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") export LDAP_SUPERUSER_GROUP fi +# export if base ldap parameters are used +export USE_LDAP # have default values REDIS_HOST="$(cat "$__object/parameter/redis-host")" From 1ef4420c537570206780757fa5b78aa6895f5829 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 27 Aug 2020 19:46:09 +0200 Subject: [PATCH 087/332] __netbox_uwsgi: new type to handle uwsgi for netbox --- type/__netbox/gencode-remote | 2 +- type/__netbox/man.rst | 2 +- type/__netbox_gunicorn/gencode-remote | 5 +- type/__netbox_gunicorn/man.rst | 2 +- type/__netbox_uwsgi/explorer/installed | 4 + type/__netbox_uwsgi/explorer/upgradeable | 4 + type/__netbox_uwsgi/files/netbox.service | 23 ++++++ type/__netbox_uwsgi/files/uwsgi.ini.sh | 22 ++++++ type/__netbox_uwsgi/gencode-remote | 35 +++++++++ type/__netbox_uwsgi/man.rst | 75 +++++++++++++++++++ type/__netbox_uwsgi/manifest | 18 +++++ type/__netbox_uwsgi/parameter/default/bind-to | 1 + .../parameter/optional_multiple | 1 + 13 files changed, 188 insertions(+), 6 deletions(-) create mode 100755 type/__netbox_uwsgi/explorer/installed create mode 100755 type/__netbox_uwsgi/explorer/upgradeable create mode 100644 type/__netbox_uwsgi/files/netbox.service create mode 100755 type/__netbox_uwsgi/files/uwsgi.ini.sh create mode 100755 type/__netbox_uwsgi/gencode-remote create mode 100644 type/__netbox_uwsgi/man.rst create mode 100755 type/__netbox_uwsgi/manifest create mode 100644 type/__netbox_uwsgi/parameter/default/bind-to create mode 100644 type/__netbox_uwsgi/parameter/optional_multiple diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index af4d8c8..0153579 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -89,7 +89,7 @@ EOF fi # meta - printf "configuration\n" >> "$__messages_out" + printf "configured\n" >> "$__messages_out" changes=yes fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 2af2bef..6bb1e36 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -176,7 +176,7 @@ MESSAGES installed $VERSION Netbox was fresh installed or updated. The new version number is appended. -configuration +configured Some configuration files got updated and therefore the service was restarted. This message will not be echoed if configuration got updated due a standard installation. diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 6958976..e424c22 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -16,7 +16,7 @@ fi # configuration changes if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then do_restart=yes - printf "configuration\n" >> "$__messages_out" + printf "configured\n" >> "$__messages_out" fi # application @@ -27,10 +27,9 @@ fi # restart gunicorn -if [ "$do_restart"]; then +if [ "$do_restart" ]; then cat << EOF # Restart service service gunicorn-netbox restart EOF - echo restarted >> "$__messages_out" fi diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 38ac4a5..43f5864 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -34,7 +34,7 @@ MESSAGES updated $old to $new The version of the gunicorn software was updated from `$old` to `$new`. -configuration +configured Configuration for gunicorn changed. In both cases, and at messages from the `__netbox` type, it restarts the diff --git a/type/__netbox_uwsgi/explorer/installed b/type/__netbox_uwsgi/explorer/installed new file mode 100755 index 0000000..a2393d0 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/installed @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print version if available +/opt/netbox/venv/bin/pip3 show uwsgi | awk '/Version:/{print $2}' diff --git a/type/__netbox_uwsgi/explorer/upgradeable b/type/__netbox_uwsgi/explorer/upgradeable new file mode 100755 index 0000000..f4b0a22 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/upgradeable @@ -0,0 +1,4 @@ +#!/bin/sh -e + +# print latest version if availble +/opt/netbox/venv/bin/pip3 list --outdated | awk '$1 == "uwsgi" {print $3}' diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service new file mode 100644 index 0000000..cfce752 --- /dev/null +++ b/type/__netbox_uwsgi/files/netbox.service @@ -0,0 +1,23 @@ +[Unit] +Description=Netbox uWSGI WSGI Service +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +User=netbox +Group=netbox +PIDFile=/var/tmp/netbox.pid +WorkingDirectory=/opt/netbox + +ExecStart=/opt/netbox/venv/bin/uwsgi --master --wsgi-file netbox/netbox/wsgi.py uwsgi.ini + +Restart=on-failure +RestartSec=30 +PrivateTmp=true + +[Install] +Alias=netbox.service +WantedBy=multi-user.target diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh new file mode 100755 index 0000000..557a5fa --- /dev/null +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# Generates uwsgi config +# see https://uwsgi-docs.readthedocs.io/en/latest/Options.html + +# fix missing $__explorer +# see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 +__explorer="$__global/explorer" + +# size workes by cpu +cores="$(cat "$__explorer/cpu_cores")" + + +cat << EOF +[uwsgi] +; socket to bind +socket = $HOST + +; processes and threads +processes = $(( 2*cores + 1 )) +threads = 2 +EOF diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote new file mode 100755 index 0000000..5cc82d1 --- /dev/null +++ b/type/__netbox_uwsgi/gencode-remote @@ -0,0 +1,35 @@ +#!/bin/sh -e + +# not installed +if ! [ -s "$__object/explorer/installed" ]; then + echo "/opt/netbox/venv/bin/pip3 install uwsgi" + do_restart=yes + printf "installed\n" >> "$__messages_out" + +# updates available +elif [ -s "$__object/explorer/upgradeable" ]; then + echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + do_restart=yes + printf "upgraded\n" >> "$__messages_out" +fi + +# changed configuration +if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" +fi + +# application +if grep -q "^__netbox:" "$__messages_in"; then + do_restart=yes + # no messages cause this is obvious +fi + + +# restart uwsgi +if [ "$do_restart" ]; then + cat << EOF +# Restart service +service uwsgi-netbox restart +EOF +fi diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst new file mode 100644 index 0000000..5bf3160 --- /dev/null +++ b/type/__netbox_uwsgi/man.rst @@ -0,0 +1,75 @@ +cdist-type__netbox_uwsgi(7) +=========================== + +NAME +---- +cdist-type__netbox_uwsgi - run netbox with uwsgi + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +bind-to + The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. + Defaults to ``127.0.0.1:3031``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +installed + The uwsgi service was installed. + +upgraded + The uwsgi service was upgraded. + +configured + The uwsgi configuration got updated. + +In both cases, and at messages from the `__netbox` type, it restarts the +service to using the up-to-date version. + + +EXAMPLES +-------- + +.. code-block:: sh + + # simple + __netbox $args + require="__netbox" __netbox_uwsgi + + # with special bind + require="__netbox" __netbox_uwsgi --bind-to 0.0.0.0:3032 \ + --bind-to 0.0.0.0:3033 + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest new file mode 100755 index 0000000..779f93b --- /dev/null +++ b/type/__netbox_uwsgi/manifest @@ -0,0 +1,18 @@ +#!/bin/sh -e + +HOST="$__object/parameter/bind-to" +export HOST + +# process template +mkdir "$__object/files" +"$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" + +# uwsgi config file +# TODO maybe patching with __key_value cause of .ini ? +__file /opt/netbox/uwsgi.ini \ + --mode 644 --source "$__object/files/uwsgi.ini" + +# install service file +__systemd_unit uwsgi-netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled --restart diff --git a/type/__netbox_uwsgi/parameter/default/bind-to b/type/__netbox_uwsgi/parameter/default/bind-to new file mode 100644 index 0000000..c696456 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/bind-to @@ -0,0 +1 @@ +127.0.0.1:3031 diff --git a/type/__netbox_uwsgi/parameter/optional_multiple b/type/__netbox_uwsgi/parameter/optional_multiple new file mode 100644 index 0000000..331c077 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/optional_multiple @@ -0,0 +1 @@ +bind-to From 9d8b3ebe743685daac13d54ac3bf2449517c3318 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 1 Sep 2020 10:02:08 +0200 Subject: [PATCH 088/332] __netbox: add wrapper service to manage the wsgi services The wrapper service will "control" the services added from the __netbox_* types to provide a general interface. This is more dynamic than the alias approach used previously. Through this, it is possible to handle multiple wsgi services for netbox - if this works .. See as a reference: http://alesnosek.com/blog/2016/12/04/controlling-a-multi-service-application-with-systemd/ --- type/__netbox/files/netbox.service | 12 ++++++++++++ type/__netbox/gencode-remote | 1 + type/__netbox/manifest | 10 ++++++---- type/__netbox_gunicorn/files/netbox.service | 3 ++- type/__netbox_gunicorn/gencode-remote | 6 ------ type/__netbox_gunicorn/man.rst | 3 +-- type/__netbox_uwsgi/files/netbox.service | 3 ++- type/__netbox_uwsgi/gencode-remote | 6 ------ type/__netbox_uwsgi/man.rst | 3 +-- 9 files changed, 25 insertions(+), 22 deletions(-) create mode 100644 type/__netbox/files/netbox.service diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service new file mode 100644 index 0000000..88baaeb --- /dev/null +++ b/type/__netbox/files/netbox.service @@ -0,0 +1,12 @@ +[Unit] +Description=NetBox Service Wrapper +Documentation=https://netbox.readthedocs.io/en/stable/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 0153579..d598aa8 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -98,6 +98,7 @@ fi if [ "$changes" = "yes" ]; then cat << EOF # Restart service. +service netbox restart service netbox-rq restart EOF fi diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 079ea48..95688d3 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -204,7 +204,9 @@ $SECRET_KEY SECRET -# Upload systemd unit for worker service -__systemd_unit netbox-rq.service \ - --source "$__type/files/netbox-rq.service" \ - --enablement-state enabled +# Upload systemd unit for worker and wsgi service +for unit in netbox netbox-rq; do + __systemd_unit $unit.service \ + --source "$__type/files/$unit.service" \ + --enablement-state enabled +done diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 475ddc4..2154d27 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -1,6 +1,8 @@ [Unit] Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ +PartOf=netbox.service +After=netbox.service After=network-online.target Wants=network-online.target @@ -19,5 +21,4 @@ RestartSec=30 PrivateTmp=true [Install] -Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index e424c22..163bb1a 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -19,12 +19,6 @@ if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then printf "configured\n" >> "$__messages_out" fi -# application -if grep -q "^__netbox:" "$__messages_in"; then - do_restart=yes - # no message cause it should be obvious -fi - # restart gunicorn if [ "$do_restart" ]; then diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 43f5864..ac86261 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -37,8 +37,7 @@ updated $old to $new configured Configuration for gunicorn changed. -In both cases, and at messages from the `__netbox` type, it restarts the -service to using the up-to-date version. +In both cases, it restarts the service to use the up-to-date version. EXAMPLES diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index cfce752..bc10571 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -1,6 +1,8 @@ [Unit] Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ +PartOf=netbox.service +After=netbox.service After=network-online.target Wants=network-online.target @@ -19,5 +21,4 @@ RestartSec=30 PrivateTmp=true [Install] -Alias=netbox.service WantedBy=multi-user.target diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 5cc82d1..3127b8b 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -19,12 +19,6 @@ if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then printf "configured\n" >> "$__messages_out" fi -# application -if grep -q "^__netbox:" "$__messages_in"; then - do_restart=yes - # no messages cause this is obvious -fi - # restart uwsgi if [ "$do_restart" ]; then diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 5bf3160..f328496 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -39,8 +39,7 @@ upgraded configured The uwsgi configuration got updated. -In both cases, and at messages from the `__netbox` type, it restarts the -service to using the up-to-date version. +In all cases, it restarts the service to use the up-to-date version. EXAMPLES From facb5a64d3824df095c6374a39f3a78482987046 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 1 Sep 2020 12:40:05 +0200 Subject: [PATCH 089/332] __netbox{,_gunicorn}: disabled shellcheck warnings Shellcheck warned about creating content for a python array. As the string will be printed literally into the config, the warning does not match to the current case. --- type/__netbox/manifest | 3 ++- type/__netbox_gunicorn/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 95688d3..aef50c1 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -38,9 +38,10 @@ export DATABASE_PORT # list of hosts ALLOWED_HOSTS="" while read -r hostname; do + # shellcheck disable=SC2089 ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," done < "$__object/parameter/host" -export ALLOWED_HOSTS +export ALLOWED_HOSTS # shellcheck disable=SC2090 if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index dab8181..d61cbe6 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -2,9 +2,10 @@ HOST="" while read -r host; do + # shellcheck disable=SC2089 HOST="$HOST '$host'," done < "$__object/parameter/bind-to" -export HOST +export HOST # shellcheck disable=SC2090 # process template mkdir "$__object/files" From dbc91cb33914e0b3c899dd169d61613f7a766c2a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Sep 2020 07:50:12 +0200 Subject: [PATCH 090/332] __netbox*: updated man pages --- type/__netbox/man.rst | 20 +++++++++++++++++--- type/__netbox_gunicorn/man.rst | 13 ++++++++++--- type/__netbox_uwsgi/man.rst | 13 ++++++++++--- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 6bb1e36..a3f283c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -12,8 +12,12 @@ This (singleton) type installs and configures a NetBox instance, a web application to help manage and document computer networks. It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. -Netbox will be run via `gnuicorn` as WSGI service. It setup systemd unit files -for the services `netbox` and `netbox-rq`. +It setup systemd unit files for the services `netbox` and `netbox-rq`. To +access the application through WSGI, uWSGI or Gunicorn can be used. The setup +can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. + +The Gunicorn setup is recommended from the NetBox documentation. Consult each +manual page to decide. The types must be called after the `__netbox` type. REQUIRED PARAMETERS @@ -36,6 +40,7 @@ host Hostname (domain or IP address) on which the application is served. Multiple hostnames are possible; given as multiple arguments. + OPTIONAL PARAMETERS ------------------- secret-key @@ -147,6 +152,7 @@ scripts-root needed. By default, it will be stored into the installation directory (``/opt/netbox/netbox/netbox/scripts``). + BOOLEAN PARAMETERS ------------------ redis-ssl @@ -171,6 +177,7 @@ update-notify Enables the NetBox version check for new upstream updates. It checks every 24 hours for new releases and notify the admin users in the gui if any. + MESSAGES -------- installed $VERSION @@ -199,6 +206,8 @@ EXAMPLES --ldap-group-base "ou=groups,dc=domain,dc=tld" \ --ldap-require-group "cn=netbox-login,ou=groups,dc=domain,dc=tld" \ --ldap-superuser-group "cn=netbox-admin,ou=groups,dc=domain,dc=tld" + # using recommended gunicorn setup + require="__netbox" __netbox_gunicorn NOTES @@ -216,9 +225,14 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. + SEE ALSO -------- -- `NetBox documentation `_ +`NetBox documentation `_ + +:strong:`cdist-type__netbox_gunicorn`\ (7) +:strong:`cdist-type__netbox_uwsgi`\ (7) + AUTHORS ------- diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index ac86261..c51be0c 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -3,12 +3,16 @@ cdist-type__netbox_uwsgi(7) NAME ---- -cdist-type__netbox_gunicorn - run netbox with gunicorn +cdist-type__netbox_gunicorn - Run NetBox with Gunicorn DESCRIPTION ----------- -This space intentionally left blank. +This (singleton) type installs Gunicorn into the NetBox `python-venv` to host +the NetBox WSGI application. It provides the application as HTTP over the given +sockets. Static content must be served independent of Gunicorn. The Gunicorn +daemon is available as the `gunicorn-netbox` systemd service, but also +available via the `netbox` wrapper service. REQUIRED PARAMETERS @@ -58,7 +62,10 @@ EXAMPLES SEE ALSO -------- -:strong:`__netbox`\ (7) +`Gunicorn Documentation `_ + +:strong:`cdist-type__netbox`\ (7) +:strong:`cdist-type__netbox_uwsgi`\ (7) AUTHORS diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index f328496..9d5ced7 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -3,12 +3,16 @@ cdist-type__netbox_uwsgi(7) NAME ---- -cdist-type__netbox_uwsgi - run netbox with uwsgi +cdist-type__netbox_uwsgi - Run NetBox with uWSGI DESCRIPTION ----------- -This space intentionally left blank. +This (singleton) type installs uWSGI into the NetBox `python-venv`. It hosts +the NetBox WSGI application over the WSGI protocol. A further server must be +installed to provide it as HTTP. This application is available via the +`uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper +service, too. REQUIRED PARAMETERS @@ -58,7 +62,10 @@ EXAMPLES SEE ALSO -------- -:strong:`TODO`\ (7) +`uWSGI Documentation `_ + +:strong:`cdist-type__netbox`\ (7) +:strong:`cdist-type__netbox_gunicorn`\ (7) AUTHORS From 2805b6beffc61c380ebffe26a487bbdf587c992b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 2 Sep 2020 09:15:45 +0200 Subject: [PATCH 091/332] __netbox{,_gunicorn}: fix shellcheck directives --- type/__netbox/manifest | 3 ++- type/__netbox_gunicorn/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index aef50c1..273423e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -41,7 +41,8 @@ while read -r hostname; do # shellcheck disable=SC2089 ALLOWED_HOSTS="$ALLOWED_HOSTS '$hostname'," done < "$__object/parameter/host" -export ALLOWED_HOSTS # shellcheck disable=SC2090 +# shellcheck disable=SC2090 +export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index d61cbe6..25ea452 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -5,7 +5,8 @@ while read -r host; do # shellcheck disable=SC2089 HOST="$HOST '$host'," done < "$__object/parameter/bind-to" -export HOST # shellcheck disable=SC2090 +# shellcheck disable=SC2090 +export HOST # process template mkdir "$__object/files" From 243e34f0a5e6d7e012c3eee3865cc995432f6f07 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 11:20:20 +0200 Subject: [PATCH 092/332] __netbox_uwsgi: enable further protocols to bind to Enables multiple protocols like fastcgi or HTTP to bind to. This makes it more flexible to use. Also, a little fix for __netbox was done: correctly output a error msg. --- type/__netbox/manifest | 2 +- type/__netbox_uwsgi/files/uwsgi.ini.sh | 29 +++++++++++++++++-- type/__netbox_uwsgi/man.rst | 29 +++++++++++++++---- .../parameter/optional_multiple | 4 +++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 273423e..cb6062e 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -127,7 +127,7 @@ fi export SMTP_USE_SSL if [ -f "$__object/parameter/smtp-use-tls" ]; then if [ "$SMTP_USE_SSL" = "True" ]; then - echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" + echo "options --smtp-use-ssl and --smtp-use-tls are not compatible" >&2 exit 2 fi SMTP_USE_TLS="True" diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 557a5fa..324b150 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -2,6 +2,18 @@ # Generates uwsgi config # see https://uwsgi-docs.readthedocs.io/en/latest/Options.html +# or https://uwsgi-docs-additions.readthedocs.io/en/latest/Options.html + +# params: +# 1: parameter name +# 2: parameter value file +# +# output: the lines for the configuration option +multi_options() { + while read -r line; do + printf "%s = %s\n" "$1" "$line" + done < "$2" +} # fix missing $__explorer # see https://code.ungleich.ch/ungleich-public/cdist/-/issues/834 @@ -13,8 +25,21 @@ cores="$(cat "$__explorer/cpu_cores")" cat << EOF [uwsgi] -; socket to bind -socket = $HOST +; socket(s) to bind +EOF + +# special protocol to bind +for param in $(find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print); do + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" + socket_changes=yes +done +# else, default bind to +if [ -z "$socket_changes" ]; then + multi_options "socket" "$__object/parameter/bind-to" +fi + + +cat << EOF ; processes and threads processes = $(( 2*cores + 1 )) diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 9d5ced7..7501573 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -9,10 +9,11 @@ cdist-type__netbox_uwsgi - Run NetBox with uWSGI DESCRIPTION ----------- This (singleton) type installs uWSGI into the NetBox `python-venv`. It hosts -the NetBox WSGI application over the WSGI protocol. A further server must be -installed to provide it as HTTP. This application is available via the -`uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper -service, too. +the NetBox WSGI application via the WSGI protocol. A further server must be +installed to provide it as HTTP and serve static content. It supports multiple +protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This +application is available via the `uwsgi-netbox` systemd service. It is +controllable via the `netbox` wrapper service, too. REQUIRED PARAMETERS @@ -24,7 +25,15 @@ OPTIONAL PARAMETERS ------------------- bind-to The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. - Defaults to ``127.0.0.1:3031``. + Defaults to ``127.0.0.1:3031``. Can be set multiple times. + +uwsgi-bind +http-bind +fastcgi-bind +scgi-bind + Bind the application to a specific protocol instead of implicit uwsgi via + ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must + be a UNIX/TCP socket. Can be set multiple times. BOOLEAN PARAMETERS @@ -55,10 +64,18 @@ EXAMPLES __netbox $args require="__netbox" __netbox_uwsgi - # with special bind + # with multiple binds + __netbox $args require="__netbox" __netbox_uwsgi --bind-to 0.0.0.0:3032 \ --bind-to 0.0.0.0:3033 + # with multiple protocols + # parameter `--bind-to` will be ignored + __netbox $args + require="__netbox" __netbox_uwsgi --uwsgi-bind 0.0.0.0:3031 \ + --http-bind 0.0.0.0:8080 \ + --fastcgi-bind 1.2.3.4:5678 + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/parameter/optional_multiple b/type/__netbox_uwsgi/parameter/optional_multiple index 331c077..3f3e7d4 100644 --- a/type/__netbox_uwsgi/parameter/optional_multiple +++ b/type/__netbox_uwsgi/parameter/optional_multiple @@ -1 +1,5 @@ bind-to +uwsgi-bind +http-bind +fastcgi-bind +scgi-bind From af91b6e3a1ad15f6579983e25570a0bda8796a9b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 11:26:15 +0200 Subject: [PATCH 093/332] __netbox_uwsgi: mark as singleton This should have been done much earlier .. :/ --- type/__netbox_uwsgi/singleton | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 type/__netbox_uwsgi/singleton diff --git a/type/__netbox_uwsgi/singleton b/type/__netbox_uwsgi/singleton new file mode 100644 index 0000000..e69de29 From 314a0d4d8ef6235c70fbf3d7b1421f1b34cb18c2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 13:53:05 +0200 Subject: [PATCH 094/332] __netbox: kill all running venv processes To avoid aborts because of the python venv could not be updated by killing all processes that uses the venv. It will be done all times to prevent any error, because it could not be reliably detected if the type installs or updates NetBox. --- type/__netbox/gencode-remote | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d598aa8..f200408 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -30,6 +30,17 @@ tar xf '$archive' cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" +EOF + + # Stop everything in the pyenv to update + cat << EOF +# Try to kill everything in the venv +systemctl stop netbox gunicorn-netbox uwsgi-netbox || true +ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true + +EOF + + cat << EOF # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' From 755bd9098ede2dbad6c557f7c4983037fe3aab22 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 15:18:49 +0200 Subject: [PATCH 095/332] __netbox*: update systemd service files The `uwsgi-netbox` service now works, also the `netbox` wrapper service. The PID file was removed from the Gunicorn service as it is not required and a bit more efford to move it to `/run/` due to permissions. Generally, all depend on `network.target` instead of `network-online.target` now, and signals for reload, stop and kill were added (especially required the uwsgi service). --- type/__netbox/files/netbox.service | 5 +++-- type/__netbox_gunicorn/files/netbox.service | 14 ++++++++------ type/__netbox_uwsgi/files/netbox.service | 16 +++++++++------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service index 88baaeb..738e178 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox/files/netbox.service @@ -1,12 +1,13 @@ [Unit] Description=NetBox Service Wrapper Documentation=https://netbox.readthedocs.io/en/stable/ -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=oneshot RemainAfterExit=yes +ExecStart=/bin/true [Install] WantedBy=multi-user.target diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 2154d27..55f3549 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -3,22 +3,24 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service After=netbox.service -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=simple User=netbox Group=netbox -PIDFile=/var/tmp/netbox.pid WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/gunicorn --pid /var/tmp/netbox.pid --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi +ExecStart=/opt/netbox/venv/bin/gunicorn --pythonpath /opt/netbox/netbox --config /opt/netbox/gunicorn.py netbox.wsgi +# signals: https://docs.gunicorn.org/en/stable/signals.html +ExecReload=kill -HUP $MAINPID +ExecStop=kill -TERM $MAINPID +KillSignal=SIGQUIT Restart=on-failure RestartSec=30 -PrivateTmp=true [Install] -WantedBy=multi-user.target +WantedBy=netbox.service diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index bc10571..10ea734 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -3,22 +3,24 @@ Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service After=netbox.service -After=network-online.target -Wants=network-online.target +After=network.target +Wants=network.target [Service] Type=simple User=netbox Group=netbox -PIDFile=/var/tmp/netbox.pid -WorkingDirectory=/opt/netbox +WorkingDirectory=/opt/netbox/netbox/ -ExecStart=/opt/netbox/venv/bin/uwsgi --master --wsgi-file netbox/netbox/wsgi.py uwsgi.ini +ExecStart=/opt/netbox/venv/bin/uwsgi --master --module netbox.wsgi uwsgi.ini +# signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi +ExecReload=kill -HUP $MAINPID +ExecStop=kill -INT $MAINPID +KillSignal=SIGQUIT Restart=on-failure RestartSec=30 -PrivateTmp=true [Install] -WantedBy=multi-user.target +WantedBy=netbox.service From 5a403de0571b57c0da07824c91d703a21231e303 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 19:09:34 +0200 Subject: [PATCH 096/332] __netbox_uwsgi: fix shellcheck Fixed shellcheck cause of the find command. This required a bigger change due to variables and subshell. Now, input is used through a here-document. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 10 +++++++--- type/__netbox_uwsgi/manifest | 3 --- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 324b150..0f85804 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -29,10 +29,14 @@ cat << EOF EOF # special protocol to bind -for param in $(find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print); do +while read -r param; do multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" - socket_changes=yes -done + socket_changes="yes" + +done << INPUT # here-doc cause of SC2031 +$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print ) +INPUT + # else, default bind to if [ -z "$socket_changes" ]; then multi_options "socket" "$__object/parameter/bind-to" diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index 779f93b..3d9c1b4 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,8 +1,5 @@ #!/bin/sh -e -HOST="$__object/parameter/bind-to" -export HOST - # process template mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" From ffba3ae776ac4e4bd67d4812b8073c2e81a69cdd Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 21:48:04 +0200 Subject: [PATCH 097/332] __netbox_uwsgi: add --serve-static parameter New option to directly serve static content via uWSGI. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 11 +++++++++++ type/__netbox_uwsgi/man.rst | 11 ++++++++++- type/__netbox_uwsgi/manifest | 6 ++++++ type/__netbox_uwsgi/parameter/boolean | 1 + 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 type/__netbox_uwsgi/parameter/boolean diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 0f85804..4abc59a 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -43,9 +43,20 @@ if [ -z "$socket_changes" ]; then fi +# multi-process settings cat << EOF ; processes and threads processes = $(( 2*cores + 1 )) threads = 2 EOF + + +# optional mapping of static content +if [ "$STATIC_MAP" != "" ]; then + cat << EOF + +; map static content +static-map = /static=/opt/netbox/netbox/static +EOF +fi diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 7501573..f5eff0b 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -38,7 +38,12 @@ scgi-bind BOOLEAN PARAMETERS ------------------ -None. +serve-static + Setup uWSGI to serve the static content, too. This is generally not + recommended for real production setups, as it is the job of the reverse + proxy server, who will thread it as static cachable content. This option + is only recommended for small setups or direct usage of the uWSGI socket + like using it as standalone HTTP server for NetBox. MESSAGES @@ -76,6 +81,10 @@ EXAMPLES --http-bind 0.0.0.0:8080 \ --fastcgi-bind 1.2.3.4:5678 + # as standalone server + __netbox $args + require="__netbox" __netbox_uwsgi --serve-static --http-bind 0.0.0.0:80 + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index 3d9c1b4..bb4f23f 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,5 +1,11 @@ #!/bin/sh -e +# *bind* parameters are directly processed in the gen script +if [ -f "$__object/parameter/serve-static" ]; then + STATIC_MAP="yes" + export STATIC_MAP +fi + # process template mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" diff --git a/type/__netbox_uwsgi/parameter/boolean b/type/__netbox_uwsgi/parameter/boolean new file mode 100644 index 0000000..aa08196 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/boolean @@ -0,0 +1 @@ +serve-static From 090a8f015e86f8dd0385b0b37a234a2538c254da Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:08:37 +0200 Subject: [PATCH 098/332] __netbox_uwsgi: fix uwsgi netbox service file Revert working directory changes as the configuration file still needs to be accessable. An absolute path would work, too, but it is not the preferred way. To still work with the python wsgi application, `--chdir` is used. --- type/__netbox_uwsgi/files/netbox.service | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index 10ea734..8d41cd0 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -11,9 +11,9 @@ Type=simple User=netbox Group=netbox -WorkingDirectory=/opt/netbox/netbox/ +WorkingDirectory=/opt/netbox -ExecStart=/opt/netbox/venv/bin/uwsgi --master --module netbox.wsgi uwsgi.ini +ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi ExecReload=kill -HUP $MAINPID ExecStop=kill -INT $MAINPID From 9d330a91bc5bdbdbe69a4db98ddad0eeaed344cb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:27:42 +0200 Subject: [PATCH 099/332] __netbox_{gunicorn,uwsgi}: add netbox ownership to config file --- type/__netbox_gunicorn/manifest | 3 ++- type/__netbox_uwsgi/manifest | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index 25ea452..b72b8fd 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -14,7 +14,8 @@ mkdir "$__object/files" # gunicorn config file __file /opt/netbox/gunicorn.py \ - --mode 644 --source "$__object/files/gunicorn.py" + --mode 644 --owner netbox \ + --source "$__object/files/gunicorn.py" # install service file __systemd_unit gunicorn-netbox.service \ diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index bb4f23f..b6c0308 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -13,7 +13,8 @@ mkdir "$__object/files" # uwsgi config file # TODO maybe patching with __key_value cause of .ini ? __file /opt/netbox/uwsgi.ini \ - --mode 644 --source "$__object/files/uwsgi.ini" + --mode 644 --owner netbox \ + --source "$__object/files/uwsgi.ini" # install service file __systemd_unit uwsgi-netbox.service \ From b10f6b71d051377f023fd6e96abbc00ff06e5d5d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:28:48 +0200 Subject: [PATCH 100/332] __netbox: avoid useless generated code-remote Because `set -e` got printed all the time, the type __netbox always had some generated code for the remote side. This line was removed because this is already done by cdist when executing the code-remote script. Rather, the exit-on-error option was set to some scirpts (two ..). --- type/__netbox/gencode-remote | 4 +--- type/__netbox/manifest | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index f200408..9ae6b5b 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -1,6 +1,4 @@ -#!/bin/sh - -echo "set -e" +#!/bin/sh -e old_version="$(cat "$__object/explorer/version")" VERSION=$(cat "$__object/parameter/version") diff --git a/type/__netbox/manifest b/type/__netbox/manifest index cb6062e..5f1762a 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e os=$(cat "$__global/explorer/os") From 3b07a660b3a7dd442e3184600fad25ed27769a5f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 5 Sep 2020 22:55:43 +0200 Subject: [PATCH 101/332] __netbox_uwsgi: add hint to the --serve-static parameter --- type/__netbox_uwsgi/man.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index f5eff0b..6826cca 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -45,6 +45,11 @@ serve-static is only recommended for small setups or direct usage of the uWSGI socket like using it as standalone HTTP server for NetBox. + **Hint**: This parameter does not work in junction with the `__netbox` + parameter ``--basepath``. It is because this type does not know the + parameter value and this case is very unlikly to happen; although an + implementation is not difficult. + MESSAGES -------- From c9e4e8d7dce88c3f82a4bbf053b174d88a70a4ce Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 13:44:03 +0200 Subject: [PATCH 102/332] __netbox_{gunicorn,uwsgi}: add state parameter Adds the --state parameter to both types. With it, the transition between both types can be done smothly. --- type/__netbox_gunicorn/gencode-remote | 59 +++++++++++------ type/__netbox_gunicorn/man.rst | 40 ++++++++++-- type/__netbox_gunicorn/manifest | 64 ++++++++++++++----- .../__netbox_gunicorn/parameter/default/state | 1 + type/__netbox_gunicorn/parameter/optional | 1 + type/__netbox_uwsgi/gencode-remote | 58 +++++++++++------ type/__netbox_uwsgi/man.rst | 32 +++++++++- type/__netbox_uwsgi/manifest | 62 +++++++++++++----- type/__netbox_uwsgi/parameter/default/state | 1 + type/__netbox_uwsgi/parameter/optional | 1 + 10 files changed, 244 insertions(+), 75 deletions(-) create mode 100644 type/__netbox_gunicorn/parameter/default/state create mode 100644 type/__netbox_gunicorn/parameter/optional create mode 100644 type/__netbox_uwsgi/parameter/default/state create mode 100644 type/__netbox_uwsgi/parameter/optional diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 163bb1a..0da92f0 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -1,29 +1,50 @@ #!/bin/sh -e -curr_installed="$(cat "$__object/explorer/installed")" -should_installed="$(cat "$__object/explorer/should_installed")" +# control state +state="$(cat "$__object/parameter/state")" -# gunicorn version change -if [ "$curr_installed" != "$should_installed" ]; then - # (re)installing gunicorn - echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" +case "$state" in + # install gunicorn + enabled|disabled) + curr_installed="$(cat "$__object/explorer/installed")" + should_installed="$(cat "$__object/explorer/should_installed")" - do_restart=yes - printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ - >> "$__messages_out" -fi + # gunicorn version change + if [ "$curr_installed" != "$should_installed" ]; then + # (re)installing gunicorn + echo "/opt/netbox/venv/bin/pip3 install 'gunicorn==$should_installed'" -# configuration changes -if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then - do_restart=yes - printf "configured\n" >> "$__messages_out" -fi + if [ "$curr_installed" != "" ]; then + printf "updated %s to %s\n" "$curr_installed" "$should_installed" \ + >> "$__messages_out" + else + printf "installed\n" >> "$__messages_out" + fi + do_restart=yes + fi + + # configuration changes + if grep -q "^__file/opt/netbox/gunicorn.py:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" + fi -# restart gunicorn -if [ "$do_restart" ]; then - cat << EOF + # restart gunicorn + if [ "$do_restart" ] && [ "$state" != "disabled" ]; then + cat << EOF # Restart service service gunicorn-netbox restart EOF -fi + fi + ;; + + # uninstall + absent) + # check if installed + if [ -s "$__object/explorer/installed" ]; then + # service already disabled + echo "/opt/netbox/venv/bin/pip3 uninstall -y gunicorn" + printf "uninstalled\n" >> "$__messages_out" + fi +esac diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index c51be0c..1c09d78 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -1,5 +1,5 @@ -cdist-type__netbox_uwsgi(7) -=========================== +cdist-type__netbox_gunicorn(7) +============================== NAME ---- @@ -22,6 +22,20 @@ None. OPTIONAL PARAMETERS ------------------- +state + Represents the state of the Gunciron application. Defaults to ``enabled``. + + enabled + The Gunicorn service is enabled and running. + disabled + The Gunicorn service is installed, but disabled. + absent + The uWSGI service is not installed and all configuration removed. + + This type does not guarantee anything about the running state of the + service. To be sure about the service is stopped or not, use the type + :strong:`cdist-type__systemd_service`\ (7) after this execution. + bind-to The hosts the gunicorn socket should be bind to. Formats are `IP`, `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple @@ -35,13 +49,20 @@ None. MESSAGES -------- -updated $old to $new +installed + The software was installed. + +upgraded $old to $new The version of the gunicorn software was updated from `$old` to `$new`. configured Configuration for gunicorn changed. -In both cases, it restarts the service to use the up-to-date version. +uninstalled + The Gunicorn application was removed. + +In all cases where the application is still present, it restarts the service to +use the up-to-date version. EXAMPLES @@ -59,6 +80,17 @@ EXAMPLES --bind-to 0.0.0.0:8001 \ --bind-to 1.2.3.4:5678 + # replace uwsgi with gunicorn + __netbox $args + require="__netbox" __netbox_uwsgi --state absent + # it should depend on __netbox_uwsgi if they use the same socket + require="__netbox_uwsgi" __netbox_gunicorn --state enabled + + # be sure the service is disabled + __netbox $args + require="__netbox" __netbox_gunicorn --state disabled + require="__netbox_gunicorn" __systemd_service gunicorn-netbox --state stopped + SEE ALSO -------- diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index b72b8fd..2e5ccce 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -1,23 +1,55 @@ #!/bin/sh -e +# __netbox_gunicorn/manifest -HOST="" -while read -r host; do - # shellcheck disable=SC2089 - HOST="$HOST '$host'," -done < "$__object/parameter/bind-to" -# shellcheck disable=SC2090 -export HOST +# Check states +state="" +unit_state="" +param_state="$(cat "$__object/parameter/state")" -# process template -mkdir "$__object/files" -"$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" +case "$param_state" in + enabled|disabled) + state="present" + unit_state="$param_state" + ;; + + absent) + state="absent" + unit_state="disabled" + ;; + + *) + # does not exist + printf "The state '%s' does not exist, can't continue!\n" "$param_state" >&2 + exit 2 + ;; +esac + + +if [ "$state" = "present" ]; then + HOST="" + while read -r host; do + # shellcheck disable=SC2089 + HOST="$HOST '$host'," + done < "$__object/parameter/bind-to" + # shellcheck disable=SC2090 + export HOST + + # process template + mkdir "$__object/files" + "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" + + # gunicorn config file + __file /opt/netbox/gunicorn.py \ + --mode 644 --owner netbox \ + --source "$__object/files/gunicorn.py" + +else + # absent config file + __file /opt/netbox/gunicorn.py --state absent +fi -# gunicorn config file -__file /opt/netbox/gunicorn.py \ - --mode 644 --owner netbox \ - --source "$__object/files/gunicorn.py" # install service file __systemd_unit gunicorn-netbox.service \ - --source "$__type/files/netbox.service" \ - --enablement-state enabled --restart + --state "$state" --enablement-state "$unit_state" \ + --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_gunicorn/parameter/default/state b/type/__netbox_gunicorn/parameter/default/state new file mode 100644 index 0000000..86981e6 --- /dev/null +++ b/type/__netbox_gunicorn/parameter/default/state @@ -0,0 +1 @@ +enabled diff --git a/type/__netbox_gunicorn/parameter/optional b/type/__netbox_gunicorn/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__netbox_gunicorn/parameter/optional @@ -0,0 +1 @@ +state diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 3127b8b..9b6a603 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -1,29 +1,47 @@ #!/bin/sh -e -# not installed -if ! [ -s "$__object/explorer/installed" ]; then - echo "/opt/netbox/venv/bin/pip3 install uwsgi" - do_restart=yes - printf "installed\n" >> "$__messages_out" +# control state +state="$(cat "$__object/parameter/state")" -# updates available -elif [ -s "$__object/explorer/upgradeable" ]; then - echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" - do_restart=yes - printf "upgraded\n" >> "$__messages_out" -fi +case "$state" in + # install uwsgi + enabled|disabled) + # not installed + if ! [ -s "$__object/explorer/installed" ]; then + echo "/opt/netbox/venv/bin/pip3 install uwsgi" + do_restart=yes + printf "installed\n" >> "$__messages_out" -# changed configuration -if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then - do_restart=yes - printf "configured\n" >> "$__messages_out" -fi + # updates available + elif [ -s "$__object/explorer/upgradeable" ]; then + echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + do_restart=yes + printf "upgraded\n" >> "$__messages_out" + fi + + # changed configuration + if grep -q "^__file/opt/netbox/uwsgi.ini:" "$__messages_in"; then + do_restart=yes + printf "configured\n" >> "$__messages_out" + fi -# restart uwsgi -if [ "$do_restart" ]; then - cat << EOF + # restart uwsgi + if [ "$do_restart" ] && [ "$state" != "disabled" ]; then + cat << EOF # Restart service service uwsgi-netbox restart EOF -fi + fi + ;; + + # uninstall + absent) + # check if installed + if [ -s "$__object/explorer/installed" ]; then + # service already disabled + echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi" + printf "uninstalled\n" >> "$__messages_out" + fi + ;; +esac diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 6826cca..3fb8515 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -23,6 +23,21 @@ None. OPTIONAL PARAMETERS ------------------- +state + Represents the state of the uWSGI application. Defaults to ``enabled``. + + enabled + The uWSGI service is enabled and running. + disabled + The uWSGI service is installed, but disabled. + absent + The uWSGI service is not installed and all configuration removed. + + This type does not guarantee anything about the running state of the + service. To be sure about the service is stopped or not, use the type + :strong:`cdist-type__systemd_service`\ (7) after this execution. + + bind-to The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. Defaults to ``127.0.0.1:3031``. Can be set multiple times. @@ -62,7 +77,11 @@ upgraded configured The uwsgi configuration got updated. -In all cases, it restarts the service to use the up-to-date version. +uninstalled + The uWSGI application was removed. + +In all cases where the application is still present, it restarts the service to +use the up-to-date version. EXAMPLES @@ -90,6 +109,17 @@ EXAMPLES __netbox $args require="__netbox" __netbox_uwsgi --serve-static --http-bind 0.0.0.0:80 + # replace gunicorn with uwsgi + __netbox $args + require="__netbox" __netbox_gunicorn --state absent + # it should depend on __netbox_gunicorn if they use the same socket + require="__netbox_gunicorn" __netbox_uwsgi --state enabled + + # be sure the service is disabled + __netbox $args + require="__netbox" __netbox_uwsgi --state disabled + require="__netbox_uwsgi" __systemd_service uwsgi-netbox --state stopped + SEE ALSO -------- diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index b6c0308..c5885c9 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -1,22 +1,54 @@ #!/bin/sh -e +# __netbox_uwsgi/manifest -# *bind* parameters are directly processed in the gen script -if [ -f "$__object/parameter/serve-static" ]; then - STATIC_MAP="yes" - export STATIC_MAP +# Check states +state="" +unit_state="" +param_state="$(cat "$__object/parameter/state")" + +case "$param_state" in + enabled|disabled) + state="present" + unit_state="$param_state" + ;; + + absent) + state="absent" + unit_state="disabled" + ;; + + *) + # does not exist + printf "The state '%s' does not exist, can't continue!\n" "$param_state" >&2 + exit 2 + ;; +esac + + +if [ "$state" = "present" ]; then + # *bind* parameters are directly processed in the gen script + if [ -f "$__object/parameter/serve-static" ]; then + STATIC_MAP="yes" + export STATIC_MAP + fi + + # process template + mkdir "$__object/files" + "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" + + # uwsgi config file + # TODO maybe patching with __key_value cause of .ini ? + __file /opt/netbox/uwsgi.ini \ + --mode 644 --owner netbox \ + --source "$__object/files/uwsgi.ini" + +else + # absent config file + __file /opt/netbox/uwsgi.ini --state absent fi -# process template -mkdir "$__object/files" -"$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" - -# uwsgi config file -# TODO maybe patching with __key_value cause of .ini ? -__file /opt/netbox/uwsgi.ini \ - --mode 644 --owner netbox \ - --source "$__object/files/uwsgi.ini" # install service file __systemd_unit uwsgi-netbox.service \ - --source "$__type/files/netbox.service" \ - --enablement-state enabled --restart + --state "$state" --enablement-state "$unit_state" \ + --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/parameter/default/state b/type/__netbox_uwsgi/parameter/default/state new file mode 100644 index 0000000..86981e6 --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/state @@ -0,0 +1 @@ +enabled diff --git a/type/__netbox_uwsgi/parameter/optional b/type/__netbox_uwsgi/parameter/optional new file mode 100644 index 0000000..ff72b5c --- /dev/null +++ b/type/__netbox_uwsgi/parameter/optional @@ -0,0 +1 @@ +state From 0afc7136f8e4f304f179965f229ed368ce73abb6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 6 Sep 2020 18:39:07 +0200 Subject: [PATCH 103/332] __netbox_uwsgi: fix if only --bind-to given If the here-document is empty because no file could be found, an empty line will be read. The new line of code will skip in those cases. --- type/__netbox_uwsgi/files/uwsgi.ini.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 4abc59a..7835a3e 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -30,6 +30,8 @@ EOF # special protocol to bind while read -r param; do + if [ -z "$param" ]; then continue; fi # ignore empty lines from the here-doc + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" socket_changes="yes" From eed3515424668b8684058b0d02368746ee5f5c06 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Sep 2020 17:10:59 +0200 Subject: [PATCH 104/332] __netbox: add --ldap-staff-group parameter Required to get "admin access". --- type/__netbox/files/ldap_config.py.sh | 15 +++++++++++---- type/__netbox/man.rst | 4 ++++ type/__netbox/manifest | 4 ++++ type/__netbox/parameter/optional | 1 + 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/type/__netbox/files/ldap_config.py.sh b/type/__netbox/files/ldap_config.py.sh index a49d800..4e6b099 100755 --- a/type/__netbox/files/ldap_config.py.sh +++ b/type/__netbox/files/ldap_config.py.sh @@ -53,6 +53,8 @@ AUTH_LDAP_GROUP_TYPE = PosixGroupType() # Mirror LDAP group assignments. AUTH_LDAP_MIRROR_GROUPS = True +# For more granular permissions, map LDAP groups to Django groups. +AUTH_LDAP_FIND_GROUP_PERMS = True EOF if [ "$LDAP_REQUIRE_GROUP" != "" ]; then @@ -63,13 +65,18 @@ AUTH_LDAP_REQUIRE_GROUP = "$LDAP_REQUIRE_GROUP" EOF fi - if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then - cat << EOF + cat << EOF # Define special user types using groups. Exercise great caution when assigning superuser status. AUTH_LDAP_USER_FLAGS_BY_GROUP = { - "is_superuser": "$LDAP_SUPERUSER_GROUP", -} EOF + # superuser + if [ "$LDAP_SUPERUSER_GROUP" != "" ]; then + echo " \"is_superuser\": \"$LDAP_SUPERUSER_GROUP\"," fi + # staff user + if [ "$LDAP_STAFF_GROUP" != "" ]; then + echo " \"is_staff\": \"$LDAP_STAFF_GROUP\"," + fi + echo "}" fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index a3f283c..bc4bbc9 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -78,6 +78,10 @@ ldap-group-base ldap-require-group Group required to login. +ldap-staff-group + Make members of this group to "staff". This gives the users "Admin Access", + which means access to the "NetBox Administration" site. + ldap-superuser-group Make members of this groups superusers. diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 5f1762a..3681ea0 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -89,6 +89,10 @@ if [ -f "$__object/parameter/ldap-superuser-group" ]; then LDAP_SUPERUSER_GROUP=$(cat "$__object/parameter/ldap-superuser-group") export LDAP_SUPERUSER_GROUP fi +if [ -f "$__object/parameter/ldap-staff-group" ]; then + LDAP_STAFF_GROUP="$(cat "$__object/parameter/ldap-staff-group")" + export LDAP_STAFF_GROUP +fi # export if base ldap parameters are used export USE_LDAP diff --git a/type/__netbox/parameter/optional b/type/__netbox/parameter/optional index 47722e6..9495f7a 100644 --- a/type/__netbox/parameter/optional +++ b/type/__netbox/parameter/optional @@ -7,6 +7,7 @@ ldap-bind-password ldap-user-base ldap-group-base ldap-require-group +ldap-staff-group ldap-superuser-group redis-host redis-port From 3389752decfb48975f539b1702d37fe07d4e7755 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 7 Sep 2020 17:23:22 +0200 Subject: [PATCH 105/332] __netbox: fix detection of the secret-key to use Mixed up a parameter. Should be correct now. --- type/__netbox/manifest | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 3681ea0..06d0c77 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -47,13 +47,13 @@ export ALLOWED_HOSTS if [ -f "$__object/parameter/secret-key" ]; then SECRET_KEY=$(cat "$__object/parameter/secret-key") elif [ -s "$__object/explorer/secretkey" ]; then + # take the key that is already used + SECRET_KEY="$(cat "$__object/explorer/secretkey")" +else # Can be done over netbox/generate_secret_key.py too, but it's to # complicated with the variable setup (can't generated right now!). # Generates a 50-character long key (without ' cause of python quotes) SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" -else - # take the key that is already used - SECRET_KEY="$(cat "$__object/explorer/secretkey")" fi export SECRET_KEY From 398a3da10eeaf67caa042b2a99a1a3e652b31e0f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 18:10:15 +0200 Subject: [PATCH 106/332] __netbox: fix gerneration random generated secret key Because `/dev/random` was used, the `cdist config` could hang a long time to get real random values. The pseudo-generated values through `/dev/urandom` are fully enought for the secret key. --- type/__netbox/manifest | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 06d0c77..4ad7901 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -50,10 +50,11 @@ elif [ -s "$__object/explorer/secretkey" ]; then # take the key that is already used SECRET_KEY="$(cat "$__object/explorer/secretkey")" else - # Can be done over netbox/generate_secret_key.py too, but it's to - # complicated with the variable setup (can't generated right now!). + # Can be done over netbox/generate_secret_key.py too, but it can't be + # generated right now where it's required (only if it's preloaded for + # this type to execute it now). # Generates a 50-character long key (without ' cause of python quotes) - SECRET_KEY="$(tr -cd '[:graph:]' < /dev/random | tr -d \' | head -c50)" + SECRET_KEY="$(tr -cd '[:graph:]' < /dev/urandom | tr -d \' | head -c50)" fi export SECRET_KEY From 549feb87f96518a38ab7d3368bd3f2a2ed552d0d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 19:20:22 +0200 Subject: [PATCH 107/332] __netbox: fix --update-notify Template script for the `configuration.py` interpreted the boolean flag incorrectly and did the thing the user do not wanted. --- type/__netbox/files/configuration.py.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index ea73133..06a1c86 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -272,12 +272,12 @@ EOF if [ "$UPDATE_CHECK" != "" ]; then cat << EOF -RELEASE_CHECK_URL = None +RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' EOF else cat << EOF -RELEASE_CHECK_URL = 'https://api.github.com/repos/netbox-community/netbox/releases' +RELEASE_CHECK_URL = None EOF fi From 5d437839f68da0c29787172ea3daa3170926f8d2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 8 Sep 2020 19:52:03 +0200 Subject: [PATCH 108/332] __netbox: replaced secret key character set The used character set should now only contain characters used by the `netbox/generate_secret_key.py` program. Finially got correct escaping. --- type/__netbox/manifest | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 4ad7901..d754ae6 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -53,8 +53,9 @@ else # Can be done over netbox/generate_secret_key.py too, but it can't be # generated right now where it's required (only if it's preloaded for # this type to execute it now). - # Generates a 50-character long key (without ' cause of python quotes) - SECRET_KEY="$(tr -cd '[:graph:]' < /dev/urandom | tr -d \' | head -c50)" + # Generates a 50-character long key with the same character set like + # the helper script. Must escape the '-' to be no character range. + SECRET_KEY="$(tr -cd '!@#$%^&*(\-_=+)[:alnum:]' < /dev/urandom | head -c50)" fi export SECRET_KEY From dd167f075d25ecec4f1aca461658d2b93ff1ae8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 9 Sep 2020 19:08:46 +0200 Subject: [PATCH 109/332] __netbox*: fix service restart order Cause of corrupt databases if the services are restarted incorrectly, the order and dependencies are adjusted. Now, the `netbox-rq` service will be included in restarts of `netbox` and required for the WSGI servers that it must running. For these changes, the restart command of `__netbox` was adjusted. The other ones where edited too, to use the same command. All services now require redis and postgresql to be started before them to prevent any start order issues. If someone asked for what the RQ worker is required, see here: https://netbox.readthedocs.io/en/stable/additional-features/webhooks/#webhook-processing --- type/__netbox/files/netbox-rq.service | 7 ++++-- type/__netbox/files/netbox.service | 2 +- type/__netbox/gencode-remote | 7 +++--- type/__netbox/man.rst | 24 ++++++++++++++++++++- type/__netbox_gunicorn/files/netbox.service | 4 +++- type/__netbox_gunicorn/gencode-remote | 2 +- type/__netbox_uwsgi/files/netbox.service | 4 +++- type/__netbox_uwsgi/gencode-remote | 2 +- 8 files changed, 41 insertions(+), 11 deletions(-) diff --git a/type/__netbox/files/netbox-rq.service b/type/__netbox/files/netbox-rq.service index 77d7091..330e675 100644 --- a/type/__netbox/files/netbox-rq.service +++ b/type/__netbox/files/netbox-rq.service @@ -1,8 +1,11 @@ [Unit] Description=NetBox Request Queue Worker Documentation=https://netbox.readthedocs.io/en/stable/ -After=network-online.target -Wants=network-online.target +PartOf=netbox.service +Wants=network.target +After=netbox.service +After=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox/files/netbox.service b/type/__netbox/files/netbox.service index 738e178..68010e9 100644 --- a/type/__netbox/files/netbox.service +++ b/type/__netbox/files/netbox.service @@ -1,8 +1,8 @@ [Unit] Description=NetBox Service Wrapper Documentation=https://netbox.readthedocs.io/en/stable/ -After=network.target Wants=network.target +After=network.target [Service] Type=oneshot diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 9ae6b5b..0370949 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -105,9 +105,10 @@ fi # check for changes if [ "$changes" = "yes" ]; then + # to avoid database corruption at config changes, both services must be + # stopped at the same time (noted in the manual, too). cat << EOF -# Restart service. -service netbox restart -service netbox-rq restart +# Restart service. All required services are included with netbox.service. +systemctl restart netbox EOF fi diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index bc4bbc9..3330b1b 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -12,7 +12,11 @@ This (singleton) type installs and configures a NetBox instance, a web application to help manage and document computer networks. It installs it with the user ``netbox`` at ``/opt/netbox`` with `python-venv`. -It setup systemd unit files for the services `netbox` and `netbox-rq`. To +It setup systemd unit files for the services `netbox` and `netbox-rq`. The +`netbox` service only wrap all netbox related services, e.g. restarting and +so one will be delegated to all related services. + +The application is still not accessable because WSGI server is required. To access the application through WSGI, uWSGI or Gunicorn can be used. The setup can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. @@ -229,6 +233,24 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. +If you change a configuration, the database may go corrupt if two instances of +the application are running with different configurations at the same time. +This most commonly happens when the WSGI server and RQ-worker restarts after a +configuration change. This occours in the following case for example: + +.. code-block:: sh + + systemctl restart gunicorn-netbox # WSGI-server already online with new + # configuration after this command. + systemctl restart netbox-rq # RQ-Worker still worked with the old + # configuration till here. + +This type handles the restart of both services correctly to avoid such database +corruptions. To safely manual restart the whole netbox instance manual, simply +restart all services in one ``systemctl restart netbox`` command, as it ensures +that first all services are shut down before starting one of them. The service +``netbox`` wraps all required services that are available. + SEE ALSO -------- diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index 55f3549..dcf1a66 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -2,9 +2,11 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service +Requires=netbox-rq.service +Wants=network.target After=netbox.service After=network.target -Wants=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox_gunicorn/gencode-remote b/type/__netbox_gunicorn/gencode-remote index 0da92f0..4fae788 100755 --- a/type/__netbox_gunicorn/gencode-remote +++ b/type/__netbox_gunicorn/gencode-remote @@ -34,7 +34,7 @@ case "$state" in if [ "$do_restart" ] && [ "$state" != "disabled" ]; then cat << EOF # Restart service -service gunicorn-netbox restart +systemctl restart gunicorn-netbox EOF fi ;; diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service index 8d41cd0..cd06e82 100644 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service @@ -2,9 +2,11 @@ Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service +Requires=netbox-rq.service +Wants=network.target After=netbox.service After=network.target -Wants=network.target +After=redis-server.service postgresql.service [Service] Type=simple diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index 9b6a603..c219643 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -30,7 +30,7 @@ case "$state" in if [ "$do_restart" ] && [ "$state" != "disabled" ]; then cat << EOF # Restart service -service uwsgi-netbox restart +systemctl restart uwsgi-netbox EOF fi ;; From e800f42a6d95b9378516252d53944383f673653d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 9 Sep 2020 20:33:20 +0200 Subject: [PATCH 110/332] __netbox: consistency with __systemd_unit --restart This commit brings consistency into the --restart parameter for systemd units. All units except the netbox wrapper service will be restarted on unit change. --- type/__netbox/manifest | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index d754ae6..f8ed5fc 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -213,8 +213,10 @@ SECRET # Upload systemd unit for worker and wsgi service -for unit in netbox netbox-rq; do - __systemd_unit $unit.service \ - --source "$__type/files/$unit.service" \ - --enablement-state enabled -done +# does not restart netbox on change cause it only restart all other services +__systemd_unit netbox.service \ + --source "$__type/files/netbox.service" \ + --enablement-state enabled +__systemd_unit netbox-rq.service \ + --source "$__type/files/netbox-rq.service" \ + --enablement-state enabled --restart From 445bc75deba18b8fa978e7d5d8ed8189908457d6 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 10:59:13 +0200 Subject: [PATCH 111/332] [type/__dma_auth] Drop --server parameter Currently, dma does not differentiate between login users on the SMTP server. It will pick whatever entry it finds first (https://github.com/corecode/dma/blob/v0.13/net.c#L531). As a result, the --server parameter only adds confusion. --- type/__dma_auth/explorer/state | 29 +++++++++++++---------------- type/__dma_auth/gencode-remote | 24 +++++++++--------------- type/__dma_auth/man.rst | 6 ++++-- type/__dma_auth/parameter/optional | 1 - 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/type/__dma_auth/explorer/state b/type/__dma_auth/explorer/state index 668b50f..621e5a2 100755 --- a/type/__dma_auth/explorer/state +++ b/type/__dma_auth/explorer/state @@ -17,25 +17,18 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer looks for a line matching the login and server parameters +# This explorer looks for a line matching the server parameter # in dma's auth.conf and reports: # present: a line matching login + host + password exists # absent: no line matching login + host exists -# different_password: a line exists but with a different pasword -# multiple: multiple lines matching login + host exist -# (should never happen) +# different_login: a line exists but with a different login user +# different_password: a line exists but with a different password +# multiple: multiple lines matching host exist (should not happen) auth_conf=$("${__type_explorer}/auth_conf") test -r "${auth_conf}" || exit 0 -if test -f "${__object}/parameter/server" -then - server=$(cat "${__object}/parameter/server") -else - server=$__object_id -fi - -awk -F'\n' -v server="${server}" ' +awk -F'\n' ' function getvalue(path) { # Reads the first line of the file located at path and returns it. getline < path @@ -49,8 +42,7 @@ BEGIN { parameter_dir = ENVIRON["__object"] "/parameter/" # Read the parameters of this object - host_param = getvalue(parameter_dir "server") - if (!host_param) host_param = ENVIRON["__object_id"] + host_param = ENVIRON["__object_id"] login_param = getvalue(parameter_dir "login") passwd_param = getvalue(parameter_dir "password") @@ -78,10 +70,15 @@ BEGIN { } } -host == host_param && login == login_param { +host == host_param { # a match… if (state == "absent") { - state = ((passwd == passwd_param) ? "present" : "different_password") + if (login != login_param) + state = "different_login" + else if (passwd != passwd_param) + state = "different_password" + else + state = "present" } else { # report "multiple" to that the type can remove the duplicates. state = "multiple" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index c57e5cc..e73c424 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -21,14 +21,16 @@ state_is=$(cat "${__object}/explorer/state") state_should=$(cat "${__object}/parameter/state") -if test -f "${__object}/parameter/server" -then - server=$(cat "${__object}/parameter/server") -else - server=$__object_id -fi +server=$__object_id login=$(cat "${__object}/parameter/login") + +auth_conf=$(cat "${__object}/explorer/auth_conf") +test -n "${auth_conf}" || { + echo 'Cannot determine path of dma auth.conf' >&2 + exit 1 +} + if test "${state_is}" = "${state_should}" then # state is as it should @@ -61,13 +63,6 @@ in ;; esac -auth_conf=$(cat "${__object}/explorer/auth_conf") - -test -n "${auth_conf}" || { - echo 'Cannot determine path of dma auth.conf' >&2 - exit 1 -} - cat < Date: Mon, 28 Sep 2020 16:34:12 +0200 Subject: [PATCH 112/332] Move auth_conf explorer from __dma_auth to __dma --- type/__dma/explorer/auth_conf | 50 +++++++++++++++++++++++++++++- type/__dma_auth/explorer/auth_conf | 50 +----------------------------- 2 files changed, 50 insertions(+), 50 deletions(-) mode change 120000 => 100755 type/__dma/explorer/auth_conf mode change 100755 => 120000 type/__dma_auth/explorer/auth_conf diff --git a/type/__dma/explorer/auth_conf b/type/__dma/explorer/auth_conf deleted file mode 120000 index db038ae..0000000 --- a/type/__dma/explorer/auth_conf +++ /dev/null @@ -1 +0,0 @@ -../../__dma_auth/explorer/auth_conf \ No newline at end of file diff --git a/type/__dma/explorer/auth_conf b/type/__dma/explorer/auth_conf new file mode 100755 index 0000000..cef0aca --- /dev/null +++ b/type/__dma/explorer/auth_conf @@ -0,0 +1,49 @@ +#!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# This explorer determines the path of dma's auth.conf file + +# No dma.conf -> use default +test -f /etc/dma/dma.conf || { + echo /etc/dma/auth.conf + exit 0 +} +test -r /etc/dma/dma.conf || { + echo 'Cannot read /etc/dma/dma.conf' >&2 + exit 1 +} + +# Get AUTHPATH from dma.conf +awk -F'[ \t]' ' +{ + sub(/#.*$/, "", $0) # remove comments + if (!$0) next # ignore empty lines +} +$1 == "AUTHPATH" { + # Store authpath. In dma conf parsing last wins. + if ($2) authpath = substr($0, index($0, " ") + 1) +} +END { + if (authpath) { + print authpath + exit 0 + } else exit 1 +} +' /etc/dma/dma.conf \ +|| echo /etc/dma/auth.conf # default diff --git a/type/__dma_auth/explorer/auth_conf b/type/__dma_auth/explorer/auth_conf deleted file mode 100755 index cef0aca..0000000 --- a/type/__dma_auth/explorer/auth_conf +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -e -# -# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# This explorer determines the path of dma's auth.conf file - -# No dma.conf -> use default -test -f /etc/dma/dma.conf || { - echo /etc/dma/auth.conf - exit 0 -} -test -r /etc/dma/dma.conf || { - echo 'Cannot read /etc/dma/dma.conf' >&2 - exit 1 -} - -# Get AUTHPATH from dma.conf -awk -F'[ \t]' ' -{ - sub(/#.*$/, "", $0) # remove comments - if (!$0) next # ignore empty lines -} -$1 == "AUTHPATH" { - # Store authpath. In dma conf parsing last wins. - if ($2) authpath = substr($0, index($0, " ") + 1) -} -END { - if (authpath) { - print authpath - exit 0 - } else exit 1 -} -' /etc/dma/dma.conf \ -|| echo /etc/dma/auth.conf # default diff --git a/type/__dma_auth/explorer/auth_conf b/type/__dma_auth/explorer/auth_conf new file mode 120000 index 0000000..e89de93 --- /dev/null +++ b/type/__dma_auth/explorer/auth_conf @@ -0,0 +1 @@ +../../__dma/explorer/auth_conf \ No newline at end of file From 3feaea1d96a361bf16893f1a2442ce4338ac6c89 Mon Sep 17 00:00:00 2001 From: Marko Seric Date: Mon, 28 Sep 2020 16:43:31 +0200 Subject: [PATCH 113/332] [type/__dma_auth] Externalise AWK update script to separate file --- type/__dma_auth/files/update_dma_auth.awk | 93 +++++++++++++++++++++++ type/__dma_auth/gencode-remote | 90 ++-------------------- 2 files changed, 98 insertions(+), 85 deletions(-) create mode 100644 type/__dma_auth/files/update_dma_auth.awk diff --git a/type/__dma_auth/files/update_dma_auth.awk b/type/__dma_auth/files/update_dma_auth.awk new file mode 100644 index 0000000..c50198b --- /dev/null +++ b/type/__dma_auth/files/update_dma_auth.awk @@ -0,0 +1,93 @@ +#!/usr/bin/awk -f +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +function getvalue(path) { + # Reads the first line of the file located at path and returns it. + getline < path + close(path) + return $0 +} + +function print_should() { + printf "%s|%s:%s\n", login_param, host_param, passwd_param +} + +BEGIN { + FS = "\n" + DP = "[: \t]" # copied from dma/conf.c + + parameter_dir = ENVIRON["__object"] "/parameter/" + + mode = (getvalue(parameter_dir "state") != "absent") + + host_param = ENVIRON["__object_id"] + login_param = getvalue(parameter_dir "login") + passwd_param = getvalue(parameter_dir "password") +} + +# skip comments and empty lines +/^#/ || /^$/ { + print + next +} + +{ + # parse line (like dma/conf.c would) + + login = substr($0, 1, index($0, "|") - 1) + if (!login) { login = $0 } # if no "|" found + + host = substr($0, length(login) + 2) + + if (match(host, DP)) { + passwd = substr(host, RSTART + 1) + host = substr(host, 1, RSTART - 1) + } else { + passwd = "" + } +} + +host == host_param { + if (mode) { + # state_should == present + if (!written) { + # replace first line if host matches (but only if no line has + # been written already -> no duplicates) + print_should() + written = 1 + } + next + } else { + # state_should == absent + next + } +} + +# leave other lines alone +{ + print +} + +END { + if (mode && !written) { + # append line if no match to replace was found + print_should() + } +} diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index e73c424..d8be7e8 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,6 +18,8 @@ # along with cdist. If not, see . # +drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } + state_is=$(cat "${__object}/explorer/state") state_should=$(cat "${__object}/parameter/state") @@ -42,8 +44,6 @@ in (present) test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1; } - mode=1 - if test "${state_is}" = 'absent' then printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" @@ -52,8 +52,6 @@ in fi ;; (absent) - mode=0 - printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" ;; (*) @@ -65,87 +63,9 @@ esac cat < no duplicates) - print_should() - written = 1 - } - next - } else { - # state_should == absent - if (!login_param || login == login_param) { - # empty --login -> drop all lines for this host - next - } - } -} - -# leave other lines alone -{ - print -} - -END { - if (mode && !written) { - # append line if no match to replace was found - print_should() - } -} -' <"${auth_conf}" >"${auth_conf}.tmp" \ - && mv "${auth_conf}.tmp" "${auth_conf}" +awk '$(drop_awk_comments "${__type}/files/update_dma_auth.awk")' <'${auth_conf}' >'${auth_conf}.tmp' \ +&& cat '${auth_conf}.tmp' >'${auth_conf}' +rm -f '${auth_conf}.tmp' EOF From 6ae08085607f1a73832b80a20ad4671950e5c82e Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 16:54:21 +0200 Subject: [PATCH 114/332] [type/__dma] Externalise AWK update script to separate file --- type/__dma/files/update_dma_conf.awk | 170 +++++++++++++++++++++++++++ type/__dma/gencode-remote | 167 ++------------------------ 2 files changed, 179 insertions(+), 158 deletions(-) create mode 100644 type/__dma/files/update_dma_conf.awk diff --git a/type/__dma/files/update_dma_conf.awk b/type/__dma/files/update_dma_conf.awk new file mode 100644 index 0000000..67661fd --- /dev/null +++ b/type/__dma/files/update_dma_conf.awk @@ -0,0 +1,170 @@ +#!/usr/bin/awk -f +# +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +function comment_line(line) { + # returns the position in line at which the comment'\''s text starts + # (0 if the line is not a comment) + match(line, /^[ \t]*\#+[ \t]*/) + return RSTART ? (RLENGTH + 1) : 0 +} +function empty_line(line) { return line ~ /^[ \t]*$/ } +function is_word(s) { return s ~ /^[A-Z_]+$/ } # "looks like a plausible word" + +function first(line, sep_re) { + # returns the part of the line until sep is found + # (or the whole line if sep is not found) + if (!sep_re) sep_re = "[" SUBSEP "]" + match(line, sep_re) + return RSTART ? substr(line, 1, RSTART - 1) : line +} + +function rest(line, sep_re) { + # returns the part of the line after the first occurrence of sep is found. + # (or nothing if sep is not found) + if (!sep_re) sep_re = "[" SUBSEP "]" + if (match(line, sep_re)) + return substr(line, RSTART + RLENGTH) +} + +function conf_pop(word, value) { + # returns the next value for the config `word` and delete it from the list. + # if value is set, this function will only return value if it is the first + # option in the list, otherwise it returns 0. + + if (!(word in conf)) return 0 + if (!value) { + if (index(conf[word], SUBSEP)) # more than one element? + value = substr(conf[word], 1, index(conf[word], SUBSEP) - 1) + else + value = conf[word] + } + + if (index(conf[word], SUBSEP)) { + if (index(conf[word], value SUBSEP) != 1) return 0 + conf[word] = substr(conf[word], length(value) + 2) + } else { + if (conf[word] != value) return 0 + delete conf[word] + } + return value +} + +function print_conf(word, value) { + # print a config line with the given parameters + printf "%s", word + if (value) printf " %s", value + printf "\n" +} + +function print_confs(word, value) { + # print config lines for all values stored in conf[word]. + if (!(word in conf)) return + if (conf[word]) { + while (value = conf_pop(word)) + print_conf(word, value) + } else { + print_conf(word) + delete conf[word] + } +} + +BEGIN { + FS = "\n" + EQS = "[ \t]" # copied from dma/conf.c + + # read the "should" state into the `conf` array. + while (getline < "/dev/stdin") { + word = first($0, EQS) + if ((word in conf)) + conf[word] = conf[word] SUBSEP rest($0, EQS) + else + conf[word] = rest($0, EQS) + } +} + +# first pass, gather information about where which information is stored in the +# current config file. This information will be used in the second pass. +NR == FNR { + if (comment_line($0)) { + # comment line + word = first(substr($0, comment_line($0)), " ") + if (is_word(word)) last_occ["#" word] = FNR + } else { + word = first($0, EQS) + if (is_word(word)) last_occ[word] = FNR + } +} + +# before second pass prepare hashes containing location information to be used +# in the second pass. +NR > FNR && FNR == 1 { + # First we drop the locations of commented-out options if a non-commented + # option is available. If a non-commented option is available, we will + # append new config options there to have them all at one place. + for (k in last_occ) + if (k ~ /^\#/ && (substr(k, 2) in last_occ)) + delete last_occ[k] + + # Reverse the option => line mapping. The line_map allows for easier lookups + # in the second pass. + for (k in last_occ) line_map[last_occ[k]] = k +} + +# second pass, generate and output new config +NR > FNR { + if (comment_line($0) || empty_line($0)) { + # comment or empty line + print + + if ((FNR in line_map)) { + if (line_map[FNR] ~ /^\#/) { + # This line contains a commented config option. If the conf hash + # contains options to be set, we output them here because this + # option is not used in the current config. + k = substr(line_map[FNR], 2) + if ((k in conf)) print_confs(k) + } + + if (("INSECURE" in conf) && line_map[FNR] ~ /^\#?SECURE$/) { + # INSECURE goes where SECURE comment is. + print_confs("INSECURE") + } + } + } else { + word = first($0, EQS) + value = rest($0, EQS) + sub(/[ \t]*\#.*$/, "", value) # ignore comments in value + + if ((word in conf) && value == first(conf[word])) { + # keep config options we want + conf_pop(word) + print + } + + if ((FNR in line_map) && line_map[FNR] == word) { + # rest of config options should be here + print_confs(word) + } + } +} + +END { + # print rest of config options ( + for (word in conf) print_confs(word) +} diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index a6aca0d..8177de9 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -1,5 +1,7 @@ #!/bin/sh -e +drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } + CONF_PATH=/etc/dma # set in Makefile # Determine mailname @@ -100,7 +102,7 @@ config_updated=false if ! echo "$conf_should" | cmp -s "${__object}/explorer/conf" - then # config needs to be updated - echo "dma_conf='${CONF_PATH:?}/dma.conf'" + dma_conf="${CONF_PATH:?}/dma.conf" # The following AWK script will output the new config file to be stored on # disk. To do so it reads the current dma.conf file and the config options @@ -119,164 +121,13 @@ then # options are grouped by word (the first word in the line) and appended # at the end of the file. - cat <<'EOF' -awk -F '\n' ' -function comment_line(line) { - # returns the position in line at which the comment'\''s text starts - # (0 if the line is not a comment) - match(line, /^[ \t]*\#+[ \t]*/) - return RSTART ? (RLENGTH + 1) : 0 -} -function empty_line(line) { return line ~ /^[ \t]*$/ } -function is_word(s) { return s ~ /^[A-Z_]+$/ } # "looks like a plausible word" - -function first(line, sep_re) { - # returns the part of the line until sep is found - # (or the whole line if sep is not found) - if (!sep_re) sep_re = "[" SUBSEP "]" - match(line, sep_re) - return RSTART ? substr(line, 1, RSTART - 1) : line -} - -function rest(line, sep_re) { - # returns the part of the line after the first occurrence of sep is found. - # (or nothing if sep is not found) - if (!sep_re) sep_re = "[" SUBSEP "]" - if (match(line, sep_re)) - return substr(line, RSTART + RLENGTH) -} - -function conf_pop(word, value) { - # returns the next value for the config `word` and delete it from the list. - # if value is set, this function will only return value if it is the first - # option in the list, otherwise it returns 0. - - if (!(word in conf)) return 0 - if (!value) { - if (index(conf[word], SUBSEP)) # more than one element? - value = substr(conf[word], 1, index(conf[word], SUBSEP) - 1) - else - value = conf[word] - } - - if (index(conf[word], SUBSEP)) { - if (index(conf[word], value SUBSEP) != 1) return 0 - conf[word] = substr(conf[word], length(value) + 2) - } else { - if (conf[word] != value) return 0 - delete conf[word] - } - return value -} - -function print_conf(word, value) { - # print a config line with the given parameters - printf "%s", word - if (value) printf " %s", value - printf "\n" -} - -function print_confs(word, value) { - # print config lines for all values stored in conf[word]. - if (!(word in conf)) return - if (conf[word]) { - while (value = conf_pop(word)) - print_conf(word, value) - } else { - print_conf(word) - delete conf[word] - } -} - -BEGIN { - EQS = "[ \t]" # copied from dma/conf.c - - # read the "should" state into the `conf` array. - while (getline < "/dev/stdin") { - word = first($0, EQS) - if ((word in conf)) - conf[word] = conf[word] SUBSEP rest($0, EQS) - else - conf[word] = rest($0, EQS) - } -} - -# first pass, gather information about where which information is stored in the -# current config file. This information will be used in the second pass. -NR == FNR { - if (comment_line($0)) { - # comment line - word = first(substr($0, comment_line($0)), " ") - if (is_word(word)) last_occ["#" word] = FNR - } else { - word = first($0, EQS) - if (is_word(word)) last_occ[word] = FNR - } -} - -# before second pass prepare hashes containing location information to be used -# in the second pass. -NR > FNR && FNR == 1 { - # First we drop the locations of commented-out options if a non-commented - # option is available. If a non-commented option is available, we will - # append new config options there to have them all at one place. - for (k in last_occ) - if (k ~ /^\#/ && (substr(k, 2) in last_occ)) - delete last_occ[k] - - # Reverse the option => line mapping. The line_map allows for easier lookups - # in the second pass. - for (k in last_occ) line_map[last_occ[k]] = k -} - -# second pass, generate and output new config -NR > FNR { - if (comment_line($0) || empty_line($0)) { - # comment or empty line - print - - if ((FNR in line_map)) { - if (line_map[FNR] ~ /^\#/) { - # This line contains a commented config option. If the conf hash - # contains options to be set, we output them here because this - # option is not used in the current config. - k = substr(line_map[FNR], 2) - if ((k in conf)) print_confs(k) - } - - if (("INSECURE" in conf) && line_map[FNR] ~ /^\#?SECURE$/) { - # INSECURE goes where SECURE comment is. - print_confs("INSECURE") - } - } - } else { - word = first($0, EQS) - value = rest($0, EQS) - sub(/[ \t]*\#.*$/, "", value) # ignore comments in value - - if ((word in conf) && value == first(conf[word])) { - # keep config options we want - conf_pop(word) - print - } - - if ((FNR in line_map) && line_map[FNR] == word) { - # rest of config options should be here - print_confs(word) - } - } -} - -END { - # print rest of config options ( - for (word in conf) print_confs(word) -} -' "${dma_conf}" "${dma_conf}" <<'EOF' >"${dma_conf}.tmp" \ - && mv "${dma_conf}.tmp" "${dma_conf}" + cat <'${dma_conf}.tmp' \ +&& cat '${dma_conf}.tmp' >'${dma_conf}' +${conf_should} EOF - # Pass in "conf_should" via stdin - echo "${conf_should}" - echo 'EOF' +rm '${dma_conf}.tmp' +CODE config_updated=true echo 'config updated' >>"${__messages_out}" From b48b48e4047d8855e9ee2d635a0c345cad007997 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 17:29:41 +0200 Subject: [PATCH 115/332] [type/__mail_alias] Externalise AWK update script to separate file --- type/__mail_alias/files/update_aliases.awk | 98 ++++++++++++++++++ type/__mail_alias/gencode-remote | 115 ++++----------------- 2 files changed, 119 insertions(+), 94 deletions(-) create mode 100644 type/__mail_alias/files/update_aliases.awk diff --git a/type/__mail_alias/files/update_aliases.awk b/type/__mail_alias/files/update_aliases.awk new file mode 100644 index 0000000..87ea202 --- /dev/null +++ b/type/__mail_alias/files/update_aliases.awk @@ -0,0 +1,98 @@ +#!/usr/bin/awk -f +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +function getvalue(path) { + # Reads the first line of the file located at path and returns it. + getline < path + close(path) + return $0 +} + +function sepafter(f, def, _) { + # finds the separator between field $f and $(f+1) + _ = substr($0, length($f)+1, index(substr($0, length($f)+1), $(f+1))-1) + return _ ? _ : def +} + +function write_aliases() { + if (aliases_written) return + + # print aliases line + printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") + while ((getline < aliases_should_file) > 0) { + if (aliases_written) printf ", " + printf "%s", $0 + aliases_written = 1 + } + printf "\n" + close(aliases_should_file) +} + +BEGIN { + FS = ":[ \t]*" + + parameter_dir = ENVIRON["__object"] "/parameter/" + + mode = (getvalue(parameter_dir "state") != "absent") + aliases_should_file = (parameter_dir "/alias") +} + +/^[ \t]*\#/ { + # comment line (leave alone) + select = 0; cont = 0 # comments terminate alias lists and continuations + print + next +} + +{ + # is this line a continuation line? + # (the prev. line ended in a backslash or the line starts with whitespace) + is_cont = /^[ \t]/ || cont + + # detect if the line is a line to be continued (ends with a backslash) + cont = ($0 ~ /\\$/) + # if it is, we drop the backslash from the line. + if (cont) sub(/[ \t]*\\$/, "", $0) +} + +is_cont { + # we ignore the line as it has been rewritten previously or is not + # interesting + next +} + +$1 == ENVIRON["__object_id"] { + # "target" user -> rewrite aliases list + select = 1 + if (mode) write_aliases() + next +} + +{ + # other user + select = 0 + print +} + +END { + # if the last line was an alias, the separator will be reused (looks better) + if (mode && !aliases_written) + write_aliases() +} diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 9f4af1b..e5bc2b7 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -18,6 +18,16 @@ # along with cdist. If not, see . # +drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } + +aliases_file=$(cat "${__object}/explorer/aliases_file") + +test -n "${aliases_file}" || { + echo 'Could not determine aliases file path.' >&2 + exit 1 +} + + state_should=$(cat "${__object}/parameter/state") case $state_should @@ -35,119 +45,36 @@ in else echo "add aliases" >>"$__messages_out" fi - - mode=1 ;; (absent) # nothing to do if no aliases found. test -s "${__object}/explorer/aliases" || exit 0 echo "delete aliases" >>"$__messages_out" - - mode=0 ;; (*) - printf 'Invalid --state: %s.\n' "$state_should" >&2 + printf 'Invalid --state: %s.\n' "${state_should}" >&2 printf 'Acceptable values are: present, absent.\n' >&2 exit 1 esac -aliases_file=$(cat "${__object}/explorer/aliases_file") +cat <&2 - exit 1 -} - -# "export" variables to remote -printf 'mode=%u\n' "${mode}" -printf "aliases_file='%s'\n" "${aliases_file}" - -cat <<'EOF' -test -f "${aliases_file}" || touch "${aliases_file}" - -awk -F ':[ \t]*' -v mode=$mode ' -function sepafter(f, def, _) { - # finds the separator between field $f and $(f+1) - _ = substr($0, length($f)+1, index(substr($0, length($f)+1), $(f+1))-1) - return _ ? _ : def -} - -function write_aliases() { - if (aliases_written) return - - # print aliases line - printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") - while ((getline < aliases_should_file) > 0) { - if (aliases_written) printf ", " - printf "%s", $0 - aliases_written = 1 - } - printf "\n" - close(aliases_should_file) -} - -BEGIN { - aliases_should_file = (ENVIRON["__object"] "/parameter/alias") -} - -/^[ \t]*\#/ { - # comment line (leave alone) - select = 0; cont = 0 # comments terminate alias lists and continuations - print - next -} - -{ - # is this line a continuation line? - # (the prev. line ended in a backslash or the line starts with whitespace) - is_cont = /^[ \t]/ || cont - - # detect if the line is a line to be continued (ends with a backslash) - cont = ($0 ~ /\\$/) - # if it is, we drop the backslash from the line. - if (cont) sub(/[ \t]*\\$/, "", $0) -} - -is_cont { - # we ignore the line as it has been rewritten previously or is not - # interesting - next -} - -$1 == ENVIRON["__object_id"] { - # "target" user -> rewrite aliases list - select = 1 - if (mode) write_aliases() - next -} - -{ - # other user - select = 0 - print -} - -END { - # if the last line was an alias, the separator will be reused (looks better) - if (mode && !aliases_written) - write_aliases() -} -' <"${aliases_file}" >"${aliases_file}.tmp" || { - rm -f "${aliases_file}.tmp" +awk '$(drop_awk_comments "${__type}/files/update_aliases.awk")' <'${aliases_file}' >'${aliases_file}.tmp' \ +|| { + rm -f '${aliases_file}.tmp' echo 'Generating new aliases file failed!' >&2 exit 1 } -if ! cmp -s "${aliases_file}" "${aliases_file}.tmp" +if ! cmp -s '${aliases_file}' '${aliases_file}.tmp' then - # aliases file was modified, replace and run `newaliases`. - mv "${aliases_file}.tmp" "${aliases_file}" + # aliases file was modified, replace: + cat '${aliases_file}.tmp' >'${aliases_file}' - # run newaliases if present + # then, run newaliases if present ("missing" on Alpine Linux because of typo) command -v newaliases >/dev/null 2>&1 && newaliases || true -else - # no modifications were made, delete the temp file. - rm "${aliases_file}.tmp" fi +rm -f '${aliases_file}.tmp' EOF From 2270c32ddb7d2fa2c9bd904755e20fd0fbae13dc Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 17:30:53 +0200 Subject: [PATCH 116/332] [type/__dma] Add missing license headers --- type/__dma/gencode-remote | 18 ++++++++++++++++++ type/__dma/manifest | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 8177de9..bcd4530 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } diff --git a/type/__dma/manifest b/type/__dma/manifest index 75e42d7..df62308 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Dennis Camera (dennis.camera@ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# os=$(cat "${__global}/explorer/os") From b55186544f4cfeb9f371c4c0ff429bdc2a63c6cc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 28 Sep 2020 19:49:29 +0200 Subject: [PATCH 117/332] __netbox: adding seperate /opt/netbox/cdist/ directory Adding a place to keep all configuration stuff for cdist seperated from the installation directory. For safety reasons. --- type/__netbox/explorer/secretkey | 2 +- type/__netbox/explorer/version | 2 +- type/__netbox/gencode-remote | 26 +++++++------------------- type/__netbox/manifest | 17 ++++++++++------- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/type/__netbox/explorer/secretkey b/type/__netbox/explorer/secretkey index 9f0331c..7cce279 100755 --- a/type/__netbox/explorer/secretkey +++ b/type/__netbox/explorer/secretkey @@ -2,7 +2,7 @@ # Explorer will output the key if he exists. -secretkey="/opt/netbox/.secretkey" +secretkey="/opt/netbox/cdist/secretkey" if [ -f "$secretkey" ]; then cat "$secretkey" fi diff --git a/type/__netbox/explorer/version b/type/__netbox/explorer/version index 5231e16..ee3dde8 100755 --- a/type/__netbox/explorer/version +++ b/type/__netbox/explorer/version @@ -1,5 +1,5 @@ #!/bin/sh -e # output version if exist -version_path="/opt/netbox/netbox/cdist/version" +version_path="/opt/netbox/cdist/version" if [ -f "$version_path" ]; then cat "$version_path"; fi diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 0370949..aa3864e 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -24,16 +24,13 @@ cd "\$tmpdir" curl -sS -L '$url' > '$archive' tar xf '$archive' -# Save cdist-upload configuration file. -cp '$install_dir/cdist/configuration.py' "\$tmpdir/configuration.py" -cp '$install_dir/cdist/ldap_config.py' "\$tmpdir/ldap_config.py" - EOF # Stop everything in the pyenv to update cat << EOF # Try to kill everything in the venv -systemctl stop netbox gunicorn-netbox uwsgi-netbox || true +systemctl -q --wait stop netbox gunicorn-netbox uwsgi-netbox || true +# don't know if this is required since using --wait ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true EOF @@ -42,11 +39,10 @@ EOF # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' -mkdir '$install_dir/cdist' cp '$src/requirements.txt' /opt/netbox/ # backup dependency info -cp \$tmpdir/configuration.py '$install_dir/netbox/configuration.py' -cp \$tmpdir/ldap_config.py '$install_dir/netbox/ldap_config.py' +ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. # forcing python3 to be sure (till python4 gets released ..) @@ -79,24 +75,16 @@ cd / rm -r "\$tmpdir" # Save version after successful installation -printf "%s\\n" "$VERSION" > '$install_dir/cdist/version' +printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version EOF # meta printf "installed %s\n" "$VERSION" >> "$__messages_out" changes=yes +fi # check if configuration changed -elif grep -q "^__file/opt/netbox/" "$__messages_in"; then - # check if coping is required - if grep -q "^__file/opt/netbox/netbox/cdist/" "$__messages_in"; then - cat << EOF -# Copy configuration -cp '$install_dir/cdist/configuration.py' '$install_dir/netbox/configuration.py' -cp '$install_dir/cdist/ldap_config.py' '$install_dir/netbox/ldap_config.py' -EOF - fi - +if grep -q "^__file/opt/netbox/" "$__messages_in"; then # meta printf "configured\n" >> "$__messages_out" changes=yes diff --git a/type/__netbox/manifest b/type/__netbox/manifest index f8ed5fc..89b89ac 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -194,20 +194,23 @@ mkdir -p "$__object/files" "$__type/files/configuration.py.sh" > "$__object/files/configuration.py" "$__type/files/ldap_config.py.sh" > "$__object/files/ldap_config.py" -require="__user/netbox" __directory /opt/netbox/netbox/cdist --parents -require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/configuration.py --mode 640 --owner netbox \ +require="__user/netbox" __directory /opt/netbox/cdist +require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/configuration.py --mode 640 --owner netbox \ --source "$__object/files/configuration.py" if [ -f "$__object/parameter/ldap-server" ]; then - require="__directory/opt/netbox/netbox/cdist " __file \ - /opt/netbox/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ + require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/ldap_config.py --mode 640 --owner netbox \ --source "$__object/files/ldap_config.py" +else + require="__directory/opt/netbox/cdist" __file \ + /opt/netbox/cdist/ldap_config.py --state absent fi # save secret -require="__user/netbox" __file /opt/netbox/.secretkey --mode 400 \ - --owner netbox --source - << SECRET +require="__directory/opt/netbox/cdist" __file /opt/netbox/cdist/secretkey \ + --mode 400 --owner netbox --source - << SECRET $SECRET_KEY SECRET From 050812305b282e65be51c3902dba691c4718ca4d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 28 Sep 2020 20:04:22 +0200 Subject: [PATCH 118/332] __netbox: add mange.py calls recommended from upstream Calls where added because there are used upstream in the `upgrade.sh`-Script, too. Upgrade-Script: https://github.com/netbox-community/netbox/blob/develop/upgrade.sh --- type/__netbox/gencode-remote | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index aa3864e..d7bc219 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -64,11 +64,17 @@ EOF # Set final permissions. chown -R netbox /opt/netbox +# NetBox manage scripts # Run database migrations. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate - # Generate static assets. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py collectstatic --no-input +# Delete any stale content types +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py remove_stale_contenttypes --no-input +# Delete any expired user sessions +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py clearsessions +# Clear all cached data +sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py invalidate all # Remove temporary working directory. cd / From 231f96de18cad4b53232baa504b8bf266bd015de Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 17:37:41 +0200 Subject: [PATCH 119/332] Error if expected environment variables are unset --- type/__dma/gencode-remote | 58 ++++++++++++------------- type/__dma/manifest | 2 +- type/__dma_auth/gencode-remote | 18 ++++---- type/__mail_alias/explorer/aliases | 4 +- type/__mail_alias/explorer/aliases_file | 2 +- type/__mail_alias/gencode-remote | 18 ++++---- 6 files changed, 52 insertions(+), 50 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index bcd4530..e254323 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -23,18 +23,18 @@ drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } CONF_PATH=/etc/dma # set in Makefile # Determine mailname -if test -f "${__object}/parameter/mailname" +if test -f "${__object:?}/parameter/mailname" then - mailname=$(cat "${__object}/parameter/mailname") + mailname=$(cat "${__object:?}/parameter/mailname") else - case $(cat "${__global}/explorer/os") + case $(cat "${__global:?}/explorer/os") in (debian|devuan|ubuntu) # On Debian-like systems use /etc/mailname unless --mailname is used mailname='/etc/mailname' ;; (*) - mailname=$__target_fqdn + mailname=${__target_fqdn:?} ;; esac fi @@ -42,19 +42,19 @@ fi # Generate "should" values for config conf_should=$( - if test -s "${__object}/parameter/smarthost" + if test -s "${__object:?}/parameter/smarthost" then - printf 'SMARTHOST %s\n' "$(cat "${__object}/parameter/smarthost")" + printf 'SMARTHOST %s\n' "$(cat "${__object:?}/parameter/smarthost")" fi printf 'MAILNAME %s\n' "${mailname}" - if test -s "${__object}/explorer/auth_conf" + if test -s "${__object:?}/explorer/auth_conf" then - printf "AUTHPATH %s\n" "$(cat "${__object}/explorer/auth_conf")" + printf "AUTHPATH %s\n" "$(cat "${__object:?}/explorer/auth_conf")" fi - case $(cat "${__object}/parameter/security") + case $(cat "${__object:?}/parameter/security") in (ssl|tls) default_smtp_port=465 @@ -77,35 +77,35 @@ conf_should=$( ;; esac - if test -s "${__object}/parameter/port" + if test -s "${__object:?}/parameter/port" then - printf 'PORT %u\n' "$(cat "${__object}/parameter/port")" + printf 'PORT %u\n' "$(cat "${__object:?}/parameter/port")" elif test "${default_smtp_port}" -ne 25 # DMA uses port 25 by default then printf 'PORT %u\n' "${default_smtp_port}" fi - if test -f "${__object}/parameter/masquerade" + if test -f "${__object:?}/parameter/masquerade" then while read -r line do printf 'MASQUERADE %s\n' "${line}" - done <"${__object}/parameter/masquerade" + done <"${__object:?}/parameter/masquerade" fi - if test -f "${__object}/parameter/defer" + if test -f "${__object:?}/parameter/defer" then echo 'DEFER' fi - if test -f "${__object}/parameter/fullbounce" + if test -f "${__object:?}/parameter/fullbounce" then echo 'FULLBOUNCE' fi - if test -f "${__object}/parameter/nullclient" + if test -f "${__object:?}/parameter/nullclient" then - test -s "${__object}/parameter/smarthost" || { + test -s "${__object:?}/parameter/smarthost" || { echo '--nullclient requires a --smarthost to be defined' >&2 exit 1 } @@ -114,10 +114,10 @@ conf_should=$( fi ) # Sort conf_should to compare against "conf_is" -conf_should=$(echo "$conf_should" | sort -s -k 1,1) +conf_should=$(echo "${conf_should}" | sort -s -k 1,1) config_updated=false -if ! echo "$conf_should" | cmp -s "${__object}/explorer/conf" - +if ! echo "${conf_should}" | cmp -s "${__object:?}/explorer/conf" - then # config needs to be updated dma_conf="${CONF_PATH:?}/dma.conf" @@ -140,7 +140,7 @@ then # at the end of the file. cat <'${dma_conf}.tmp' \ +awk '$(drop_awk_comments "${__type:?}/files/update_dma_conf.awk")' '${dma_conf}' '${dma_conf}' <<'EOF' >'${dma_conf}.tmp' \ && cat '${dma_conf}.tmp' >'${dma_conf}' ${conf_should} EOF @@ -148,28 +148,28 @@ rm '${dma_conf}.tmp' CODE config_updated=true - echo 'config updated' >>"${__messages_out}" + echo 'config updated' >>"${__messages_out:?}" fi -if test -f "${__object}/parameter/send-test-email" +if test -f "${__object:?}/parameter/send-test-email" then - if grep -q '^__mail_alias/root:' "${__messages_in}" \ - || grep -q '^__dma_auth/' "${__messages_in}" \ + if grep -q '^__mail_alias/root:' "${__messages_in:?}" \ + || grep -q '^__dma_auth/' "${__messages_in:?}" \ || $config_updated then - cat <<-EOF - sendmail root <. # -os=$(cat "${__global}/explorer/os") +os=$(cat "${__global:?}/explorer/os") # Install DMA case $os diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index d8be7e8..aee4d7f 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -20,14 +20,14 @@ drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } -state_is=$(cat "${__object}/explorer/state") -state_should=$(cat "${__object}/parameter/state") +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") -server=$__object_id -login=$(cat "${__object}/parameter/login") +server=${__object_id:?} +login=$(cat "${__object:?}/parameter/login") -auth_conf=$(cat "${__object}/explorer/auth_conf") +auth_conf=$(cat "${__object:?}/explorer/auth_conf") test -n "${auth_conf}" || { echo 'Cannot determine path of dma auth.conf' >&2 exit 1 @@ -46,13 +46,13 @@ in if test "${state_is}" = 'absent' then - printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + printf 'add authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out:?}" else - printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + printf 'set authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out:?}" fi ;; (absent) - printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out}" + printf 'delete authuser %s on %s\n' "${login}" "${server}" >>"${__messages_out:?}" ;; (*) printf 'Invalid --state: %s.\n' "${state_should}" >&2 @@ -65,7 +65,7 @@ esac cat <'${auth_conf}.tmp' \ +awk '$(drop_awk_comments "${__type:?}/files/update_dma_auth.awk")' <'${auth_conf}' >'${auth_conf}.tmp' \ && cat '${auth_conf}.tmp' >'${auth_conf}' rm -f '${auth_conf}.tmp' EOF diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index 4fffd3b..5b5d68d 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -20,9 +20,11 @@ # Find aliases for a given user name and print the aliases (each one on a # separate line) -aliases_file=$("${__type_explorer}/aliases_file") +aliases_file=$("${__type_explorer:?}/aliases_file") test -r "${aliases_file}" || exit 0 +: "${__object_id:?}" # assert __object_id is set, because it is used in AWK + awk -F ':[ \t]*' ' function print_aliases(aliases, matches) { # prints comma-separated aliases (one per line) diff --git a/type/__mail_alias/explorer/aliases_file b/type/__mail_alias/explorer/aliases_file index a59bb99..7f09f88 100755 --- a/type/__mail_alias/explorer/aliases_file +++ b/type/__mail_alias/explorer/aliases_file @@ -28,7 +28,7 @@ check_file() { fi } -case $("${__explorer}/os") +case $("${__explorer:?}/os") in (freebsd|openbsd|solaris) check_file /etc/mail/aliases diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index e5bc2b7..7ef2f7c 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -20,7 +20,7 @@ drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } -aliases_file=$(cat "${__object}/explorer/aliases_file") +aliases_file=$(cat "${__object:?}/explorer/aliases_file") test -n "${aliases_file}" || { echo 'Could not determine aliases file path.' >&2 @@ -28,29 +28,29 @@ test -n "${aliases_file}" || { } -state_should=$(cat "${__object}/parameter/state") +state_should=$(cat "${__object:?}/parameter/state") case $state_should in (present) - if cmp -s "${__object}/explorer/aliases" "${__object}/parameter/alias" + if cmp -s "${__object:?}/explorer/aliases" "${__object:?}/parameter/alias" then # all good! exit 0 fi - if test -s "${__object}/explorer/aliases" + if test -s "${__object:?}/explorer/aliases" then - echo "update aliases" >>"$__messages_out" + echo "update aliases" >>"${__messages_out:?}" else - echo "add aliases" >>"$__messages_out" + echo "add aliases" >>"${__messages_out:?}" fi ;; (absent) # nothing to do if no aliases found. - test -s "${__object}/explorer/aliases" || exit 0 + test -s "${__object:?}/explorer/aliases" || exit 0 - echo "delete aliases" >>"$__messages_out" + echo "delete aliases" >>"${__messages_out:?}" ;; (*) printf 'Invalid --state: %s.\n' "${state_should}" >&2 @@ -61,7 +61,7 @@ esac cat <'${aliases_file}.tmp' \ +awk '$(drop_awk_comments "${__type:?}/files/update_aliases.awk")' <'${aliases_file}' >'${aliases_file}.tmp' \ || { rm -f '${aliases_file}.tmp' echo 'Generating new aliases file failed!' >&2 From 161e1e85f4b7696036bc1684426e33962d713fa7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 28 Sep 2020 17:54:35 +0200 Subject: [PATCH 120/332] [scripts/run-shellcheck.sh] Do not shellcheck AWK and Python scripts --- scripts/run-shellcheck.sh | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/scripts/run-shellcheck.sh b/scripts/run-shellcheck.sh index 769f853..fcca722 100755 --- a/scripts/run-shellcheck.sh +++ b/scripts/run-shellcheck.sh @@ -1,21 +1,29 @@ -#!/bin/sh +#!/bin/sh -eu -SHELLCHECKCMD="shellcheck -s sh -f gcc -x" +SHELLCHECKCMD='shellcheck -s sh -f gcc -x' # Skip SC2154 for variables starting with __ since such variables are cdist # environment variables. SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]' -SHELLCHECKTMP=".shellcheck.tmp" +SHELLCHECKTMP='.shellcheck.tmp' # Move to top-level cdist-contrib directory. -cd $(dirname $0)/.. +cd "$(dirname $0)"/.. -check () { - find type/ -type f $1 $2 -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}" - test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } +check() { + find type/ -type f "$@" -exec ${SHELLCHECKCMD} {} + \ + | grep -v "${SHELLCHECK_SKIP}" >>"${SHELLCHECKTMP}" || true } -check -path "*/explorer/*" -check -path "*/files/*" +rm -f "${SHELLCHECKTMP}" + +check -path '*/explorer/*' +check -path '*/files/*' ! -name '*.awk' ! -name '*.py' check -name manifest check -name gencode-local check -name gencode-remote + +if test -s "${SHELLCHECKTMP}" +then + cat "${SHELLCHECKTMP}" >&2 + exit 1 +fi From c6b795b3f9ac5472b00818fd53b6da02feabb52a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 1 Oct 2020 12:55:38 +0200 Subject: [PATCH 121/332] [type/__mail_alias] Update man.rst and make --alias required --- type/__mail_alias/gencode-remote | 6 ++++++ type/__mail_alias/man.rst | 15 ++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 7ef2f7c..3eea452 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -39,6 +39,12 @@ in exit 0 fi + test -s "${__object:?}/parameter/alias" || { + printf 'The --alias parameter is required if --state present.\n' >&2 + printf 'Use --state absent to remove all aliases.\n' >&2 + exit 1 + } + if test -s "${__object:?}/explorer/aliases" then echo "update aliases" >>"${__messages_out:?}" diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst index 3782ffb..d6d9742 100644 --- a/type/__mail_alias/man.rst +++ b/type/__mail_alias/man.rst @@ -8,7 +8,7 @@ cdist-type__mail_alias - Manage mail aliases. DESCRIPTION ----------- -This cdist type allows you to configure mail aliases (/etc/mail/aliases). +This cdist type allows you to configure mail aliases (/etc/aliases). REQUIRED PARAMETERS @@ -21,11 +21,12 @@ OPTIONAL PARAMETERS state 'present' or 'absent', defaults to 'present' alias - the aliases where mail for the given user should be redirected to. - This parameter can be specified multiple times to redirect to more than one - recipient. - See the `aliases(5)` man page for the different forms this parameter can - take.. + an alias, i.e. a mail address where mail for the user should be redirected + to. + This parameter can be specified multiple times to redirect to multiple + recipients. + If ``--state`` is ``present`` this parameter is required. + See `aliases(5)` for the different forms this parameter can take. BOOLEAN PARAMETERS @@ -51,7 +52,7 @@ BUGS containing ``,`` (commas) are treated incorrectly (they are treated as two addresses/aliases.) Make sure that email addresses do not contain commas. - +- ``:include:`` directives in the aliases file are ignored by this type. SEE ALSO -------- From f202d11124347783dbd679ae521f1bcb09e73cbb Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 1 Oct 2020 13:45:20 +0200 Subject: [PATCH 122/332] [type/__mail_alias] Fix continuation line processing --- type/__mail_alias/explorer/aliases | 12 ++++-------- type/__mail_alias/files/update_aliases.awk | 8 +++----- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/type/__mail_alias/explorer/aliases b/type/__mail_alias/explorer/aliases index 5b5d68d..ac13d7c 100755 --- a/type/__mail_alias/explorer/aliases +++ b/type/__mail_alias/explorer/aliases @@ -31,7 +31,7 @@ function print_aliases(aliases, matches) { split(aliases, matches, /,[ \t]*/) for (i in matches) { gsub(/^[ \t]*|[ \t]*$/, "", matches[i]) - print matches[i] + if (matches[i]) print matches[i] } } @@ -47,14 +47,10 @@ function print_aliases(aliases, matches) { is_cont = /^[ \t]/ || cont # detect if the line is a line to be continued (ends with a backslash) - cont = ($0 ~ /\\$/) + cont = /\\$/ - # if it is, we drop the backslash from the line and skip to next line - # (the contents have been printed above if they should) - if (cont) { - sub(/[ \t]*\\$/, "", $0) - next - } + # if it is, we drop the backslash from the line + if (cont) sub(/[ \t]*\\$/, "", $0) } is_cont { diff --git a/type/__mail_alias/files/update_aliases.awk b/type/__mail_alias/files/update_aliases.awk index 87ea202..336009f 100644 --- a/type/__mail_alias/files/update_aliases.awk +++ b/type/__mail_alias/files/update_aliases.awk @@ -67,14 +67,12 @@ BEGIN { is_cont = /^[ \t]/ || cont # detect if the line is a line to be continued (ends with a backslash) - cont = ($0 ~ /\\$/) - # if it is, we drop the backslash from the line. - if (cont) sub(/[ \t]*\\$/, "", $0) + cont = /\\$/ } is_cont { - # we ignore the line as it has been rewritten previously or is not - # interesting + # we only print the line if it has not been rewritten (select) + if (!select) print next } From 1c9ab6e07b1808ce708cc9926bce4983a7faf0d7 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 1 Oct 2020 17:18:01 +0200 Subject: [PATCH 123/332] [type/__dma] Update man.rst --- type/__dma/gencode-remote | 5 ++-- type/__dma/man.rst | 56 ++++++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index e254323..1987106 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -66,7 +66,7 @@ conf_should=$( echo 'STARTTLS' ;; (opportunistic) - default_smtp_port=25 # XXX: correct? + default_smtp_port=25 echo 'SECURETRANSFER' echo 'STARTTLS' echo 'OPPORTUNISTIC_TLS' @@ -152,7 +152,8 @@ CODE fi -if test -f "${__object:?}/parameter/send-test-email" +# Send a test email if enabled and necessary (=configuration changed) +if test -f "${__object:?}/parameter/send-test-mail" then if grep -q '^__mail_alias/root:' "${__messages_in:?}" \ || grep -q '^__dma_auth/' "${__messages_in:?}" \ diff --git a/type/__dma/man.rst b/type/__dma/man.rst index ba4a5a6..29a71fa 100644 --- a/type/__dma/man.rst +++ b/type/__dma/man.rst @@ -8,47 +8,46 @@ cdist-type__dma - Setup the DragonFly Mail Agent as the MTA. DESCRIPTION ----------- -This (singleton) type uses dma, a small Mail Transport Agent (MTA), to accept -mails from locally installed Mail User Agents (MUA) and deliver the mails -to a remote destination. - -Remote delivery happens over TLS to one or more mailboxes that are local to the -mail server configured in the ``smarthost`` parameter. +This (singleton) type uses DMA, a small Mail Transport Agent (MTA), to accept +mails from locally installed Mail User Agents (MUA) and either deliver the mails +to a remote smart host for delivery or communicate with remote SMTP servers +directly. REQUIRED PARAMETERS ------------------- -smarthost - The mail server used to send email. - It must be configured to act as a relay for the host being configured by - this type so that mail can be sent to users non-local to the smarthost. +None. BOOLEAN PARAMETERS ------------------ defer - If enabled, the mail queue has to be manually flushed with the `-q` option. -full-bounce - Enable if the bounce message should include the complete original message, + If enabled, mail will not be sent immediately, but stored in a queue. + To flush the queue and send the mails, ```dma -q`` has to be run + periodically (e.g. using a cron job.) + This type does not manage such a cron job, but some operating systems ship + such a cron job with the package. +fullbounce + Enable if bounce messages should include the complete original message, not just the headers. -null-client +nullclient Enable to bypass aliases and local delivery, and instead forward all mails to the defined ``--smarthost``. -send-test-email - If present, after setup this type will send an email to root, to allow you - to easily test your setup. +send-test-mail + If set, this type will send a test email to root after setup, to check if + the configured settings work. OPTIONAL PARAMETERS ------------------- mailname If present, this will be the hostname used to identify this host and the - remote part of the from addresses. - If not defined, it defaults to `/etc/mailname` on Debian derivatives and to - `__target_fqdn` otherwise. + remote part of the sender addresses. + If not defined, it defaults to ``/etc/mailname`` on Debian derivatives and + to ``__target_fqdn`` otherwise. See `dma(8)` for more information. - Note: on Debian derivatives the `/etc/mailname` file should be updated + Note: on Debian derivatives the ``/etc/mailname`` file should be updated instead of using this parameter. masquerade Masquerade the envelope-from addresses with this address/hostname. @@ -59,7 +58,7 @@ masquerade port The port on which to deliver email. If not provided, a sensible default port will be used based on the - `--security` argument. + ``--security`` argument. security Configures whether and how DMA should use secure connections. @@ -74,14 +73,23 @@ security messages directly to the outside mail exchangers. insecure allow plain text SMTP login over an insecure connection. - Should really not be used anymore! + Should really *not* be used anymore! +smarthost + The mail server used to send email. + It must be configured to act as a relay for the host being configured by + this type so that mail can be sent to users non-local to the smarthost. + EXAMPLES -------- .. code-block:: sh - __dma --smarthost mx1.domain.tld --send-test-email + # Install DMA and use the smarthost mx1.domain.tld to send mail. + __dma --smarthost mx1.domain.tld --send-test-mail + + # Install DMA in a default configuration. + __dma SEE ALSO From f76bcd35742d4dc0db2af247b2be51fc6ae6132c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 1 Oct 2020 17:21:19 +0200 Subject: [PATCH 124/332] [type/__dma_auth] Update man.rst --- type/__dma_auth/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__dma_auth/man.rst b/type/__dma_auth/man.rst index f56bfb7..da76883 100644 --- a/type/__dma_auth/man.rst +++ b/type/__dma_auth/man.rst @@ -27,7 +27,7 @@ password OPTIONAL PARAMETERS ------------------- state - Either `present` or `absent`. Defaults to `present`. + Either ``present`` or ``absent``. Defaults to ``present``. BOOLEAN PARAMETERS ------------------ From 14f7cf88396f8a2870b4edfdb12f9b8448ac48b8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 4 Oct 2020 17:40:01 +0200 Subject: [PATCH 125/332] __netbox: set --data-root default value The default value is set to preserve all data saved by netbox. As explained in the manpage, it sets the directory for all data directories to `~netbox/data/` (would resolve to `/opt/netbox/data/`), so upgrades will not remove this data. --- type/__netbox/man.rst | 17 ++++++++++------- type/__netbox/manifest | 11 +++++------ type/__netbox/parameter/default/data-root | 1 + 3 files changed, 16 insertions(+), 13 deletions(-) create mode 100644 type/__netbox/parameter/default/data-root diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 3330b1b..6d50268 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -131,7 +131,7 @@ data-root This parameter set's the media, reports and scripts root to subdirectories of the given directory. Values can be overwritten by special parameters like `--media-root` for example. Use this option if you want to store persistant - data of netbox on an other partition. A trailing shlash is not needed. + data of netbox on an other partition. A trailing slash is not needed. The data directories have following predefined sub-directory names: @@ -142,23 +142,26 @@ data-root scripts root: ``$data_root/scripts`` + To preserve all data from installation upgrades - which just replace the + installation directory - the data will be kept in the netbox home directory + rather than the installation directory by default (``/opt/netbox/data/``). + This way, no data will be deleted after the installation directory + replacement because it remains outside of the installation directory. + media-root The file path to where media files (like image attachments) are stored. Change this path if you require to store data on an other partiotion. - A trailing slash is not needed. By default, it will be stored into the - installation directory (``/opt/netbox/netbox/netbox/media``). + A trailing slash is not needed. Defaults to ``$data_root/media``. reports-root The file path of where custom reports are kept. Change this path if you require to store data on an other partition. A trailing slash is not - needed. By default, it will be stored into the installation directory - (``/opt/netbox/netbox/netbox/reports``). + needed. Defaults to ``$data_root/reports``. scripts-root The file path of where custom scripts are kept. Change this path if you require to store data on an other partition. A trailing slash is not - needed. By default, it will be stored into the installation directory - (``/opt/netbox/netbox/netbox/scripts``). + needed. Defaults to ``$data_root/scripts``. BOOLEAN PARAMETERS diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 89b89ac..fbcf253 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -161,12 +161,11 @@ else fi export LOGIN_REQUIRED -if [ -f "$__object/parameter/data-root" ]; then - data_root="$(cat "$__object/parameter/data-root")" - MEDIA_ROOT="$data_root/media" - REPORTS_ROOT="$data_root/reports" - SCRIPTS_ROOT="$data_root/scripts" -fi +data_root="$(cat "$__object/parameter/data-root")" +MEDIA_ROOT="$data_root/media" +REPORTS_ROOT="$data_root/reports" +SCRIPTS_ROOT="$data_root/scripts" + if [ -f "$__object/parameter/media-root" ]; then MEDIA_ROOT="$(cat "$__object/parameter/media-root")" fi diff --git a/type/__netbox/parameter/default/data-root b/type/__netbox/parameter/default/data-root new file mode 100644 index 0000000..45faab0 --- /dev/null +++ b/type/__netbox/parameter/default/data-root @@ -0,0 +1 @@ +/opt/netbox/data From 0c85b2d3fd3e9dee6a79091064e4fb7078adcdfb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 5 Oct 2020 18:26:47 +0200 Subject: [PATCH 126/332] __netbox: uninstall not anymore required packages This code tries to remove packages not needed anymore. As it presumably not removing dependencies, it's not that perfect pice of script. --- type/__netbox/gencode-remote | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index d7bc219..1cd4939 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -36,18 +36,35 @@ ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || tr EOF cat << EOF +# backup requirement files +if [ -f /opt/netbox/requirements.txt]; then + cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt +else + # preseve file-not-found errors and warnings + touch /opt/netbox/old-requirements.txt +fi +cp '$src/requirements.txt' /opt/netbox/ + # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' -cp '$src/requirements.txt' /opt/netbox/ # backup dependency info ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' + # Setup & enter python virtualenv. # forcing python3 to be sure (till python4 gets released ..) virtualenv -p python3 /opt/netbox/venv +# Uninstall packages not required anymore +# if versions not shortend, they will be ignored by pip, but not by comm +# all of this could be done with grep, too, but it's still must be shortend with awk +awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" +awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" +comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ + | xargs /opt/netbox/venv/bin/pip3 uninstall -y + # Install python dependencies. # avoid gunicorn, because it will be done in an other type grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ @@ -57,7 +74,7 @@ EOF if [ -f "$__object/parameter/ldap-server" ]; then echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" else - echo "/opt/netbox/venv/bin/pip3 uninstall django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 uninstall -y django-auth-ldap" fi cat << EOF From 8f1b56026ca75b6a9d66dd40d40c239c34f6927b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 5 Oct 2020 19:02:05 +0200 Subject: [PATCH 127/332] __netbox: force link creation for config files This adds the force flag to `ln` to avoid aborts cause the link already exists and so forth. It also adds robustness to the execution. --- type/__netbox/gencode-remote | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 1cd4939..3b4e05c 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -48,9 +48,9 @@ cp '$src/requirements.txt' /opt/netbox/ # Deploy sources and restore configuration. rm -r '$install_dir' cp -r '$src/netbox' '$install_dir' - -ln -s /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' -ln -s /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' +# force links to the cdist directory +ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' # Setup & enter python virtualenv. From 0b3bc1453073de6e45101e9386b5de1e8f93227a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 6 Oct 2020 17:29:31 +0200 Subject: [PATCH 128/332] __netbox: adjusted command flags Mostly added -q to the pip calls. Also forced the temp directory removal. --- type/__netbox/gencode-remote | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 3b4e05c..eb3bcd4 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -63,18 +63,18 @@ virtualenv -p python3 /opt/netbox/venv awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ - | xargs /opt/netbox/venv/bin/pip3 uninstall -y + | xargs /opt/netbox/venv/bin/pip3 uninstall -qy # Install python dependencies. # avoid gunicorn, because it will be done in an other type grep -v "^gunicorn==" "\$tmpdir/$src/requirements.txt" \ - | xargs /opt/netbox/venv/bin/pip3 install + | xargs /opt/netbox/venv/bin/pip3 install -q EOF if [ -f "$__object/parameter/ldap-server" ]; then - echo "/opt/netbox/venv/bin/pip3 install django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 install -q django-auth-ldap" else - echo "/opt/netbox/venv/bin/pip3 uninstall -y django-auth-ldap" + echo "/opt/netbox/venv/bin/pip3 uninstall -qy django-auth-ldap" fi cat << EOF @@ -95,7 +95,7 @@ sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py invalid # Remove temporary working directory. cd / -rm -r "\$tmpdir" +rm -rf "\$tmpdir" # Save version after successful installation printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version From 3b780c4794b1c9df9e3f6582e7ad17ad1cfd0402 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 8 Oct 2020 19:54:48 +0200 Subject: [PATCH 129/332] __netbox: fixup small bugs and venv process killing Changed flag (force to ignore a non-existant directory), typo and swapped arguments are done. Also, the process to stop all processes from the virtal environment has changed: Now, it stops all potential services and ignore errors (because a service doesn't exist). After that, it sends a kill signal to all processes and then gracefully wait since there is no option to do that with systemd. --- type/__netbox/gencode-remote | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index eb3bcd4..1ef78cd 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -28,16 +28,24 @@ EOF # Stop everything in the pyenv to update cat << EOF -# Try to kill everything in the venv -systemctl -q --wait stop netbox gunicorn-netbox uwsgi-netbox || true -# don't know if this is required since using --wait -ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | xargs kill || true +# Try to stop everything in the venv (ignore non-existant services) +systemctl -q stop netbox gunicorn-netbox uwsgi-netbox || true + +# kill and poll till all venv processes end +# at least if they are called with full path in the cmd (like in the services) +pids="\$( ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' )" +if [ "\$pids" ]; then + kill \$pids + while ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | grep -q . ; do + sleep 0.5 + done +fi EOF cat << EOF # backup requirement files -if [ -f /opt/netbox/requirements.txt]; then +if [ -f /opt/netbox/requirements.txt ]; then cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt else # preseve file-not-found errors and warnings @@ -46,7 +54,7 @@ fi cp '$src/requirements.txt' /opt/netbox/ # Deploy sources and restore configuration. -rm -r '$install_dir' +rm -rf '$install_dir' cp -r '$src/netbox' '$install_dir' # force links to the cdist directory ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' @@ -62,7 +70,7 @@ virtualenv -p python3 /opt/netbox/venv # all of this could be done with grep, too, but it's still must be shortend with awk awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" -comm -23 "\$tmpdir/curr-reqs.txt" "\$tmpdir/old-reqs.txt" \ +comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" \ | xargs /opt/netbox/venv/bin/pip3 uninstall -qy # Install python dependencies. From 13e97d171bb700f577cdcfb03dde81a193b0c10c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:39:19 +0200 Subject: [PATCH 130/332] __netbox*: added systemd socket support The Gunicorn type now supports systemd sockets only. With uWSGI, you can choose between it and the native sockets based on the parameters chosen. This is done because it could not be implemented to have multiple protocols with the systemd sockets (so you may choose). The systemd socket unit file is generally available, so both types use the same script to generate the socket unit file. --- type/__netbox/files/netbox.socket.sh | 33 ++++++++++ type/__netbox_gunicorn/files/gunicorn.py.sh | 2 +- type/__netbox_gunicorn/files/netbox.service | 3 +- type/__netbox_gunicorn/files/netbox.socket.sh | 1 + type/__netbox_gunicorn/man.rst | 8 ++- type/__netbox_gunicorn/manifest | 22 +++---- type/__netbox_uwsgi/explorer/bind-capability | 12 ++++ .../{netbox.service => netbox.service.sh} | 18 +++++- type/__netbox_uwsgi/files/netbox.socket.sh | 1 + type/__netbox_uwsgi/files/uwsgi.ini.sh | 24 ++++---- type/__netbox_uwsgi/gencode-remote | 60 ++++++++++++++++++- type/__netbox_uwsgi/man.rst | 33 +++++++++- type/__netbox_uwsgi/manifest | 38 +++++++++++- .../__netbox_uwsgi/parameter/default/protocol | 1 + type/__netbox_uwsgi/parameter/optional | 1 + 15 files changed, 218 insertions(+), 39 deletions(-) create mode 100755 type/__netbox/files/netbox.socket.sh create mode 120000 type/__netbox_gunicorn/files/netbox.socket.sh create mode 100755 type/__netbox_uwsgi/explorer/bind-capability rename type/__netbox_uwsgi/files/{netbox.service => netbox.service.sh} (70%) mode change 100644 => 100755 create mode 120000 type/__netbox_uwsgi/files/netbox.socket.sh create mode 100644 type/__netbox_uwsgi/parameter/default/protocol diff --git a/type/__netbox/files/netbox.socket.sh b/type/__netbox/files/netbox.socket.sh new file mode 100755 index 0000000..03e3d44 --- /dev/null +++ b/type/__netbox/files/netbox.socket.sh @@ -0,0 +1,33 @@ +#!/bin/sh -e +# __netbox/files/netbox.socket.sh + +# This is shared between all WSGI-server types. + +# Arguments: +# 1: File which list all sockets to listen on (sepearated by \n) + +if [ $# -ne 1 ]; then + printf "netbox.socket.sh: argument \$1 missing or too much given!\n" >&2 + exit 1 +fi + + +cat << UNIT +[Unit] +Description=Socket for NetBox via $TYPE + +[Socket] +UNIT + +# read all sockets to listen to +while read line; do + printf "ListenStream=%s\n" "$line" +done < "$1" + +cat << UNIT +SocketUser=netbox +SocketGroup=www-data + +[Install] +WantedBy=sockets.target +UNIT diff --git a/type/__netbox_gunicorn/files/gunicorn.py.sh b/type/__netbox_gunicorn/files/gunicorn.py.sh index 862ba06..c1e6ee5 100755 --- a/type/__netbox_gunicorn/files/gunicorn.py.sh +++ b/type/__netbox_gunicorn/files/gunicorn.py.sh @@ -13,7 +13,7 @@ cores="$(cat "$__explorer/cpu_cores")" cat << EOF # The IP address (typically localhost) and port that the Netbox WSGI process should listen on -bind = [$HOST ] +#bind = done via systemd socket 'gunicorn-netbox.socket' # Number of gunicorn workers to spawn. This should typically be 2n+1, where # n is the number of CPU cores present. diff --git a/type/__netbox_gunicorn/files/netbox.service b/type/__netbox_gunicorn/files/netbox.service index dcf1a66..28b6b45 100644 --- a/type/__netbox_gunicorn/files/netbox.service +++ b/type/__netbox_gunicorn/files/netbox.service @@ -3,13 +3,14 @@ Description=NetBox Gunicorn WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service Requires=netbox-rq.service +Requires=gunicorn-netbox.socket Wants=network.target After=netbox.service After=network.target After=redis-server.service postgresql.service [Service] -Type=simple +Type=notify User=netbox Group=netbox diff --git a/type/__netbox_gunicorn/files/netbox.socket.sh b/type/__netbox_gunicorn/files/netbox.socket.sh new file mode 120000 index 0000000..28ce920 --- /dev/null +++ b/type/__netbox_gunicorn/files/netbox.socket.sh @@ -0,0 +1 @@ +../../__netbox/files/netbox.socket.sh \ No newline at end of file diff --git a/type/__netbox_gunicorn/man.rst b/type/__netbox_gunicorn/man.rst index 1c09d78..8860294 100644 --- a/type/__netbox_gunicorn/man.rst +++ b/type/__netbox_gunicorn/man.rst @@ -14,6 +14,9 @@ sockets. Static content must be served independent of Gunicorn. The Gunicorn daemon is available as the `gunicorn-netbox` systemd service, but also available via the `netbox` wrapper service. +It will use systemd socket activation to listen to the given sockets. This +should allow to bind to privileaged ports (all below 1024) and hot reloads. + REQUIRED PARAMETERS ------------------- @@ -38,8 +41,9 @@ state bind-to The hosts the gunicorn socket should be bind to. Formats are `IP`, - `IP:PORT`, `unix:PATH` and `fd://FD`. Parameter can be set a multiple - times. Defaults to ``127.0.0.1:8001``. + `IP:PORT`, `PATH` or anything other that systemd socket units will + understand as stream. Parameter can be set multiple times. Defaults + to ``127.0.0.1:8001``. BOOLEAN PARAMETERS diff --git a/type/__netbox_gunicorn/manifest b/type/__netbox_gunicorn/manifest index 2e5ccce..5748e9d 100755 --- a/type/__netbox_gunicorn/manifest +++ b/type/__netbox_gunicorn/manifest @@ -25,17 +25,10 @@ case "$param_state" in esac -if [ "$state" = "present" ]; then - HOST="" - while read -r host; do - # shellcheck disable=SC2089 - HOST="$HOST '$host'," - done < "$__object/parameter/bind-to" - # shellcheck disable=SC2090 - export HOST +mkdir "$__object/files" +if [ "$state" = "present" ]; then # process template - mkdir "$__object/files" "$__type/files/gunicorn.py.sh" > "$__object/files/gunicorn.py" # gunicorn config file @@ -49,7 +42,16 @@ else fi -# install service file +TYPE="Gunicorn" +export TYPE + +"$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \ + > "$__object/files/netbox.socket" + +# install systemd files +__systemd_unit gunicorn-netbox.socket \ + --state "$state" --enablement-state "$unit_state" \ + --source "$__object/files/netbox.socket" --restart __systemd_unit gunicorn-netbox.service \ --state "$state" --enablement-state "$unit_state" \ --source "$__type/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/explorer/bind-capability b/type/__netbox_uwsgi/explorer/bind-capability new file mode 100755 index 0000000..c5c0365 --- /dev/null +++ b/type/__netbox_uwsgi/explorer/bind-capability @@ -0,0 +1,12 @@ +#!/bin/sh -e +# explorer/bind-capablility + +# Checks if the uWSGI binary have the capability to bind to privileaged ports +# as a non-root user. It's required if no systemd sockets are used (cause of +# the use of multiple protocols etc.) + +binary="/opt/netbox/venv/bin/uwsgi" +# -v verifies if capability is set +if setcap -q -v CAP_NET_BIND_SERVICE+ep "$binary"; then + echo set +fi diff --git a/type/__netbox_uwsgi/files/netbox.service b/type/__netbox_uwsgi/files/netbox.service.sh old mode 100644 new mode 100755 similarity index 70% rename from type/__netbox_uwsgi/files/netbox.service rename to type/__netbox_uwsgi/files/netbox.service.sh index cd06e82..3705769 --- a/type/__netbox_uwsgi/files/netbox.service +++ b/type/__netbox_uwsgi/files/netbox.service.sh @@ -1,15 +1,26 @@ +#!/bin/sh -e + +cat << EOF [Unit] Description=Netbox uWSGI WSGI Service Documentation=https://netbox.readthedocs.io/en/stable/ PartOf=netbox.service Requires=netbox-rq.service +EOF + +# Add dependency to own socket +if [ "$(cat "$__object/files/systemd_socket")" = "yes" ]; then + echo "Requires=uwsgi-netbox.socket" +fi + +cat << EOF Wants=network.target After=netbox.service After=network.target After=redis-server.service postgresql.service [Service] -Type=simple +Type=notify User=netbox Group=netbox @@ -17,8 +28,8 @@ WorkingDirectory=/opt/netbox ExecStart=/opt/netbox/venv/bin/uwsgi --master --chdir /opt/netbox/netbox --module netbox.wsgi uwsgi.ini # signals: https://uwsgi-docs.readthedocs.io/en/latest/Management.html#signals-for-controlling-uwsgi -ExecReload=kill -HUP $MAINPID -ExecStop=kill -INT $MAINPID +ExecReload=kill -HUP \$MAINPID +ExecStop=kill -INT \$MAINPID KillSignal=SIGQUIT Restart=on-failure @@ -26,3 +37,4 @@ RestartSec=30 [Install] WantedBy=netbox.service +EOF diff --git a/type/__netbox_uwsgi/files/netbox.socket.sh b/type/__netbox_uwsgi/files/netbox.socket.sh new file mode 120000 index 0000000..28ce920 --- /dev/null +++ b/type/__netbox_uwsgi/files/netbox.socket.sh @@ -0,0 +1 @@ +../../__netbox/files/netbox.socket.sh \ No newline at end of file diff --git a/type/__netbox_uwsgi/files/uwsgi.ini.sh b/type/__netbox_uwsgi/files/uwsgi.ini.sh index 7835a3e..4bae613 100755 --- a/type/__netbox_uwsgi/files/uwsgi.ini.sh +++ b/type/__netbox_uwsgi/files/uwsgi.ini.sh @@ -28,20 +28,16 @@ cat << EOF ; socket(s) to bind EOF -# special protocol to bind -while read -r param; do - if [ -z "$param" ]; then continue; fi # ignore empty lines from the here-doc - - multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" - socket_changes="yes" - -done << INPUT # here-doc cause of SC2031 -$( find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print ) -INPUT - -# else, default bind to -if [ -z "$socket_changes" ]; then - multi_options "socket" "$__object/parameter/bind-to" +if [ "$SYSTEMD_SOCKET" != "yes" ]; then + # special protocol to bind + find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print \ + | while read -r param; do + multi_options "$(basename "$param" | awk -F'-' '{print $1}')-socket" "$param" + done +else + # else, systemd will offer socket + echo "; sockets managed via 'uwsgi-netbox.socket'" + printf "protocol = %s\n" "$PROTOCOL" fi diff --git a/type/__netbox_uwsgi/gencode-remote b/type/__netbox_uwsgi/gencode-remote index c219643..7c3b826 100755 --- a/type/__netbox_uwsgi/gencode-remote +++ b/type/__netbox_uwsgi/gencode-remote @@ -3,18 +3,67 @@ # control state state="$(cat "$__object/parameter/state")" +# Set capabilities to aquire privileaged ports as netbox user. Two modes are +# available to efficiently set capabilites. Assumes libcap-bin is installed as +# default on debian systems. +# +# Arguments: +# 1: mode to detect if capabilites are required to set ('set' or 'correct') +set_bind_cap() { + cap_mode="" # reset variable from the execution before + + # check if capabilites are required after given mode + case "$1" in + # assumes capabilites are not set (cause of new binaries) + set) + if [ "$SYSTEMD_SOCKET" != "yes" ]; then + cap_mode="+ep" + fi + ;; + + # check if capabilities have changed + correct) + if [ -s "$__object/explorer/bind-capability" ]; then + # capabilites are set + if [ "$SYSTEMD_SOCKET" = "yes" ]; then + cap_mode="-ep" # unset + fi + else + # capabilities are unset + if [ "$SYSTEMD_SOCKET" != "yes" ]; then + cap_mode="+ep" # set + fi + fi + ;; + + # faulty mode + *) + echo "called set_bind_cap incorrect (\$1 missing)" >&2 + ;; + esac + + # set capabilities if any + if [ "$cap_mode" ]; then + printf "setcap -q CAP_NET_BIND_SERVICE%s /opt/netbox/venv/bin/uwsgi\n" "$cap_mode" + fi +} +SYSTEMD_SOCKET="$(cat "$__object/files/systemd_socket")" + + case "$state" in # install uwsgi enabled|disabled) # not installed if ! [ -s "$__object/explorer/installed" ]; then - echo "/opt/netbox/venv/bin/pip3 install uwsgi" + echo "/opt/netbox/venv/bin/pip3 install -q uwsgi" + set_bind_cap set do_restart=yes printf "installed\n" >> "$__messages_out" # updates available elif [ -s "$__object/explorer/upgradeable" ]; then - echo "/opt/netbox/venv/bin/pip3 install --upgrade uwsgi" + echo "/opt/netbox/venv/bin/pip3 install -q --upgrade uwsgi" + set_bind_cap set do_restart=yes printf "upgraded\n" >> "$__messages_out" fi @@ -25,6 +74,11 @@ case "$state" in printf "configured\n" >> "$__messages_out" fi + # if no capabilities were set yet, check if any are required + if [ -z "$cap_mode" ]; then + set_bind_cap correct + fi + # restart uwsgi if [ "$do_restart" ] && [ "$state" != "disabled" ]; then @@ -40,7 +94,7 @@ EOF # check if installed if [ -s "$__object/explorer/installed" ]; then # service already disabled - echo "/opt/netbox/venv/bin/pip3 uninstall -y uwsgi" + echo "/opt/netbox/venv/bin/pip3 uninstall -qy uwsgi" printf "uninstalled\n" >> "$__messages_out" fi ;; diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index 3fb8515..d54401e 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -15,6 +15,19 @@ protocols like uwsgi, fastcgi or HTTP to comunicate with the proxy server. This application is available via the `uwsgi-netbox` systemd service. It is controllable via the `netbox` wrapper service, too. +**As uWSGI will be started as netbox user, it does not have privileges to +bind to a privileaged port (all ports below 1024).** Because uWSGI will +drop privileages anyway before binding to a port, solutions are to use +the systemd sockets to activate the ports as root or set linux kernel +capabilites to bind to such a privileaged port. + +As systemd sockets (or uwsgi itself) do not allow to distinguish multiple +sockets if different protocols are used for different sockets, this type does +not use systemd sockets if it is requested from the user. Using the +``--bind-to`` and ``--protocol`` parameters, it uses the systemd socket +activation. Else, it set the different sockets and protocols natively to uwsgi +and add kernel capabilities to be able to listen to privileaged ports. + REQUIRED PARAMETERS ------------------- @@ -39,8 +52,18 @@ state bind-to - The socket uwsgi should bind to. Must be UNIX/TCP for the uwsgi protocol. - Defaults to ``127.0.0.1:3031``. Can be set multiple times. + The socket uwsgi should bind to. Must be UNIX/TCP (or anything that + systemd sockets accept as stream). Defaults to ``127.0.0.1:3031``. Can be + set multiple times. The used protocol is defined by ``--protocol``. + + **By setting up the socket via this parameter, it uses systemd sockets to + handle these.** This parameter will be ignored if a more detailed paramter + is given (``--$proto-bind``). + +protocol + The protocol which should be used for the socket given by the ``--bind-to`` + parameter. Possible values are ``uwsgi``, ``http``, ``fastcgi`` and + ``scgi``. If nothing given, it defaults to ``uwsgi``. uwsgi-bind http-bind @@ -50,6 +73,12 @@ scgi-bind ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must be a UNIX/TCP socket. Can be set multiple times. + **By using such parameters instead of ``--bind-to``, no systemd sockets + will be used because it can not handle sockets for multiple protocols.** + Instead, the native socket binding will be used. It will add kernel + capabilites to bind to privileaged ports, too. This allow binds to ports + like 80 as netbox user. + BOOLEAN PARAMETERS ------------------ diff --git a/type/__netbox_uwsgi/manifest b/type/__netbox_uwsgi/manifest index c5885c9..7c593e8 100755 --- a/type/__netbox_uwsgi/manifest +++ b/type/__netbox_uwsgi/manifest @@ -25,15 +25,30 @@ case "$param_state" in esac +mkdir "$__object/files" + +# check if systemd sockets will be used +if [ -f "$__object/parameter/bind-to" ]; then + SYSTEMD_SOCKET="yes" +fi +if find "$__object/parameter/" -maxdepth 1 -name "*-bind" -print -quit | grep -q .; then + SYSTEMD_SOCKET="no" +fi +echo "$SYSTEMD_SOCKET" > "$__object/files/systemd_socket" + if [ "$state" = "present" ]; then - # *bind* parameters are directly processed in the gen script + # already checked outside this if-clause + export SYSTEMD_SOCKET + + PROTOCOL="$(cat "$__object/parameter/protocol")" + export PROTOCOL + if [ -f "$__object/parameter/serve-static" ]; then STATIC_MAP="yes" export STATIC_MAP fi # process template - mkdir "$__object/files" "$__type/files/uwsgi.ini.sh" > "$__object/files/uwsgi.ini" # uwsgi config file @@ -48,7 +63,24 @@ else fi +# handle the systemd socket +if [ "$SYSTEMD_SOCKET" = "yes" ]; then + TYPE="uWSGI" + export TYPE + + # generate and set the socket unit + "$__type/files/netbox.socket.sh" "$__object/parameter/bind-to" \ + > "$__object/files/netbox.socket" + __systemd_unit uwsgi-netbox.socket \ + --state "$state" --enablement-state "$unit_state" \ + --source "$__object/files/netbox.socket" --restart +else + # remove the systemd socket unit + __systemd_unit uwsgi-netbox.socket --state absent +fi + # install service file +"$__type/files/netbox.service.sh" > "$__object/files/netbox.service" __systemd_unit uwsgi-netbox.service \ --state "$state" --enablement-state "$unit_state" \ - --source "$__type/files/netbox.service" --restart + --source "$__object/files/netbox.service" --restart diff --git a/type/__netbox_uwsgi/parameter/default/protocol b/type/__netbox_uwsgi/parameter/default/protocol new file mode 100644 index 0000000..caf986e --- /dev/null +++ b/type/__netbox_uwsgi/parameter/default/protocol @@ -0,0 +1 @@ +uwsgi diff --git a/type/__netbox_uwsgi/parameter/optional b/type/__netbox_uwsgi/parameter/optional index ff72b5c..3284ccc 100644 --- a/type/__netbox_uwsgi/parameter/optional +++ b/type/__netbox_uwsgi/parameter/optional @@ -1 +1,2 @@ state +protocol From 32c95ccb220cb3302d1c43cca5ace01fa66a6755 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:50:22 +0200 Subject: [PATCH 131/332] __netbox: don't call `pip3 uninstall` if nothing available To avoid an error from pip, it will be checked if there are packages to uninstall. This is better than just doing `|| true`. --- type/__netbox/gencode-remote | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 1ef78cd..07b0fcf 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -66,12 +66,16 @@ ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' virtualenv -p python3 /opt/netbox/venv # Uninstall packages not required anymore -# if versions not shortend, they will be ignored by pip, but not by comm +# if versions not be shortend, they will be ignored by pip, but not by comm # all of this could be done with grep, too, but it's still must be shortend with awk awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" -comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" \ - | xargs /opt/netbox/venv/bin/pip3 uninstall -qy +comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" > "\$tmpdir/pip-uninstall.txt" + +# only uninstall if something is available (to avoid errors cause of this) +if [ -s "\$tmpdir/pip-uninstall.txt" ]; then + /opt/netbox/venv/bin/pip3 uninstall -qy -r "\$tmpdir/pip-uninstall.txt" +fi # Install python dependencies. # avoid gunicorn, because it will be done in an other type @@ -86,6 +90,7 @@ EOF fi cat << EOF + # Set final permissions. chown -R netbox /opt/netbox From 72f2996c509e64a9f35f53d7b347ae65d1e6df59 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 11 Oct 2020 16:56:54 +0200 Subject: [PATCH 132/332] __netbox*: fix shellcheck SC2162 --- type/__netbox/files/netbox.socket.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/files/netbox.socket.sh b/type/__netbox/files/netbox.socket.sh index 03e3d44..2ef9e81 100755 --- a/type/__netbox/files/netbox.socket.sh +++ b/type/__netbox/files/netbox.socket.sh @@ -20,7 +20,7 @@ Description=Socket for NetBox via $TYPE UNIT # read all sockets to listen to -while read line; do +while read -r line; do printf "ListenStream=%s\n" "$line" done < "$1" From 144cae3b796831d4c96e221611b25945f4462f68 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 10:16:25 +0200 Subject: [PATCH 133/332] __netbox: use __pyenv and remove obsolete netbox stop Now, the python venv is now created via `pyvenv` or `python3 -m venv` instead of the legacy `virtualenv`. For this, not all python processes from the venv need to be stopped. Migration from previous versions of this type may be difficult, but solvable if the venv will be recreated. --- type/__netbox/gencode-remote | 32 +++++++------------------------- type/__netbox/manifest | 2 ++ 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 07b0fcf..591717f 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -24,26 +24,7 @@ cd "\$tmpdir" curl -sS -L '$url' > '$archive' tar xf '$archive' -EOF - # Stop everything in the pyenv to update - cat << EOF -# Try to stop everything in the venv (ignore non-existant services) -systemctl -q stop netbox gunicorn-netbox uwsgi-netbox || true - -# kill and poll till all venv processes end -# at least if they are called with full path in the cmd (like in the services) -pids="\$( ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' )" -if [ "\$pids" ]; then - kill \$pids - while ps -axo pid,cmd | awk '\$2 ~ "^/opt/netbox/venv/"{print \$1}' | grep -q . ; do - sleep 0.5 - done -fi - -EOF - - cat << EOF # backup requirement files if [ -f /opt/netbox/requirements.txt ]; then cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt @@ -61,9 +42,7 @@ ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' -# Setup & enter python virtualenv. -# forcing python3 to be sure (till python4 gets released ..) -virtualenv -p python3 /opt/netbox/venv +# virtualenv is given already by __pyvenv, just using it # Uninstall packages not required anymore # if versions not be shortend, they will be ignored by pip, but not by comm @@ -112,6 +91,7 @@ rm -rf "\$tmpdir" # Save version after successful installation printf "%s\\n" "$VERSION" > /opt/netbox/cdist/version + EOF # meta @@ -127,10 +107,12 @@ if grep -q "^__file/opt/netbox/" "$__messages_in"; then fi -# check for changes +# Check for changes if [ "$changes" = "yes" ]; then - # to avoid database corruption at config changes, both services must be - # stopped at the same time (noted in the manual, too). + # After the upstream upgrade.sh script, it's ok to migrate while the + # application is running ;) + + # restarting after changes cat << EOF # Restart service. All required services are included with netbox.service. systemctl restart netbox diff --git a/type/__netbox/manifest b/type/__netbox/manifest index fbcf253..475cadb 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -187,6 +187,8 @@ fi # Create system user used to run netbox. __user netbox --system --home /opt/netbox --create-home +# Generate python environment (user will be set by gencode-remote) +require="__user/netbox" __pyvenv /opt/netbox/venv/ # Generate and upload netbox configuration. mkdir -p "$__object/files" From 9def5700c8a1de7e59ed0ac3c40cdd1b7c484e05 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 10:21:52 +0200 Subject: [PATCH 134/332] __netbox: replace redis DEFAULT_TIMEOUT with RQ_DEFAULT_TIMEOUT The first one was deprecated by the second one. --- type/__netbox/files/configuration.py.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/type/__netbox/files/configuration.py.sh b/type/__netbox/files/configuration.py.sh index 06a1c86..31ebd05 100755 --- a/type/__netbox/files/configuration.py.sh +++ b/type/__netbox/files/configuration.py.sh @@ -36,7 +36,6 @@ REDIS = { # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '$REDIS_PASSWORD', 'DATABASE': $((REDIS_DBID_OFFSET + 0)), - 'DEFAULT_TIMEOUT': 300, 'SSL': $REDIS_SSL, }, 'caching': { @@ -47,10 +46,10 @@ REDIS = { # 'SENTINEL_SERVICE': 'netbox', 'PASSWORD': '$REDIS_PASSWORD', 'DATABASE': $((REDIS_DBID_OFFSET + 1)), - 'DEFAULT_TIMEOUT': 300, 'SSL': $REDIS_SSL, } } +RQ_DEFAULT_TIMEOUT = 300 # This key is used for secure generation of random numbers and strings. It must never be exposed outside of this file. # For optimal security, SECRET_KEY should be at least 50 characters in length and contain a mix of letters, numbers, and From 558021d2eeb10697d2dec0e7dc563fbf26e63ba0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 11:08:11 +0200 Subject: [PATCH 135/332] __netbox: reorder generated code for shorter transition Now, the venv and pip things are done before the program files are replaced, which should minimize the outage where the application is not available. --- type/__netbox/gencode-remote | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/type/__netbox/gencode-remote b/type/__netbox/gencode-remote index 591717f..5d4b7be 100755 --- a/type/__netbox/gencode-remote +++ b/type/__netbox/gencode-remote @@ -25,30 +25,22 @@ curl -sS -L '$url' > '$archive' tar xf '$archive' +# virtualenv is given already by __pyvenv, just using it + # backup requirement files if [ -f /opt/netbox/requirements.txt ]; then - cp /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt + mv /opt/netbox/requirements.txt /opt/netbox/old-requirements.txt else # preseve file-not-found errors and warnings touch /opt/netbox/old-requirements.txt fi cp '$src/requirements.txt' /opt/netbox/ -# Deploy sources and restore configuration. -rm -rf '$install_dir' -cp -r '$src/netbox' '$install_dir' -# force links to the cdist directory -ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' -ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' - - -# virtualenv is given already by __pyvenv, just using it - # Uninstall packages not required anymore # if versions not be shortend, they will be ignored by pip, but not by comm # all of this could be done with grep, too, but it's still must be shortend with awk -awk -F== '{print $1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" -awk -F== '{print $1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" +awk -F== '{print \$1}' '/opt/netbox/requirements.txt' | sort > "\$tmpdir/curr-reqs.txt" +awk -F== '{print \$1}' '/opt/netbox/old-requirements.txt' | sort > "\$tmpdir/old-reqs.txt" comm -23 "\$tmpdir/old-reqs.txt" "\$tmpdir/curr-reqs.txt" > "\$tmpdir/pip-uninstall.txt" # only uninstall if something is available (to avoid errors cause of this) @@ -70,9 +62,17 @@ EOF cat << EOF +# Deploy sources and restore configuration. +rm -rf '$install_dir' +cp -r '$src/netbox' '$install_dir' +# force links to the cdist directory +ln -fs /opt/netbox/cdist/configuration.py '$install_dir/netbox/configuration.py' +ln -fs /opt/netbox/cdist/ldap_config.py '$install_dir/netbox/ldap_config.py' + # Set final permissions. chown -R netbox /opt/netbox + # NetBox manage scripts # Run database migrations. sudo -u netbox /opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py migrate From 46bff41f6d8bb9a0fd50ac7b659690f5e94377dc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 17 Oct 2020 11:22:47 +0200 Subject: [PATCH 136/332] __netbox{,_uwsgi}: updated manpages Mostly added notes of what does not work that well. --- type/__netbox/man.rst | 28 +++++++++++++--------------- type/__netbox_uwsgi/man.rst | 15 +++++++++++++++ 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 6d50268..f34ead7 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -16,7 +16,7 @@ It setup systemd unit files for the services `netbox` and `netbox-rq`. The `netbox` service only wrap all netbox related services, e.g. restarting and so one will be delegated to all related services. -The application is still not accessable because WSGI server is required. To +The application is still not accessable because a WSGI server is required. To access the application through WSGI, uWSGI or Gunicorn can be used. The setup can be done via there own types `__netbox_gunicorn` and `__netbox_uwsgi`. @@ -236,23 +236,21 @@ If you not setup ldap authentification, you may be interested into how to ` directly on the machine to be able to access and use NetBox. -If you change a configuration, the database may go corrupt if two instances of -the application are running with different configurations at the same time. -This most commonly happens when the WSGI server and RQ-worker restarts after a -configuration change. This occours in the following case for example: +You may also be interested of writing a own type which handles the creation of +the super user. To do this non-interactivly, see the ansible role as `reference +`. -.. code-block:: sh +If you change the secret key while the netbox instance is running, there is a +time frame where the access to the application corrupts the whole database. +Then, you need to restore a backup or wipe the database. - systemctl restart gunicorn-netbox # WSGI-server already online with new - # configuration after this command. - systemctl restart netbox-rq # RQ-Worker still worked with the old - # configuration till here. +Currently, the cause is not clear, but it should work if you do not touch +netbox while the configuration is done (do not shut it down, too). It only +applies for changes of the secret key, which not happen normally. -This type handles the restart of both services correctly to avoid such database -corruptions. To safely manual restart the whole netbox instance manual, simply -restart all services in one ``systemctl restart netbox`` command, as it ensures -that first all services are shut down before starting one of them. The service -``netbox`` wraps all required services that are available. +Maybe the `--restart` flag for the `__systemd_unit` types is not the best idea, +but avoids that the changes will not be applied. It could be solved if the type +would send messages from his actions. SEE ALSO diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index d54401e..c5e51ca 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -129,6 +129,7 @@ EXAMPLES # with multiple protocols # parameter `--bind-to` will be ignored + # avoids systemd sockets, but can handle multiple protocols __netbox $args require="__netbox" __netbox_uwsgi --uwsgi-bind 0.0.0.0:3031 \ --http-bind 0.0.0.0:8080 \ @@ -150,6 +151,20 @@ EXAMPLES require="__netbox_uwsgi" __systemd_service uwsgi-netbox --state stopped +NOTES +----- +If systemd sockets are used, uwsgi can not be reloaded because it does not +handle the socket correctly. It works by completly restarting uwsgi (because +it is near the same cause of the systemd socket) or tweaking the service unit +with the line ``StandardInput=socket``, which limits you to only one address +to bind to (else, the service will not start). + +Maybe someone is interested in enabling log files, because the "log to stdout" +is not the fanciest approach (because it is shown in the journal). See the +`uwsgi documentation ` for +reference. + + SEE ALSO -------- `uWSGI Documentation `_ From c1d5d5b088724c22c8b8e79f00a1b82ed88436f2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 08:52:43 +0200 Subject: [PATCH 137/332] __netbox: remove dependency to package virtualenv This is not required any more because it was completly replaced by `python3-venv`. The venv handling is already given to `__pyvenv`. --- type/__netbox/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__netbox/manifest b/type/__netbox/manifest index 475cadb..4be49bd 100755 --- a/type/__netbox/manifest +++ b/type/__netbox/manifest @@ -6,7 +6,7 @@ case "$os" in debian|ubuntu) # Install netbox dependencies. for pkg in python3-pip python3-venv python3-dev build-essential libxml2-dev \ - libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl virtualenv sudo; do + libxslt1-dev libffi-dev libpq-dev libssl-dev zlib1g-dev curl sudo; do __package $pkg done From 979398e398e45c08bd50dc78b72a63341f5339b8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 09:05:22 +0200 Subject: [PATCH 138/332] Initial __nextcloud type This type should work, but there are still many things to do here to work nice and well. Things that currently not work (well): - not all parameters work for now - nextcloud installations in subfolders (e.g. slashes in the object id) --- type/__nextcloud/explorer/config | 28 +++ type/__nextcloud/explorer/installdir | 16 ++ type/__nextcloud/explorer/version | 19 ++ type/__nextcloud/explorer/webroot | 34 +++ type/__nextcloud/gencode-remote | 209 ++++++++++++++++++ type/__nextcloud/manifest | 132 +++++++++++ type/__nextcloud/map-conf-changes.sh | 196 ++++++++++++++++ type/__nextcloud/parameter/boolean | 1 + .../parameter/default/database-type | 1 + type/__nextcloud/parameter/default/group | 1 + type/__nextcloud/parameter/default/mode | 1 + type/__nextcloud/parameter/default/user | 1 + type/__nextcloud/parameter/optional | 13 ++ type/__nextcloud/parameter/optional_multiple | 1 + type/__nextcloud/parameter/required | 2 + 15 files changed, 655 insertions(+) create mode 100755 type/__nextcloud/explorer/config create mode 100755 type/__nextcloud/explorer/installdir create mode 100755 type/__nextcloud/explorer/version create mode 100755 type/__nextcloud/explorer/webroot create mode 100755 type/__nextcloud/gencode-remote create mode 100755 type/__nextcloud/manifest create mode 100755 type/__nextcloud/map-conf-changes.sh create mode 100644 type/__nextcloud/parameter/boolean create mode 100644 type/__nextcloud/parameter/default/database-type create mode 100644 type/__nextcloud/parameter/default/group create mode 100644 type/__nextcloud/parameter/default/mode create mode 100644 type/__nextcloud/parameter/default/user create mode 100644 type/__nextcloud/parameter/optional create mode 100644 type/__nextcloud/parameter/optional_multiple create mode 100644 type/__nextcloud/parameter/required diff --git a/type/__nextcloud/explorer/config b/type/__nextcloud/explorer/config new file mode 100755 index 0000000..998264e --- /dev/null +++ b/type/__nextcloud/explorer/config @@ -0,0 +1,28 @@ +#!/bin/sh -e +# __nextcloud/explorer/config + +# Checks the nextcloud configuration + + +# Get the installdir +user="$( cat "$__object/parameter/user" )" +installdir="$( "$__type_explorer/installdir" )" + +# Check if the tools are available +if [ -d "$installdir" ]; then + cd "$installdir" + + # if those files exist, everything should be good + if [ -f "occ" ] && [ -f "config/config.php" ]; then + # dump out config instead of fuzz every possible option through + # `occ config:system:get`. Or parse through the whole json or + # yaml-like output of `occ config:list system --private`. + php -r 'require("lib/private/Config.php"); $config = new OC\Config("config/"); + function printv($key, $value) {printf("%s = %s\n", $key, $value);} + foreach($config->getKeys() as $key){ + $value = $config->getValue($key); + if(is_array($value)) foreach($value as $n => $in) printv($n."|".$key, $in); + else printv($key, $value); + };' + fi +fi diff --git a/type/__nextcloud/explorer/installdir b/type/__nextcloud/explorer/installdir new file mode 100755 index 0000000..7b34f2e --- /dev/null +++ b/type/__nextcloud/explorer/installdir @@ -0,0 +1,16 @@ +#!/bin/sh -e +# __nextcloud/explorer/installdir + +# Detects the directory nextcloud should be installed to. + + +# by parameter or auto-detection +webroot="$( "$__type_explorer/webroot" )" +if [ -z "$webroot" ]; then + echo "no installdir given and no webroot directory found" >&2 + echo "no place to install found; set it via --webroot" >&2 + exit 1 +fi + +# assemble directory with the object id +printf "%s/%s\n" "$webroot" "$__object_id" diff --git a/type/__nextcloud/explorer/version b/type/__nextcloud/explorer/version new file mode 100755 index 0000000..fc79206 --- /dev/null +++ b/type/__nextcloud/explorer/version @@ -0,0 +1,19 @@ +#!/bin/sh -e +# __nextcloud/explorer/version + +# Check the currently installed version. Outputs nothing if nothing found. + + +# Get the install directory +installdir="$( "$__type_explorer/installdir" )" + +# Check if the installation directory exists +if [ -d "$installdir" ]; then + cd "$installdir" + + # if those files exist, everything should be good + if [ -f "occ" ] && [ -f "version.php" ]; then + # detect php version with the version file + php -r 'require("version.php"); print($OC_VersionString);' + fi +fi diff --git a/type/__nextcloud/explorer/webroot b/type/__nextcloud/explorer/webroot new file mode 100755 index 0000000..7eab286 --- /dev/null +++ b/type/__nextcloud/explorer/webroot @@ -0,0 +1,34 @@ +#!/bin/sh -e +# __nextcloud/explorer/webroot + +# Detects the webroot if any + + +# Just check if there is some directory and echo + exit on success +# +# Arguments: +# 1: the directory to check +check_dir() { + if [ -d "$1" ]; then + echo "$1" + exit + fi +} + + +# Check the user choice +parameter="$__object/parameter/webroot" +if [ -f "$parameter" ]; then + cat "$parameter" + exit +fi + +# Maybe checking standard webserver configs .. + +# Check if there are default directories +check_dir "/srv/www" +check_dir "/var/www/html" +check_dir "/var/www" + + +# do nothing if no webroot found diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote new file mode 100755 index 0000000..60d9b42 --- /dev/null +++ b/type/__nextcloud/gencode-remote @@ -0,0 +1,209 @@ +#!/bin/sh -e +# __nextcloud/gencode-remote + +# Install if not installed + +# Legacy: +# curl -sS -L '$nextcloud_uri' | tar xj --strip-components=1 nextcloud/ + + +# Call the nextcloud occ script as the designed user. Maybe this can be a bit +# more effictive with user switching, but currently the easiest way of doing +# it. +# +# All arguments are directly passed to occ (injection alarm ;-) ) +occ() { + # su creates a new shell, so it does not affect the current session + # will not use -q as it supresses errors, too + cat << SHELL +su -s /bin/sh -l "$user" -- -e <> "$__messages_out" + +# Apply some misc to the installation folder. +elif [ "$install" ]; then + cat << REMOTE +chown '$user':'$group' -R '$installdir' +REMOTE +fi + + +# Check if the nextcloud application needs to be installed. +# This checks the state of the configuration, not of the directory. +if ! grep -q -F "installed = 1" "$__object/explorer/config"; then + # argument construction + occ_install_args="" + + # Database + db_setup() { + if ! [ -f "$__object/parameter/db-host" ]; then + echo "no hostname given! can't proceed." >&2 + exit 3 + fi + occ_install_args="$occ_install_args --database '$1'" + occ_install_args="$occ_install_args --database-host '$(cat "$__object/parameter/db-host")'" + + db_name="$__object/parameter/database-name" + if [ -f "$db_name" ]; then + occ_install_args="$occ_install_args --database-name '$(cat "$db_name")'" + fi + db_user="$__object/parameter/database-user" + if [ -f "$db_user" ]; then + occ_install_args="$occ_install_args --database-user '$(cat "$db_user")'" + fi + db_pass="$__object/parameter/database-password" + if [ -f "$db_pass" ]; then + occ_install_args="$occ_install_args --database-pass '$(cat "$db_pass")'" + fi + db_prefix="$__object/parameter/database-prefix" + if [ -f "$db_prefix" ]; then + occ_install_args="$occ_install_args --database-table-prefix '$(cat "$db_prefix")'" + fi + } + + database_type="$(cat "$__object/parameter/database-type")" + case "$database_type" in + sqlite|sqlite3) + occ_install_args="$occ_install_args --database sqlite" + ;; + mysql|mariadb) + db_setup mysql + ;; + pgsql|postgres|postgresql) + db_setup pgsql + ;; + + *) + printf "Database type '%s' is unkown!\n" "" >&2 + exit 3 + ;; + esac + + # Admin stuff + occ_install_args="$occ_install_args --admin-pass '$(cat "$__object/parameter/admin-password")'" + + admin_user="$__object/parameter/admin-user" + if [ -f "$admin_user" ]; then + occ_install_args="$occ_install_args --admin-user '$(cat "$admin_user")'" + fi + admin_email="$__object/parameter/admin-email" + if [ -f "$admin_email" ]; then + occ_install_args="$occ_install_args --admin-email '$(cat "$admin_email")'" + fi + + # Data directory + datadir="$__object/parameter/datadir" + if [ -f "$datadir" ]; then + occ_install_args="$occ_install_args --data-dir '$(cat "$datadir")'" + fi + + + # Execute the install command + occ maintenance:install $occ_install_args + + # send install message + echo installed >> "$__messages_out" +fi + + +# Handle the config +mkdir "$__object/files" +"$__type/map-conf-changes.sh" > "$__object/files/conf-cmds" + +# only print if there are changes listed +if [ -s "$__object/files/conf-cmds" ]; then + # save that we did changes + changes="yes" + + # print change commands incl. the switch of user context + # using -e to abort if the commands failed + printf "su -s /bin/sh -l '%s' -- -e << 'SU'\n" "$user" + printf "cd '%s'\n" "$installdir" + cat "$__object/files/conf-cmds" + printf "SU\n" + + # print a message + echo config >> "$__messages_out" +fi + + +# Check if this is the fist install +if [ "$install" ]; then + # do some convert stuff etc. + + # variable accessible from the last $install if-clause + case "$database_type" in + mysql|mariadb) + # only available for mysql + occ db:convert-mysql-charset + ;; + esac + + occ db:convert-filecache-bigint +fi + +# Disable maintainer mode +if [ "$install" ] || [ "$upgrade" ] || [ "$changes" ]; then + occ maintenance:mode --off +fi diff --git a/type/__nextcloud/manifest b/type/__nextcloud/manifest new file mode 100755 index 0000000..f6b8865 --- /dev/null +++ b/type/__nextcloud/manifest @@ -0,0 +1,132 @@ +#!/bin/sh -e +# __nextcloud/manifest + + +# Version compare function original from __sensible_editor +# +# Arguments: +# 1: version of which $2 should be checked against +# 2: version which should be bigger than or equal with $1 +# +# Return code: +# 0: $1 is bigger than $2 +# 1-n: $1 is smaller than or equal $2 +version_ge() { + printf "%s" "$1" | awk -F '[^0-9.]' -v target="$2" ' + function max(x, y) { return x > y ? x : y } + BEGIN { + getline + nx = split($1, x, ".") + ny = split(target, y, ".") + for (i = 1; i <= max(nx, ny); ++i) { + diff = int(x[i]) - int(y[i]) + if (diff == 0) continue + exit (diff < 0) + } + exit 1 + }'; return $? +} + + +# Check support status +os="$(cat "$__global/explorer/os")" + +case "$os" in + debian|ubuntu) + # PHP main + __package php-cli + # to unpack the package + __package bzip2 + # install misc packages for nextcloud + __package ffmpeg + + # PHP modules + for package in php-gd php-json php-mysql php-curl php-mbstring php-intl \ + php-imagick php-xml php-zip php-bz2 php-bcmath php-gmp + do + require="__package/php-cli" __package $package + done + + # check support database additions (but don't remove junk of old ones) + case "$(cat "$__object/parameter/database-type")" in + sqlite|sqlite3) + __package php-sqlite3 + ;; + mysql|mariadb) + __package php-mysql + ;; + pgsql|postgres|postgresql) + __package php-pgsql + ;; + esac + ;; + + # unkown distro - what to install? + *) + printf "unkown %s, don't know what to install ..\n" "$os" >&2 + echo "checkout the __nextcloud/manifest to contribute a working package list" >&2 + exit 1 + ;; +esac + + +# Get the user and group +mode="$(cat "$__object/parameter/mode")" +user="$(cat "$__object/parameter/user")" +group="$(cat "$__object/parameter/group")" + +# Get the installation directory +webroot="$(cat "$__object/explorer/webroot")" +installdir="$(cat "$__object/explorer/installdir")" + +# Set permissions after the nextcloud installation/upgrade is done +# FIXME maybe less strict if some parameter is not given by the user? +# permissions also partily set via the gencode-remote +require="__nextcloud/$__object_id" __directory "$installdir" \ + --mode "$mode" --owner "$user" --group "$group" + + +# Get version information +version_is="$( cat "$__object/explorer/version" )" +version_should="$( cat "$__object/parameter/version" )" +# The version URI +nextcloud_uri="https://download.nextcloud.com/server/releases/nextcloud-${version_should}.tar.bz2" +nextcloud_sum="${nextcloud_uri}.sha256" + + +# Only check if there is a current installation +if [ "$version_is" ]; then + # Block downgrades as there are may caused from the automatic upgrader + # if the current version is higher than the version that should be installed + if version_ge "$version_is" "$version_should"; then + # it's an error if the current version is higher than the one that should be installed + printf "The current nextcloud version '%s' is higher than the version that should be installed (%s)\n" \ + "$version_is" "$version_should" >&2 + printf "Please bump the nextcloud version to '%s' or higher!\n" "$version_is" >&2 + exit 2 + fi + + # Set destination to a temporary directory + destination="$webroot/.$__object_id" +else + # Set destination to the real destination + destination="$webroot/$__object_id" +fi + +# Upgrade the nextcloud version +if [ "$version_is" != "$version_should" ]; then + updatedir="$( dirname "$installdir" )" + + # Be sure the parent directory exists for installation + __directory "$updatedir" --parents + + require="__directory$updatedir" \ + __download "$updatedir/nextcloud.tar.bz2" \ + --url "$nextcloud_uri" \ + --sum "sha256:$(curl -sS -L "$nextcloud_sum" | awk '{print $1}')" # must be a required parameter?? + + require="__download$updatedir/nextcloud.tar.bz2" \ + __unpack "$updatedir/nextcloud.tar.bz2" \ + --tar-strip 1 \ + --destination "$destination" +fi diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh new file mode 100755 index 0000000..8a00ac6 --- /dev/null +++ b/type/__nextcloud/map-conf-changes.sh @@ -0,0 +1,196 @@ +#!/bin/sh -e +# __nextcloud/map-conf-changes.sh + + +# The environment variable "$install" should be set if nextcloud was installed +# now. This changes the behaviour to not trust gathered values from the +# explorer. + + +# Test if the value exists as given. +# +# Arguments: +# 1: The nextcloud config name +# 2: The value that should be set +# +# Return code: +# 0: value exactly matched +# 1: value not matched or do not exist +testparam() { + # short-circuit after installation; the explorer may not be valid + if [ "$install" ]; then return 1; fi + + if grep -q -F "$1 = $2" "$__object/explorer/config"; then + return 0 + else + return 1 + fi +} + +# Test if the parameter is somehow set. +# +# Arguments: +# 1: The nextcloud config name +# +# Return code: +# 0: param exists +# 1: param not found +paramexist() { + # short-circuit after installation; the explorer may not be valid + if [ "$install" ]; then return 0; fi + + if grep -q "^$1 = " "$__object/explorer/config"; then + return 0 + else + return 1 + fi +} + +# Base for the basic function types. +# +# Arguments: +# 1: cdist type parameter name +# 2: nextcloud config name +# 3: occ printf pattern to set the value +conf_base() { + if [ -f "$__object/parameter/$1" ]; then + value="$(cat "$__object/parameter/$1")" + if ! testparam "$2" "$value"; then + # set it because it does not exist + printf "php occ config:system:$3\n" "$2" "$value" + fi + else + if paramexist "$2"; then + # remove it because it exists + printf "php occ config:system:delete '%s'\n" "$2" + fi + fi +} + +# Set's the cdist parameter value to nextcloud as specific value. +# +# Arguments: +# 1: cdist type parameter name +# 2: nextcloud config name +conf_string() { + conf_base "$1" "$2" "set '%s' --type=string --value='%s'" +} +conf_number() { + conf_base "$1" "$2" "set '%s' --type=integer --value='%s'" +} +conf_decimal() { + conf_base "$1" "$2" "set '%s' --type=double --value='%s'" +} + +# Sets the nextcloud configuration option after a boolean cdist parameter. +# +# Arguments: +# 1: cdist type parameter name +# 2: nextcloud config name +conf_boolean() { + # map parameter to a php boolean (are outputted as 0 or 1) + if [ -f "$__object/parameter/$1" ]; then + testval="1" + value="true" + else + testval="0" + value="false" + fi + + if ! testparam "$2" "$testval"; then + # set it if does not already exist + printf "php occ config:system:set '%s' --type=boolean --value=%s\n" "$2" "$value" + fi +} + +# Corrects the array after all values given by the parameter. Values not given +# to this type will be removed. +# +# Arguments: +# 1: cdist type parameter name +# 2: nextcloud config name +conf_array() { + if [ -f "$__object/parameter/$1" ]; then + # reset array if installation is fresh + if [ "$install" ]; then + # just remove everything, because we don't know it + printf "php occ config:system:delete '%s' || true\n" "$2" + + # counter is zero for sure + counter=0 + + # else, default behaviour of the array + else + # save counter of the next free index + counter=$( awk -v FS=" = " -v name="$2" ' + BEGIN { counter = 0 } + split($1, header, "|") == 2 && header[1] ~ /^[[:digit:]]+$/ && header[2] == name \ + { if(counter < header[1]) counter = header[1] } + END { print counter + 1 } + ' "$__object/explorer/config" + ) + + # create a file which contains all lines not already resolved by this function + _dir="$__object/files/conf-arrays" + mkdir -p "$_dir" + grep "^[[:digit:]]*|$2 = " "$__object/explorer/config" > "$_dir/$2" || true # ignore not found + fi + + # iterate through every value + while read -r value; do + # check every value if he exists + if ! grep -q "^[[:digit:]]*|$2 = $value$" "$__object/explorer/config"; then + # add this value + printf "php occ config:system:set '%s' '%s' --type=string --value='%s'\n" \ + "$2" "$(( counter ))" "$value" + counter=$(( counter + 1 )) + fi + + if [ -z "$install" ]; then + # removes it from the list of unhandled values + grep -v "^[[:digit:]]*|$2 = $value$" "$_dir/$2" > "$_dir/$2_tmp" || true # ignore not found + mv "$_dir/$2_tmp" "$_dir/$2" # because we can't do `cat foo > foo` + fi + done < "$__object/parameter/$1" + + if [ -z "$install" ]; then + # interate through the leftover values + # remove them, as they should not exist (at least can be) + while read -r start equal value; do + # remove those specific elements from the array + printf "php occ config:system:delete '%s' '%s' --error-if-not-exists\n" \ + "$2" "$( printf "%s" "$start" | awk -F'|' '{print $1}' )" + done < "$_dir/$2" + fi + else + # remove everything because we don't know which was set by the user + if paramexist "$2"; then + # remove the whole array + printf "php occ config:system:delete '%s'\n" "$2" + fi + fi +} + +# Set the install variable if nextcloud was not installed before this type. +if ! testparam installed 1; then + install="yes" +fi + + +# Map all parameters + +# Generate the config changes + +# misc +conf_array host trusted_domains + +# Already set via the installer +if [ -z "$install" ]; then + # db + conf_string database-type dbtype + conf_string database-host dbhost # FIXME host included here (takes port also) + conf_string database-name dbname + conf_string database-user dbuser + conf_string database-password dbpassword + conf_string database-prefix dbtableprefix +fi diff --git a/type/__nextcloud/parameter/boolean b/type/__nextcloud/parameter/boolean new file mode 100644 index 0000000..0853f49 --- /dev/null +++ b/type/__nextcloud/parameter/boolean @@ -0,0 +1 @@ +install-only diff --git a/type/__nextcloud/parameter/default/database-type b/type/__nextcloud/parameter/default/database-type new file mode 100644 index 0000000..8b2f60c --- /dev/null +++ b/type/__nextcloud/parameter/default/database-type @@ -0,0 +1 @@ +sqlite3 diff --git a/type/__nextcloud/parameter/default/group b/type/__nextcloud/parameter/default/group new file mode 100644 index 0000000..5bbad18 --- /dev/null +++ b/type/__nextcloud/parameter/default/group @@ -0,0 +1 @@ +www-data diff --git a/type/__nextcloud/parameter/default/mode b/type/__nextcloud/parameter/default/mode new file mode 100644 index 0000000..20610ea --- /dev/null +++ b/type/__nextcloud/parameter/default/mode @@ -0,0 +1 @@ +755 diff --git a/type/__nextcloud/parameter/default/user b/type/__nextcloud/parameter/default/user new file mode 100644 index 0000000..5bbad18 --- /dev/null +++ b/type/__nextcloud/parameter/default/user @@ -0,0 +1 @@ +www-data diff --git a/type/__nextcloud/parameter/optional b/type/__nextcloud/parameter/optional new file mode 100644 index 0000000..cd09af4 --- /dev/null +++ b/type/__nextcloud/parameter/optional @@ -0,0 +1,13 @@ +mode +user +group +webroot +database-type +database-host +database-name +database-user +database-password +database-prefix +admin-user +admin-email +data-directory diff --git a/type/__nextcloud/parameter/optional_multiple b/type/__nextcloud/parameter/optional_multiple new file mode 100644 index 0000000..c70dc2d --- /dev/null +++ b/type/__nextcloud/parameter/optional_multiple @@ -0,0 +1 @@ +host diff --git a/type/__nextcloud/parameter/required b/type/__nextcloud/parameter/required new file mode 100644 index 0000000..3e83467 --- /dev/null +++ b/type/__nextcloud/parameter/required @@ -0,0 +1,2 @@ +version +admin-password From 0a0e4d26b4a6a765c556035725371ee0206a38ab Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 09:31:59 +0200 Subject: [PATCH 139/332] __nextcloud: fix shellcheck Mostly simply because of php inline code .. also because of the argument generation for the installation setup. --- type/__nextcloud/explorer/config | 5 +++-- type/__nextcloud/explorer/version | 3 ++- type/__nextcloud/gencode-remote | 7 +++++-- type/__nextcloud/map-conf-changes.sh | 4 ++++ 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/type/__nextcloud/explorer/config b/type/__nextcloud/explorer/config index 998264e..f97f39d 100755 --- a/type/__nextcloud/explorer/config +++ b/type/__nextcloud/explorer/config @@ -5,7 +5,6 @@ # Get the installdir -user="$( cat "$__object/parameter/user" )" installdir="$( "$__type_explorer/installdir" )" # Check if the tools are available @@ -14,9 +13,11 @@ if [ -d "$installdir" ]; then # if those files exist, everything should be good if [ -f "occ" ] && [ -f "config/config.php" ]; then - # dump out config instead of fuzz every possible option through + # Dump out config instead of fuzz every possible option through # `occ config:system:get`. Or parse through the whole json or # yaml-like output of `occ config:list system --private`. + # + # shellcheck disable=SC2016 # cause of the php inline code php -r 'require("lib/private/Config.php"); $config = new OC\Config("config/"); function printv($key, $value) {printf("%s = %s\n", $key, $value);} foreach($config->getKeys() as $key){ diff --git a/type/__nextcloud/explorer/version b/type/__nextcloud/explorer/version index fc79206..5a75258 100755 --- a/type/__nextcloud/explorer/version +++ b/type/__nextcloud/explorer/version @@ -13,7 +13,8 @@ if [ -d "$installdir" ]; then # if those files exist, everything should be good if [ -f "occ" ] && [ -f "version.php" ]; then - # detect php version with the version file + # Detect php version with the version file. + # shellcheck disable=SC2016 # cause of the php inline code php -r 'require("version.php"); print($OC_VersionString);' fi fi diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 60d9b42..e0317c1 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -91,6 +91,8 @@ fi # Check if the nextcloud application needs to be installed. # This checks the state of the configuration, not of the directory. +# +# shellcheck disable=SC2089 # disabled to write args string if ! grep -q -F "installed = 1" "$__object/explorer/config"; then # argument construction occ_install_args="" @@ -159,8 +161,9 @@ if ! grep -q -F "installed = 1" "$__object/explorer/config"; then fi - # Execute the install command - occ maintenance:install $occ_install_args + # Execute the install command. + # generated parameters will be splited in the remote shell + occ maintenance:install "$occ_install_args" # send install message echo installed >> "$__messages_out" diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index 8a00ac6..f107f5f 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -57,6 +57,7 @@ conf_base() { value="$(cat "$__object/parameter/$1")" if ! testparam "$2" "$value"; then # set it because it does not exist + # shellcheck disable=SC2059 # $3 contains patterns printf "php occ config:system:$3\n" "$2" "$value" fi else @@ -122,6 +123,7 @@ conf_array() { # else, default behaviour of the array else # save counter of the next free index + # shellcheck disable=SC1004 # the \ is required for awk counter=$( awk -v FS=" = " -v name="$2" ' BEGIN { counter = 0 } split($1, header, "|") == 2 && header[1] ~ /^[[:digit:]]+$/ && header[2] == name \ @@ -156,6 +158,8 @@ conf_array() { if [ -z "$install" ]; then # interate through the leftover values # remove them, as they should not exist (at least can be) + # + # shellcheck disable=SC2034 # $equal left for readability while read -r start equal value; do # remove those specific elements from the array printf "php occ config:system:delete '%s' '%s' --error-if-not-exists\n" \ From 3db4dcc6eab1b8f676f01a32c4bcde2d03685b34 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 11:58:16 +0200 Subject: [PATCH 140/332] __nextcloud: add --data-directory to config --- type/__nextcloud/gencode-remote | 2 +- type/__nextcloud/map-conf-changes.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index e0317c1..9da3b86 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -155,7 +155,7 @@ if ! grep -q -F "installed = 1" "$__object/explorer/config"; then fi # Data directory - datadir="$__object/parameter/datadir" + datadir="$__object/parameter/data-directory" if [ -f "$datadir" ]; then occ_install_args="$occ_install_args --data-dir '$(cat "$datadir")'" fi diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index f107f5f..ddda9c8 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -197,4 +197,7 @@ if [ -z "$install" ]; then conf_string database-user dbuser conf_string database-password dbpassword conf_string database-prefix dbtableprefix + + # data-dir + conf_string data-directory datadirectory fi From ea58f98ecb76e366eb8b01925f89b283596d9d4d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 13:36:26 +0200 Subject: [PATCH 141/332] __nextcloud: added initial manpage Still somehow a draft .. --- type/__nextcloud/man.rst | 173 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 type/__nextcloud/man.rst diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst new file mode 100644 index 0000000..96d8eac --- /dev/null +++ b/type/__nextcloud/man.rst @@ -0,0 +1,173 @@ +cdist-type__nextcloud(7) +======================== + +NAME +---- +cdist-type__nextcloud - Installs and manages a nextcloud instance + + +DESCRIPTION +----------- +This type installs, upgrades and configure a nextcloud instance. + +It installs the application in the webspace based on the object id as relative +path from the webroot. If you want to install it directly in the webroot, you +must trick out this type by setting the webroot as parent directory. + + +REQUIRED PARAMETERS +------------------- +version + The version that should be installed. If it is already installed and the + installed version lower, it will upgrade nextcloud if ``--install-only`` is + not set. + + The type will never downgrade a nextcloud instance. Rather, it will fail, + as this is a missconfiguration. Downgrades are not recommended and + supported by upstream. Such cases can happen if the nextcloud instance was + upgraded via the built-in nextcloud installer. In such cases, it is + recommended to use the ``--install-only`` option. + +admin-password + The administrator password to access the nextcloud instance. Must be given + in plain text. + + +OPTIONAL PARAMETERS +------------------- +webroot + The webroot which will be used as basis for the installation. This may be + already detected by an explorer. Must be an absolute path (starting with a + slash). + +mode + Sets the unix file mode of the nextcloud directory. This is not inherited + to child files or folders. Defaults to `755`. + +user + The user which owns the complete nextcloud directory. The php application + should be executed with this user. All nextcloud commands will be executed + with this user. This type will not create the unix user. + + The type assumes the default `www-data` user, which is common on Debian + systems. **If you change this option, please do the same with the group + parameter!** + +group + The group all files and folders of the nextcloud installation should have. + Defaults to `www-data`. Should be changed with ``--user``. + + +BOOLEAN PARAMETERS +------------------ +install-only + Skips all nextcloud upgrades done by this type. Should be used when + nextcloud upgrades are (*exclusively*) done via the built-in updater. + + +NEXTCLOUD CONFIG PARAMETERS +--------------------------- +host + All hostnames where the the users can log into nextcloud. If you access + nextcloud via a hostname not given to this list, the access fails. This + parameter can be set multiple times. + +admin-user + The username of the administrative user which will be created while the + installation. This parameter has no effect if nextcloud will not be + installed. + +admin-email + The email address of the administrative user. This parameter has no effect + if nextcloud will not be installed. + +database-type + Sets the type of database that should be used as backend. Possible backends + are: + + SQLite + Use ``sqlite`` or ``sqlite3``. Saves everything in a database file + stored in the data directory. It is only recommended for very small + installations or test environments from upstream. + + *All further database options are ignored if SQLite is selected as + database backend.* + + MariaDB + Use ``mysql`` or ``mariadb``. MariaDB and MySQL are threated the same + way. They are the recommended database backends recommended from + upstream. + + PostgreSQL + Use ``pgsql``, ``postgres`` or ``postgresql``. + + **This parameter defaults to the SQLite database backend, as it is the + simplest one to setup and do not require extra parameters.** + +database-host + The database host to connect to. Possible are hostnames, ip addresses or + UNIX sockets. UNIX sockets must set in the format of + ``localhost:/path/to/socket``. If an non-standard port is used, set it + after the hostname or ip address seperated by an colon (``:``). + +database-name + The name of the database to connect to. + +database-user + The username to access the database. + +database-password + The password required to authorize the given user. + +database-prefix + The table prefix used by nextcloud. If nothing set, it defaults to + ``oc_``. + + +WEBROOT DETECTION +----------------- +TBA. + + +MESSAGES +-------- +installed + Nextcloud was successfully installed. + +upgraded $old to $new + The nextcloud version was upgraded from `$old` to `$new`. + +configured + Nextcloud configuration was changed. + + +ABORTS +------ +Aborts in the following cases: + +The current installed version is greather than the version that should be +installed. See the parameter description of `--version` for detailed +information. The problem can be fixed by bumping the version value to at least +the version that is currently installed or use the parameter `--install-only`. + +The type aborts if there is no webroot given as parameter and no could be +detected by the type itself. Please set the webroot via `--webroot` or extend +this type. + + +EXAMPLES +-------- +TBA. + + +AUTHORS +------- +Matthias Stecher + + +COPYRIGHT +--------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. From b6f4da7692e5b13bdc8f5990189b4ff2fe88b1b4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 15:39:54 +0200 Subject: [PATCH 142/332] __nextcloud: completly respect --install-only parameter This commit changes the manifest to do not check for too low versions and does not download and unpack a tarball if available if the parameter --install-only is set. This should make it useable now. Also, a short message change to match the manual. --- type/__nextcloud/gencode-remote | 2 +- type/__nextcloud/manifest | 34 +++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 9da3b86..0a68a02 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -187,7 +187,7 @@ if [ -s "$__object/files/conf-cmds" ]; then printf "SU\n" # print a message - echo config >> "$__messages_out" + echo configured >> "$__messages_out" fi diff --git a/type/__nextcloud/manifest b/type/__nextcloud/manifest index f6b8865..2d89f01 100755 --- a/type/__nextcloud/manifest +++ b/type/__nextcloud/manifest @@ -94,27 +94,33 @@ nextcloud_uri="https://download.nextcloud.com/server/releases/nextcloud-${versio nextcloud_sum="${nextcloud_uri}.sha256" -# Only check if there is a current installation +# Check if there is a current installation. It depends where the upstream +# tarball should be unpacked (directly or moved in a later stage). if [ "$version_is" ]; then - # Block downgrades as there are may caused from the automatic upgrader - # if the current version is higher than the version that should be installed - if version_ge "$version_is" "$version_should"; then - # it's an error if the current version is higher than the one that should be installed - printf "The current nextcloud version '%s' is higher than the version that should be installed (%s)\n" \ - "$version_is" "$version_should" >&2 - printf "Please bump the nextcloud version to '%s' or higher!\n" "$version_is" >&2 - exit 2 - fi + # Only set and check the version if a upgrade is allowed. + # if this block will be skipped, no upgrade will be done + if ! [ -f "$__object/parameter/install-only" ]; then + # Block downgrades as there are may caused from the automatic upgrader + # if the current version is higher than the version that should be installed + if version_ge "$version_is" "$version_should"; then + # it's an error if the current version is higher than the one that should be installed + printf "The current nextcloud version '%s' is higher than the version that should be installed (%s)\n" \ + "$version_is" "$version_should" >&2 + printf "Please bump the nextcloud version to '%s' or higher!\n" "$version_is" >&2 + exit 2 + fi - # Set destination to a temporary directory - destination="$webroot/.$__object_id" + # Set destination to a temporary directory + destination="$webroot/.$__object_id" + fi else # Set destination to the real destination destination="$webroot/$__object_id" fi -# Upgrade the nextcloud version -if [ "$version_is" != "$version_should" ]; then +# Install/Upgrade the nextcloud version if there is a destination set. +# it checks if it is necessary and intended to upgrade +if [ "$destination" ] && [ "$version_is" != "$version_should" ]; then updatedir="$( dirname "$installdir" )" # Be sure the parent directory exists for installation From f5c988c0f2f614001d39c52f44193b6ce853cd3a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 16:46:19 +0200 Subject: [PATCH 143/332] __nextcloud: make some db parameters mandatory Some database parameters are required if the type is `mysql` or `pgsql`. Also, the variants of the parameter --database-type decreased to those that work with the configuration, as it is directly mapped to the parameter. The functions in `map-conf-changes.sh` where extended to support the conditional mandatory argument to throw an error if this parameter is set. --- type/__nextcloud/gencode-remote | 37 ++++++++++--------- type/__nextcloud/man.rst | 23 +++++++----- type/__nextcloud/map-conf-changes.sh | 55 +++++++++++++++++++++------- 3 files changed, 75 insertions(+), 40 deletions(-) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 0a68a02..9ffbc4a 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -97,27 +97,28 @@ if ! grep -q -F "installed = 1" "$__object/explorer/config"; then # argument construction occ_install_args="" - # Database + # Error function if value not found + die_err() { + echo "parameter not found but required; can't continue!!" >&2 + exit 1 + } + # Database setup for mysql and pgsql db_setup() { if ! [ -f "$__object/parameter/db-host" ]; then echo "no hostname given! can't proceed." >&2 exit 3 fi occ_install_args="$occ_install_args --database '$1'" - occ_install_args="$occ_install_args --database-host '$(cat "$__object/parameter/db-host")'" - db_name="$__object/parameter/database-name" - if [ -f "$db_name" ]; then - occ_install_args="$occ_install_args --database-name '$(cat "$db_name")'" - fi - db_user="$__object/parameter/database-user" - if [ -f "$db_user" ]; then - occ_install_args="$occ_install_args --database-user '$(cat "$db_user")'" - fi - db_pass="$__object/parameter/database-password" - if [ -f "$db_pass" ]; then - occ_install_args="$occ_install_args --database-pass '$(cat "$db_pass")'" + db_host="$__object/parameter/database-host" + if [ -f "$db_host" ]; then + occ_install_args="$occ_install_args --database-host '$(cat "$db_host")'" fi + + occ_install_args="$occ_install_args --database-name '$(cat "$__object/parameter/database-name" || die_err)'" + occ_install_args="$occ_install_args --database-user '$(cat "$__object/parameter/database-user" || die_err)'" + occ_install_args="$occ_install_args --database-pass '$(cat "$__object/parameter/database-password" || die_err)'" + db_prefix="$__object/parameter/database-prefix" if [ -f "$db_prefix" ]; then occ_install_args="$occ_install_args --database-table-prefix '$(cat "$db_prefix")'" @@ -126,18 +127,18 @@ if ! grep -q -F "installed = 1" "$__object/explorer/config"; then database_type="$(cat "$__object/parameter/database-type")" case "$database_type" in - sqlite|sqlite3) + sqlite3) occ_install_args="$occ_install_args --database sqlite" ;; - mysql|mariadb) + mysql) db_setup mysql ;; - pgsql|postgres|postgresql) + pgsql) db_setup pgsql ;; *) - printf "Database type '%s' is unkown!\n" "" >&2 + printf "Database type '%s' is unkown!\n" "$database_type" >&2 exit 3 ;; esac @@ -197,7 +198,7 @@ if [ "$install" ]; then # variable accessible from the last $install if-clause case "$database_type" in - mysql|mariadb) + mysql) # only available for mysql occ db:convert-mysql-charset ;; diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst index 96d8eac..e958a82 100644 --- a/type/__nextcloud/man.rst +++ b/type/__nextcloud/man.rst @@ -74,8 +74,8 @@ host admin-user The username of the administrative user which will be created while the - installation. This parameter has no effect if nextcloud will not be - installed. + installation. If not set, nextcloud defaults to "admin". This parameter has + no effect if nextcloud will not be installed. admin-email The email address of the administrative user. This parameter has no effect @@ -86,7 +86,7 @@ database-type are: SQLite - Use ``sqlite`` or ``sqlite3``. Saves everything in a database file + Use ``sqlite3`` as value. Saves everything in a database file stored in the data directory. It is only recommended for very small installations or test environments from upstream. @@ -94,12 +94,12 @@ database-type database backend.* MariaDB - Use ``mysql`` or ``mariadb``. MariaDB and MySQL are threated the same + Use ``mysql`` as value. MariaDB and MySQL are threated the same way. They are the recommended database backends recommended from upstream. PostgreSQL - Use ``pgsql``, ``postgres`` or ``postgresql``. + Use ``pgsql`` as value. **This parameter defaults to the SQLite database backend, as it is the simplest one to setup and do not require extra parameters.** @@ -110,17 +110,22 @@ database-host ``localhost:/path/to/socket``. If an non-standard port is used, set it after the hostname or ip address seperated by an colon (``:``). + If this value is not set, nextcloud defaults to the value ``localhost``. + database-name - The name of the database to connect to. + The name of the database to connect to. Required if MariaDB or PostgreSQL + is used. database-user - The username to access the database. + The username to access the database. Required if MariaDB or PostgreSQL is + used. database-password - The password required to authorize the given user. + The password required to authorize the given user. Required if MariaDB or + PostgreSQL is used. database-prefix - The table prefix used by nextcloud. If nothing set, it defaults to + The table prefix used by nextcloud. If nothing set, nextcloud defaults to ``oc_``. diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index ddda9c8..791edc7 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -51,16 +51,23 @@ paramexist() { # Arguments: # 1: cdist type parameter name # 2: nextcloud config name -# 3: occ printf pattern to set the value +# 3: conditially mandatory argument, value "required" if true +# 4: occ printf pattern to set the value conf_base() { if [ -f "$__object/parameter/$1" ]; then value="$(cat "$__object/parameter/$1")" if ! testparam "$2" "$value"; then # set it because it does not exist - # shellcheck disable=SC2059 # $3 contains patterns - printf "php occ config:system:$3\n" "$2" "$value" + # shellcheck disable=SC2059 # $4 contains patterns + printf "php occ config:system:$4\n" "$2" "$value" fi else + if [ "$3" = "required" ]; then + # error because the parameter should be set + printf "Parameter '%s' not set by user, but required!\n" "$1" >&2 + exit 4 + fi + if paramexist "$2"; then # remove it because it exists printf "php occ config:system:delete '%s'\n" "$2" @@ -73,14 +80,15 @@ conf_base() { # Arguments: # 1: cdist type parameter name # 2: nextcloud config name +# 3: conditional mandatory of this parameter; value "required" if true conf_string() { - conf_base "$1" "$2" "set '%s' --type=string --value='%s'" + conf_base "$1" "$2" "$3" "set '%s' --type=string --value='%s'" } conf_number() { - conf_base "$1" "$2" "set '%s' --type=integer --value='%s'" + conf_base "$1" "$2" "$3" "set '%s' --type=integer --value='%s'" } conf_decimal() { - conf_base "$1" "$2" "set '%s' --type=double --value='%s'" + conf_base "$1" "$2" "$3" "set '%s' --type=double --value='%s'" } # Sets the nextcloud configuration option after a boolean cdist parameter. @@ -110,6 +118,7 @@ conf_boolean() { # Arguments: # 1: cdist type parameter name # 2: nextcloud config name +# 3: conditional mandatory of this parameter; value "required" if true conf_array() { if [ -f "$__object/parameter/$1" ]; then # reset array if installation is fresh @@ -167,6 +176,12 @@ conf_array() { done < "$_dir/$2" fi else + if [ "$3" = "required" ]; then + # error because the parameter should be set + printf "Parameter '%s' not set by user, but required!\n" "$1" >&2 + exit 4 + fi + # remove everything because we don't know which was set by the user if paramexist "$2"; then # remove the whole array @@ -190,13 +205,27 @@ conf_array host trusted_domains # Already set via the installer if [ -z "$install" ]; then - # db - conf_string database-type dbtype - conf_string database-host dbhost # FIXME host included here (takes port also) - conf_string database-name dbname - conf_string database-user dbuser - conf_string database-password dbpassword - conf_string database-prefix dbtableprefix + # database + database_type="$(cat "$__object/parameter/database-type")" + case "$database_type" in + sqlite3) + conf_string database-type dbtype + ;; + + mysql|pgsql) + conf_string database-type dbtype + conf_string database-host dbhost + conf_string database-name dbname required + conf_string database-user dbuser required + conf_string database-password dbpassword required + conf_string database-prefix dbtableprefix + ;; + + *) + printf "Databasetype '%s' is unkown!\n" "$database_type" >&2 + exit 3 + ;; + esac # data-dir conf_string data-directory datadirectory From 3bda4cf0c8f556be659d0e0d0cf72bf1ee06e673 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 17:29:38 +0200 Subject: [PATCH 144/332] __nextcloud: respect installer default values To do not override default values from the nextcloud installer script, they are accepted as default values if the user did not set any value. --- type/__nextcloud/map-conf-changes.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index 791edc7..7d11728 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -53,9 +53,10 @@ paramexist() { # 2: nextcloud config name # 3: conditially mandatory argument, value "required" if true # 4: occ printf pattern to set the value +# 5: "installation" default value, can be used to backup the user value conf_base() { - if [ -f "$__object/parameter/$1" ]; then - value="$(cat "$__object/parameter/$1")" + if [ -f "$__object/parameter/$1" ] || [ "$5" ]; then + value="$(cat "$__object/parameter/$1" || printf "%s" "$5")" if ! testparam "$2" "$value"; then # set it because it does not exist # shellcheck disable=SC2059 # $4 contains patterns @@ -81,14 +82,15 @@ conf_base() { # 1: cdist type parameter name # 2: nextcloud config name # 3: conditional mandatory of this parameter; value "required" if true +# 4: default value; will be used if parameter is absent conf_string() { - conf_base "$1" "$2" "$3" "set '%s' --type=string --value='%s'" + conf_base "$1" "$2" "$3" "$4" "set '%s' --type=string --value='%s'" } conf_number() { - conf_base "$1" "$2" "$3" "set '%s' --type=integer --value='%s'" + conf_base "$1" "$2" "$3" "$4" "set '%s' --type=integer --value='%s'" } conf_decimal() { - conf_base "$1" "$2" "$3" "set '%s' --type=double --value='%s'" + conf_base "$1" "$2" "$3" "$4" "set '%s' --type=double --value='%s'" } # Sets the nextcloud configuration option after a boolean cdist parameter. @@ -96,6 +98,7 @@ conf_decimal() { # Arguments: # 1: cdist type parameter name # 2: nextcloud config name +# FIXME default value required for booleans? conf_boolean() { # map parameter to a php boolean (are outputted as 0 or 1) if [ -f "$__object/parameter/$1" ]; then @@ -119,6 +122,7 @@ conf_boolean() { # 1: cdist type parameter name # 2: nextcloud config name # 3: conditional mandatory of this parameter; value "required" if true +# FIXME currently no default value due to complexity of arrays conf_array() { if [ -f "$__object/parameter/$1" ]; then # reset array if installation is fresh @@ -204,6 +208,7 @@ fi conf_array host trusted_domains # Already set via the installer +# set default values from the nextcloud installer to do not override them if [ -z "$install" ]; then # database database_type="$(cat "$__object/parameter/database-type")" @@ -214,7 +219,7 @@ if [ -z "$install" ]; then mysql|pgsql) conf_string database-type dbtype - conf_string database-host dbhost + conf_string database-host dbhost installdef "localhost" conf_string database-name dbname required conf_string database-user dbuser required conf_string database-password dbpassword required @@ -228,5 +233,5 @@ if [ -z "$install" ]; then esac # data-dir - conf_string data-directory datadirectory + conf_string data-directory datadirectory installdef "$(cat "$__object/explorer/installdir")/$__object_id/data" fi From facb204749d47468e544e81649313079a713e004 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 24 Oct 2020 21:09:35 +0200 Subject: [PATCH 145/332] __nextcloud: move data directory if destination changes The type now handles the case when the data directory will be moved to an other location. To this, adjustment to the "maintainer mode" on setting and the manpage. --- type/__nextcloud/gencode-remote | 54 ++++++++++++++++++++++++++-- type/__nextcloud/man.rst | 14 ++++++++ type/__nextcloud/map-conf-changes.sh | 10 +++--- 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 9ffbc4a..38024a6 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -22,6 +22,28 @@ SU SHELL } +# Turn the maintainer mode on, but print it only once at all. +# +# No arguments. +occ_maintainer_mode_on() { + # Check if this was not already done + if [ "$_maintainer_mode_on" != "yes" ]; then + occ maintenance:mode --on + _maintainer_mode_on="yes" + fi +} + +# Print the value of the given configuration. +# +# Arguments: +# 1: the nextcloud configuration name +getparam() { + awk -v FS=" = " -v name="$1" ' + function ntostring(n) { ret=""; for(i=n; i<=NF; i++) ret=ret $i (i/dev/null || printf "%s/data" "$installdir")" + +# Move if they should be moved. Avoid false positives if $data_old is empty +if [ "$data_old" ] && [ "$data_old" != "$data_new" ]; then + # save that we did changes + changes="yes" + occ_maintainer_mode_on + + # Change the configuration variable and then move the folder. This order is + # important if SQLite is used, but the config already corrupted if it can + # not be moved. + occ config:system:set datadirectory --type=string --value "'$data_new'" + cat << REMOTE +rm -rf '$data_new' +mkdir -p '$(dirname "$data_new")' # if the parent not exists +mv '$data_old' '$data_new' + +REMOTE +fi + +# Print configured message if changes where done to the configuration +if [ "$changes" ]; then echo configured >> "$__messages_out" fi @@ -208,6 +256,6 @@ if [ "$install" ]; then fi # Disable maintainer mode -if [ "$install" ] || [ "$upgrade" ] || [ "$changes" ]; then +if [ "$_maintainer_mode_on" = "yes" ]; then occ maintenance:mode --off fi diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst index e958a82..2bff614 100644 --- a/type/__nextcloud/man.rst +++ b/type/__nextcloud/man.rst @@ -81,6 +81,15 @@ admin-email The email address of the administrative user. This parameter has no effect if nextcloud will not be installed. +data-directory + This will set or change the data directory where nextcloud will keep all + its data, including the SQLite database if any. By default, it will be + saved in the ``data`` directory below the nextcloud directory. + + If this directory change, this type will move the old location to the new + one to preserve all data. This is not supported by upstream, as some apps + may not handle this. + database-type Sets the type of database that should be used as backend. Possible backends are: @@ -159,6 +168,11 @@ The type aborts if there is no webroot given as parameter and no could be detected by the type itself. Please set the webroot via `--webroot` or extend this type. +It may abort if the data directory can not be moved correctly. Then, the +nextcloud configuration is broken and must be resolved manually: Move the data +directory to the correct location or change the configuration to point to the +old destination and retry. + EXAMPLES -------- diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index 7d11728..caea3bd 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -84,13 +84,13 @@ conf_base() { # 3: conditional mandatory of this parameter; value "required" if true # 4: default value; will be used if parameter is absent conf_string() { - conf_base "$1" "$2" "$3" "$4" "set '%s' --type=string --value='%s'" + conf_base "$1" "$2" "$3" "set '%s' --type=string --value='%s'" "$4" } conf_number() { - conf_base "$1" "$2" "$3" "$4" "set '%s' --type=integer --value='%s'" + conf_base "$1" "$2" "$3" "set '%s' --type=integer --value='%s'" "$4" } conf_decimal() { - conf_base "$1" "$2" "$3" "$4" "set '%s' --type=double --value='%s'" + conf_base "$1" "$2" "$3" "set '%s' --type=double --value='%s'" "$4" } # Sets the nextcloud configuration option after a boolean cdist parameter. @@ -232,6 +232,6 @@ if [ -z "$install" ]; then ;; esac - # data-dir - conf_string data-directory datadirectory installdef "$(cat "$__object/explorer/installdir")/$__object_id/data" + # data-dir is handled in the gencode-remote + #conf_string data-directory datadirectory installdef "$(cat "$__object/explorer/installdir")/$__object_id/data" fi From 809ad7aa9d63ca7d21375874660fe56385938bfc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 25 Oct 2020 14:55:11 +0100 Subject: [PATCH 146/332] __nextcloud: migrate if the database type change This adds a migration progess if the database type changes automaticly. --- type/__nextcloud/man.rst | 16 +++++- type/__nextcloud/map-conf-changes.sh | 81 +++++++++++++++++++++++++--- 2 files changed, 89 insertions(+), 8 deletions(-) diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst index 2bff614..2a6ec9d 100644 --- a/type/__nextcloud/man.rst +++ b/type/__nextcloud/man.rst @@ -113,13 +113,22 @@ database-type **This parameter defaults to the SQLite database backend, as it is the simplest one to setup and do not require extra parameters.** + If this parameter change, the type will migrate to the new database type. + It will not work for SQLite because the upstream migration script does not + support it. **Be aware that migrations take there time, plan at minimum + 40 seconds of migration for a stock installation.** + database-host The database host to connect to. Possible are hostnames, ip addresses or UNIX sockets. UNIX sockets must set in the format of ``localhost:/path/to/socket``. If an non-standard port is used, set it - after the hostname or ip address seperated by an colon (``:``). + after the hostname or ip address seperated by an colon (``:``). If this + value is not set, nextcloud defaults to the value ``localhost``. - If this value is not set, nextcloud defaults to the value ``localhost``. + This type will not migrate data if the type does not change. You must do + this manually by setting the maintainer mode (to avoid data changes) and + then cloning the database to the new destination. After that, run cdist to + apply the config changes. It should automaticly remove the maintainer mode. database-name The name of the database to connect to. Required if MariaDB or PostgreSQL @@ -173,6 +182,9 @@ nextcloud configuration is broken and must be resolved manually: Move the data directory to the correct location or change the configuration to point to the old destination and retry. +It aborts if it should migrate to a SQLite database. This will be done before +the upstream migration script is executed, as it would throw the same error. + EXAMPLES -------- diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index caea3bd..3c98c3b 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -7,6 +7,21 @@ # explorer. +# Print the value of the given configuration. +# +# Arguments: +# 1: the nextcloud configuration name +# +# Returns with a unsuccessful return code if no parameter found. +getparam() { + awk -v FS=" = " -v name="$1" ' + function ntostring(n) { ret=""; for(i=n; i<=NF; i++) ret=ret $i (i/dev/null || printf "localhost")" + database_name="$(cat "$__object/parameter/database-name")" + database_user="$(cat "$__object/parameter/database-user")" + database_pass="$(cat "$__object/parameter/database-password")" + + # Extract the port from the host + # this is required for pgsql, but mysql can do it itself, too + if printf "%s" "$database_host" | grep -q ":[[:digit:]]\+$"; then + # extract the last part, which is the port number + database_port="${database_host##*:}" + else + # set default port because the tool can not do this for pgsql + case "$database_type" in + mysql) + database_port=3306 + ;; + pgsql) + database_port=5432 + ;; + esac + fi + + # print out the correct command + printf "php occ db:convert-type --no-interaction --no-ansi --clear-schema --all-apps \ + '%s' '%s' --password '%s' '%s' --port '%u' '%s'\n" \ + "$database_type" "$database_user" "$database_pass" "$database_host" "$database_port" "$database_name" +} + + # Set the install variable if nextcloud was not installed before this type. if ! testparam installed 1; then install="yes" @@ -210,19 +263,35 @@ conf_array host trusted_domains # Already set via the installer # set default values from the nextcloud installer to do not override them if [ -z "$install" ]; then - # database + # Database to check if the type changed + # use the current type if no old type found to match instead of migrate database_type="$(cat "$__object/parameter/database-type")" + old_db_type="$(getparam dbtype || printf "%s" "$database_type")" + case "$database_type" in sqlite3) + if [ "$old_db_type" != "sqlite3" ]; then + echo "Migrating to a SQLite database is not supported by upstream!" >&2 + echo "Do it manually or reinstall nextcloud .." >&2 + exit 1 + fi conf_string database-type dbtype ;; mysql|pgsql) - conf_string database-type dbtype - conf_string database-host dbhost installdef "localhost" - conf_string database-name dbname required - conf_string database-user dbuser required - conf_string database-password dbpassword required + if [ "$old_db_type" != "$database_type" ]; then + # the migration will change all database parameters itself + migrate_db "$database_type" + else + # no change of dbtype cause it will cause a migration + conf_string database-host dbhost installdef "localhost" + conf_string database-name dbname required + conf_string database-user dbuser required + conf_string database-password dbpassword required + fi + + # It may not be a good idea to change this parameter, but do what + # the user want to do. conf_string database-prefix dbtableprefix ;; From 77b530eefb59d844eaf45fc0998cb35d58802e74 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 25 Oct 2020 18:57:02 +0100 Subject: [PATCH 147/332] __nextcloud: update manpage Added webroot detection, examples and notes. Should be complete now. --- type/__nextcloud/man.rst | 69 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst index 2a6ec9d..ba4d8c4 100644 --- a/type/__nextcloud/man.rst +++ b/type/__nextcloud/man.rst @@ -149,7 +149,14 @@ database-prefix WEBROOT DETECTION ----------------- -TBA. +As the `object id` is the install path relatively from the webroot, it must be +known somehow. Therefor, it will try to detect a good location for it. You can +set a custom webroot via the `--webroot` parameter. As default, following +directories will be checked if they exist to be the webroot: + +1. ``/srv/www/`` +2. ``/var/www/html/`` +3. ``/var/www/`` MESSAGES @@ -185,10 +192,68 @@ old destination and retry. It aborts if it should migrate to a SQLite database. This will be done before the upstream migration script is executed, as it would throw the same error. +The explorers will abort if they found a valid nextcloud installation, but no +installed `php`. Currently, this is intended behaviour, because it can not +safely get the current nextcloud version, also do not get the nextcloud +configuration. For more information, see the *NOTES section*. + EXAMPLES -------- -TBA. + +.. code-block:: sh + + # minimal nextcloud installation with sqlite and other defaults + # please only use sqlite for minimal or test installations as recommend :) + __nextcloud nextcloud --version 20.0.0 --admin-password "iaminsecure" \ + --host localhost --host nextcloud + + # more extensive configuration + __nextcloud cloud --version 20.0.0 --admin-password "iaminsecure" \ + --host localhost --host nextcloud --host 192.168.1.67 \ + --data-directory /var/lib/nextcloud/what \ + --database-type mysql --database-host "localhost" --database-name "nextcloud" \ + --database-user "test" --database-password "not-a-good-password" + + # install it in the webroot /var/www/html + __nextcloud html --version 20.0.0 --admin-password "notthatsecure" \ + --webroot "/var/www" --host localhost + + +NOTES +----- +This cdist type does not cover all configuration options that nextcloud offer. +If you need more configuration options for nextcloud, you are welcome to extend +this type and contribute it upstream! + +- `Nextcloud configuration reference + `_ + +Database migration is only partly supported if the database will be changed to +``mysql` or ``pgsql``, because it is supported by an upstream script. You are +welcome to extend this type for database migrations between the same database +type. For an implementation, you may use shell utilites like ``mysqldump(1)`` +(be aware that this may not already be installed) or use the already installed +php code to migrate. + +The type will abort if a valid nextcloud directory already exists in the +explorer execution, but no `php` exists to explore the setup. Therefor, the +manifest could not install `php` yet. This is not the case for a new +installation, as there does not exist a nextcloud directory with a valid +structure. While some code could be skipped and the other replaced with `awk` +with something like +``awk '$1 == "$OC_VersionString" {gsub(/['\'';]/, "", $3); print $3}' version.php``, +it is not handled for the following cases: + +1. This case should not happen very often. +2. Maybe because of ``libapache2-mod-php`` or ``php-fpm``, `php` already + exists for the cli. +3. While the `awk` replacement for the version is just a bit worser, it would + bring stable results, while it would be more difficult to dump out the + configuration without custom `php` or the help from ``php occ``. Therefor, + it would make false assumptions like it want to install nextcloud again, + do not delete configuration options and set all available nextcloud options + that are available through this type. AUTHORS From 87e115da98425c69d8bb37397ccdf47c0f2a4dc2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 25 Oct 2020 19:13:10 +0100 Subject: [PATCH 148/332] __netbox: some manpage fixes Mostly invalid links .. --- type/__netbox/man.rst | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index f34ead7..135304c 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -100,8 +100,7 @@ redis-password redis-dbid-offset Offset to set the redis database id's. The `tasks` database id is - `offset + 0` and `caching` is `offset + 1`. The offset defaults - to ``0``. + `offset + 0` and `caching` is `offset + 1`. The offset defaults to ``0``. smtp-host Host of the SMTP email server. Defaults to ``localhost``. @@ -172,12 +171,12 @@ redis-ssl smtp-use-tls Uses TLS to connect to the SMTP email server. `See documentation - `_ for more information. smtp-use-ssl Uses implicit TLS with the SMTP email server. `See documentation - `_ for more information. login-required @@ -229,16 +228,16 @@ is a more good way to inject python code for dynamic configuration variables, you are welcome to contribute! - `Possible optional settings - ` + `_ If you not setup ldap authentification, you may be interested into how to `setting up a super user -` +`_ directly on the machine to be able to access and use NetBox. You may also be interested of writing a own type which handles the creation of the super user. To do this non-interactivly, see the ansible role as `reference -`. +`_. If you change the secret key while the netbox instance is running, there is a time frame where the access to the application corrupts the whole database. From b368102bd58fd40ba9850ec866667019d3e5e5f3 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 25 Oct 2020 20:29:57 +0100 Subject: [PATCH 149/332] __nextcloud: misc adjustments Mostly to the manpage, but some minor fixes, too. --- type/__nextcloud/gencode-remote | 2 +- type/__nextcloud/man.rst | 13 ++++++++++++- type/__nextcloud/map-conf-changes.sh | 2 ++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 38024a6..1850dd0 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -17,7 +17,7 @@ occ() { # will not use -q as it supresses errors, too cat << SHELL su -s /bin/sh -l "$user" -- -e <`_ or from the `GitHub Releases + `_ page. The type will + download the tarball over the official nextcloud website. + The type will never downgrade a nextcloud instance. Rather, it will fail, as this is a missconfiguration. Downgrades are not recommended and supported by upstream. Such cases can happen if the nextcloud instance was @@ -255,13 +260,19 @@ it is not handled for the following cases: do not delete configuration options and set all available nextcloud options that are available through this type. +If the nextcloud installation does not work and you stuck in a plaintext error +screen, try to restart your Apache WWW server first! This type will install all +php dependencies, but there are not recognised by the server-internal php +environment. This can happen after a database migration between different +database types, as it installs the database module only when it is required. + AUTHORS ------- Matthias Stecher -COPYRIGHT +COPYING --------- Copyright \(C) 2020 Matthias Stecher. You can redistribute it and/or modify it under the terms of the GNU General Public License as diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index 3c98c3b..caad3bd 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -230,6 +230,7 @@ migrate_db() { database_port="${database_host##*:}" else # set default port because the tool can not do this for pgsql + # it looks like mysql get struggles, too case "$database_type" in mysql) database_port=3306 @@ -244,6 +245,7 @@ migrate_db() { printf "php occ db:convert-type --no-interaction --no-ansi --clear-schema --all-apps \ '%s' '%s' --password '%s' '%s' --port '%u' '%s'\n" \ "$database_type" "$database_user" "$database_pass" "$database_host" "$database_port" "$database_name" + printf "php occ maintenance:mode --on\n" # was disabled by database convertion } From 072ca3772047549bfb435ba521ef1d4677d0dc27 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 26 Oct 2020 19:19:49 +0100 Subject: [PATCH 150/332] __nextcloud: stricter check configuration values This now checks the configuration value exactly and does not say there are equal if just the end of the string differs. Resulting from this, it will run code the next run if the default port was in a database migration. This was fixed by instantly correct this configuration value. --- type/__nextcloud/map-conf-changes.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index caad3bd..5855761 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -35,7 +35,7 @@ testparam() { # short-circuit after installation; the explorer may not be valid if [ "$install" ]; then return 1; fi - if grep -q -F "$1 = $2" "$__object/explorer/config"; then + if grep -q -Fx "$1 = $2" "$__object/explorer/config"; then return 0 else return 1 @@ -113,7 +113,6 @@ conf_decimal() { # Arguments: # 1: cdist type parameter name # 2: nextcloud config name -# FIXME default value required for booleans? conf_boolean() { # map parameter to a php boolean (are outputted as 0 or 1) if [ -f "$__object/parameter/$1" ]; then @@ -239,6 +238,10 @@ migrate_db() { database_port=5432 ;; esac + + # Correct this value to the value set by the parameter + # this will prevent codegen in the run after the migration + correct_standard_port="yes" fi # print out the correct command @@ -246,6 +249,11 @@ migrate_db() { '%s' '%s' --password '%s' '%s' --port '%u' '%s'\n" \ "$database_type" "$database_user" "$database_pass" "$database_host" "$database_port" "$database_name" printf "php occ maintenance:mode --on\n" # was disabled by database convertion + + # Correct the database host value if it was not correctly set by the migration script + if [ "$correct_standard_port" = "yes" ]; then + printf "php occ config:system:set '%s' --type=string --value '%s'\n" "dbhost" "$database_host" + fi } From b0ed6b2f18f3b9ed4a25468eaaa71fa7b0f0c67d Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 30 Oct 2020 19:14:24 +0100 Subject: [PATCH 151/332] Initial __nextcloud_user type This type setup and updates a nextcloud user account. Currently not finished yet, but should work in all cases. --- type/__nextcloud_user/explorer/password | 49 ++++ type/__nextcloud_user/explorer/user | 39 +++ type/__nextcloud_user/gencode-remote | 239 ++++++++++++++++++ type/__nextcloud_user/parameter/default/state | 1 + .../parameter/default/www-user | 1 + type/__nextcloud_user/parameter/optional | 7 + .../parameter/optional_multiple | 1 + type/__nextcloud_user/parameter/required | 1 + 8 files changed, 338 insertions(+) create mode 100755 type/__nextcloud_user/explorer/password create mode 100755 type/__nextcloud_user/explorer/user create mode 100644 type/__nextcloud_user/gencode-remote create mode 100644 type/__nextcloud_user/parameter/default/state create mode 100644 type/__nextcloud_user/parameter/default/www-user create mode 100644 type/__nextcloud_user/parameter/optional create mode 100644 type/__nextcloud_user/parameter/optional_multiple create mode 100644 type/__nextcloud_user/parameter/required diff --git a/type/__nextcloud_user/explorer/password b/type/__nextcloud_user/explorer/password new file mode 100755 index 0000000..88260ec --- /dev/null +++ b/type/__nextcloud_user/explorer/password @@ -0,0 +1,49 @@ +#!/bin/sh +# __nextcloud/explorer/password + +# Checks if the given password is working by hacking somehow into the nextcloud +# php libary. +# +# Outputs: +# - "noop" if no password given as parameter +# - "matched" if the given parameter matched the password +# - "mismatched" if the given parameter did not matched +# - "" if no nextcloud directory could be detected + + +# Check if the password exists, else this is nonsense +password="$__object/parameter/password" +if [ -f "$password" ]; then + password="$(cat "$password")" +else + # no password to compare - it's managed by someone other + echo noop + exit +fi + +# Get parameters +user="$__object/parameter/user" +if [ -f "$user" ]; then + user="$(cat "$user")" +else + user="$__object_id" +fi +cloud="$(cat "$__object/parameter/cloud")" +www_user="$(cat "$__object/parameter/www-user")" + + + +# Check if there exists the installation +if [ -d "$cloud" ]; then + # if those files exist, everything should be good + if [ -f "$cloud/occ" ] && [ -f "$cloud/config/config.php" ]; then + # Output the information from the custom php + # change the user to be on the safe side if something is written + su -s /bin/sh -l "$www_user" -- -e <getUserSession()->getManager()->checkPasswordNoLogging("$user", getenv("pw")) ? "matched" : "mismatched");' +SU + fi +fi diff --git a/type/__nextcloud_user/explorer/user b/type/__nextcloud_user/explorer/user new file mode 100755 index 0000000..ab97a1f --- /dev/null +++ b/type/__nextcloud_user/explorer/user @@ -0,0 +1,39 @@ +#!/bin/sh -e +# __nextcloud_user/explorer/user + +# Outputs the raw nextcloud command output of the given user + + +# Parameters +user="$__object/parameter/user" +if [ -f "$user" ]; then + user="$(cat "$user")" +else + user="$__object_id" +fi +cloud="$(cat "$__object/parameter/cloud")" +www_user="$(cat "$__object/parameter/www-user")" + + +# Check if there exists the installation +if [ -d "$cloud" ]; then + # if those files exist, everything should be good + if [ -f "$cloud/occ" ] && [ -f "$cloud/config/config.php" ]; then + # Content could be gathered through php code directly, too. This can + # be done if more parameters are required than user:info will output + # or if there will be too much fuzz in the output. + + # Output the information of the user + # type will abort if explorer is empty, not if occ aborts + su -s /bin/sh -l "$www_user" -- -e <getUserSession()->getManager()->userExists("$user") ? 0 : 1);' +then + php occ --no-interaction --no-ansi --output=plain user:info '$user' +fi +SU + fi +fi diff --git a/type/__nextcloud_user/gencode-remote b/type/__nextcloud_user/gencode-remote new file mode 100644 index 0000000..f5cdaf5 --- /dev/null +++ b/type/__nextcloud_user/gencode-remote @@ -0,0 +1,239 @@ +#!/bin/sh -e +# __nextcloud_user/gencode-remote + + +# Call the nextcloud occ script as the designed user. Maybe this can be a bit +# more effictive with user switching, but currently the easiest way of doing +# it. +# +# All arguments are directly passed to occ (injection alarm ;-) ) +occ() { + # su creates a new shell, so it does not affect the current session + # will not use -q as it supresses errors, too + cat << SHELL +su -s /bin/sh -l "$www_user" -- -e <<'SU' +cd '$cloud' && php occ --no-interaction --no-ansi $@ +SU +SHELL +} + +# Creates the output for the nextcloud command to create a user. Takes all +# required parameters from existing variables. +occ_create() { + cat <> "$__messages_out" + else + occ_create + echo created >> "$__messages_out" + fi + ;; + + disabled) + if [ "$state_is" = "absent" ]; then + occ_create + echo created >> "$__messages_out" + fi + + occ user:disable "'$user'" + echo disabled >> "$__messages_out" + ;; + + present) + if [ "$state_is" = "absent" ]; then + occ_create + echo created >> "$__messages_out" + fi + # else, everything is ok + ;; + + absent) + occ user:delete "'$user'" + echo removed >> "$__messages_out" + ;; + esac +fi + + +# Check if some user configuration should be changed +# do not run this code if the user will be created in the previous code +if [ "$state_should" != "absent" ] && [ "$occ_created" != "yes" ]; then + # Check if the display name is correct if someone is set + if [ -f "$__object/parameter/displayname" ]; then + displayname="$(cat "$__object/parameter/displayname")" + if ! match_param display_name "$displayname"; then + cat <getUserSession()->getManager()->get("$user")->setDisplayName("$displayname") + or die("Couldn'\''t modify $user display name! Maybe unsupported or already set ..".PHP_EOL);' +SU +SHELL + fi + fi + + # Check if the email address is correct + if [ -f "$__object/parameter/email" ]; then + email="$(cat "$__object/parameter/email")" + if ! match_param email "$email"; then + occ user:setting -- "'$user'" settings email "'$email'" + fi + else + # remove if it doesn't exist + if ! match_param email ""; then + occ user:setting --delete -- "'$user'" settings email + fi + fi + + # Check state of the password + # explorer handles missing passwords already + if [ "$(cat "$__object/explorer/password")" = "mismatched" ]; then + cat < "$__object/files/explorer_groups" + + # Add/Remove groups not set via the parameter + if [ -s "$__object/parameter/group" ]; then + # Get all groups to remove + grep -Fxv -f "$__object/parameter/group" \ + "$__object/files/explorer_groups" > "$__object/files/group.del" || true + # Get all groups to add + grep -Fxv -f "$__object/files/explorer_groups" \ + "$__object/parameter/group" > "$__object/files/group.add" || true + + # No user groups at all if nothing wanted by the user + else + # remove all groups to stay inline with the user parameter + cp "$__object/files/explorer_groups" "$__object/files/group.del" + fi + + # Remove all groups not exist anymore + if [ -s "$__object/files/group.del" ]; then + while read -r GROUP; do + cat <getGroupManager()->get("$GROUP"); + \$user = \\OC::\$server->getUserSession()->getManager()->get("$user"); + if (\$group === NULL || \$user === NULL) + die("Can'\''t delete $user from group $GROUP! User or group doesn'\''t exist in nextcloud!".PHP_EOL); + \$group->removeUser(\$user);' +SU +SHELL + done < "$__object/files/group.del" + fi + + # Add all existing groups + if [ -s "$__object/files/group.add" ]; then + while read -r GROUP; do + cat <getGroupManager()->get("$GROUP"); + \$user = \\OC::\$server->getUserSession()->getManager()->get("$user"); + if (\$group === NULL || \$user === NULL) + die("Can'\''t delete $user from group $GROUP! User or group doesn'\''t exist in nextcloud!".PHP_EOL); + \$group->addUser(\$user);' +SU +SHELL + done < "$__object/files/group.add" + fi +fi diff --git a/type/__nextcloud_user/parameter/default/state b/type/__nextcloud_user/parameter/default/state new file mode 100644 index 0000000..e7f6134 --- /dev/null +++ b/type/__nextcloud_user/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/type/__nextcloud_user/parameter/default/www-user b/type/__nextcloud_user/parameter/default/www-user new file mode 100644 index 0000000..5bbad18 --- /dev/null +++ b/type/__nextcloud_user/parameter/default/www-user @@ -0,0 +1 @@ +www-data diff --git a/type/__nextcloud_user/parameter/optional b/type/__nextcloud_user/parameter/optional new file mode 100644 index 0000000..aaf31d0 --- /dev/null +++ b/type/__nextcloud_user/parameter/optional @@ -0,0 +1,7 @@ +user +www-user +state +displayname +email +password +quota diff --git a/type/__nextcloud_user/parameter/optional_multiple b/type/__nextcloud_user/parameter/optional_multiple new file mode 100644 index 0000000..3a60cce --- /dev/null +++ b/type/__nextcloud_user/parameter/optional_multiple @@ -0,0 +1 @@ +group diff --git a/type/__nextcloud_user/parameter/required b/type/__nextcloud_user/parameter/required new file mode 100644 index 0000000..c3de202 --- /dev/null +++ b/type/__nextcloud_user/parameter/required @@ -0,0 +1 @@ +cloud From 1e1df1016d102a1eda3b809030026e5867b8a9d8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 30 Oct 2020 20:01:53 +0100 Subject: [PATCH 152/332] __nextcloud_user: add manpage --- type/__nextcloud_user/man.rst | 99 +++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 type/__nextcloud_user/man.rst diff --git a/type/__nextcloud_user/man.rst b/type/__nextcloud_user/man.rst new file mode 100644 index 0000000..3a51a98 --- /dev/null +++ b/type/__nextcloud_user/man.rst @@ -0,0 +1,99 @@ +cdist-type__nextcloud_user(7) +============================= + +NAME +---- +cdist-type__nextcloud_user - Setup a Nextcloud user + + +DESCRIPTION +----------- +It manages a single Nextcloud user given by the object id or parameter `--user`. + + +REQUIRED PARAMETERS +------------------- +cloud + The absolute path of the Nextcloud installation. + + +OPTIONAL PARAMETERS +------------------- +state + The state the user should be in. Can be the following: + + present *(default)* + The user exists. + + enabled + The user exists and is enabled. + + disabled + The user exists and is disabled. + + absent + The user does not exist. + +user + Takes the uid of the Nextcloud user which will be handled by this type. If + this is not set, the object id will be taken instead. + +www-user + The unix user which will be used to execute Nextcloud related stuff. You + should always use the same user for all Nextcloud interactions, for the + webserver and cli execution. As default, `www-data` will be used. + +displayname + The display name the user should have. + +email + The email address of the Nextcloud user. + +password + The password of the Nextcloud user. + +quota + TBA. + +group + Multiple group names which the Nextcloud user belongs to. If not set, the + user will be removed from every group he is in. + + +MESSAGES +-------- +created + The user as created. + +enabled + The user already exists and was enabled. + +disabled + The user already exists and was disabled. + +removed + The user was removed. + + +EXAMPLES +-------- +TBA. + + +SEE ALSO +-------- +:strong:`cdist-type__nextcloud`\ (7) +:string:`cdist-type__nextcloud_app`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. From 635f03f527140a69a4caeb7091f9ae0cfc91de76 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 30 Oct 2020 20:08:32 +0100 Subject: [PATCH 153/332] __nextcloud_user: use `occ group:` commands Instead of the self-tinkered php code, the correct occ command is used now. --- type/__nextcloud_user/gencode-remote | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/type/__nextcloud_user/gencode-remote b/type/__nextcloud_user/gencode-remote index f5cdaf5..f5b0bc4 100644 --- a/type/__nextcloud_user/gencode-remote +++ b/type/__nextcloud_user/gencode-remote @@ -206,34 +206,14 @@ SHELL # Remove all groups not exist anymore if [ -s "$__object/files/group.del" ]; then while read -r GROUP; do - cat <getGroupManager()->get("$GROUP"); - \$user = \\OC::\$server->getUserSession()->getManager()->get("$user"); - if (\$group === NULL || \$user === NULL) - die("Can'\''t delete $user from group $GROUP! User or group doesn'\''t exist in nextcloud!".PHP_EOL); - \$group->removeUser(\$user);' -SU -SHELL + occ group:removeuser "'$GROUP'" "'$user'" done < "$__object/files/group.del" fi # Add all existing groups if [ -s "$__object/files/group.add" ]; then while read -r GROUP; do - cat <getGroupManager()->get("$GROUP"); - \$user = \\OC::\$server->getUserSession()->getManager()->get("$user"); - if (\$group === NULL || \$user === NULL) - die("Can'\''t delete $user from group $GROUP! User or group doesn'\''t exist in nextcloud!".PHP_EOL); - \$group->addUser(\$user);' -SU -SHELL + occ group:adduser "'$GROUP'" "'$user'" done < "$__object/files/group.add" fi fi From 82283d0b1cb9fb3a002a0add3dde3d6f7bcb479e Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 31 Oct 2020 09:47:38 +0100 Subject: [PATCH 154/332] __nextcloud_user: new parameters to prevent changes to the user This adds new parameters `--only-setup` and `--keep-*` to prevent certain parameters to be changed in Nextcloud. This will not apply to the setup. --- type/__nextcloud_user/gencode-remote | 125 +++++++++++++----------- type/__nextcloud_user/man.rst | 52 +++++++++- type/__nextcloud_user/parameter/boolean | 6 ++ 3 files changed, 125 insertions(+), 58 deletions(-) create mode 100644 type/__nextcloud_user/parameter/boolean diff --git a/type/__nextcloud_user/gencode-remote b/type/__nextcloud_user/gencode-remote index f5b0bc4..dd0c715 100644 --- a/type/__nextcloud_user/gencode-remote +++ b/type/__nextcloud_user/gencode-remote @@ -52,7 +52,7 @@ SHELL fi # save that use user will be created and no further steps are required - occ_created="yes" + ignore_config="yes" } # Checks if the key-value exists on the remote side. Only matches first-level @@ -136,84 +136,99 @@ if [ "$state_is" != "$state_should" ]; then esac fi +# Check if the user should not be modified further from the initial setup. +if [ -f "$__object/parameter/only-setup" ]; then + ignore_config="yes" +fi + # Check if some user configuration should be changed # do not run this code if the user will be created in the previous code -if [ "$state_should" != "absent" ] && [ "$occ_created" != "yes" ]; then - # Check if the display name is correct if someone is set - if [ -f "$__object/parameter/displayname" ]; then - displayname="$(cat "$__object/parameter/displayname")" - if ! match_param display_name "$displayname"; then - cat <getUserSession()->getManager()->get("$user")->setDisplayName("$displayname") - or die("Couldn'\''t modify $user display name! Maybe unsupported or already set ..".PHP_EOL);' + or print("Couldn'\''t modify $user display name! Maybe unsupported or already set ..".PHP_EOL) + and die(1);' SU SHELL + fi + fi + # the display name can not be unset + fi + + if ! [ -f "$__object/paramter/keep-email" ]; then + # Check if the email address is correct + if [ -f "$__object/parameter/email" ]; then + email="$(cat "$__object/parameter/email")" + if ! match_param email "$email"; then + occ user:setting -- "'$user'" settings email "'$email'" + fi + else + # remove if it doesn't exist + if ! match_param email ""; then + occ user:setting --delete -- "'$user'" settings email + fi fi fi - # Check if the email address is correct - if [ -f "$__object/parameter/email" ]; then - email="$(cat "$__object/parameter/email")" - if ! match_param email "$email"; then - occ user:setting -- "'$user'" settings email "'$email'" - fi - else - # remove if it doesn't exist - if ! match_param email ""; then - occ user:setting --delete -- "'$user'" settings email - fi - fi - - # Check state of the password - # explorer handles missing passwords already - if [ "$(cat "$__object/explorer/password")" = "mismatched" ]; then - cat < "$__object/files/explorer_groups" + if ! [ -f "$__object/parameter/keep-groups" ]; then + # Handle the user groups + # extract all groups set by remote + mkdir -p "$__object/files" + # check the spaces before the value to match all sub-categories + awk '/^ -/{start=0} start{print $2} $0 == " - groups:"{start=1}' \ + "$__object/explorer/user" > "$__object/files/explorer_groups" - # Add/Remove groups not set via the parameter - if [ -s "$__object/parameter/group" ]; then - # Get all groups to remove - grep -Fxv -f "$__object/parameter/group" \ - "$__object/files/explorer_groups" > "$__object/files/group.del" || true - # Get all groups to add - grep -Fxv -f "$__object/files/explorer_groups" \ - "$__object/parameter/group" > "$__object/files/group.add" || true + # Add/Remove groups not set via the parameter + if [ -s "$__object/parameter/group" ]; then + # Get all groups to remove + grep -Fxv -f "$__object/parameter/group" \ + "$__object/files/explorer_groups" > "$__object/files/group.del" || true + # Get all groups to add + grep -Fxv -f "$__object/files/explorer_groups" \ + "$__object/parameter/group" > "$__object/files/group.add" || true - # No user groups at all if nothing wanted by the user - else - # remove all groups to stay inline with the user parameter - cp "$__object/files/explorer_groups" "$__object/files/group.del" - fi + # No user groups at all if nothing wanted by the user + else + # remove all groups to stay inline with the user parameter + cp "$__object/files/explorer_groups" "$__object/files/group.del" + fi - # Remove all groups not exist anymore - if [ -s "$__object/files/group.del" ]; then - while read -r GROUP; do - occ group:removeuser "'$GROUP'" "'$user'" - done < "$__object/files/group.del" - fi + # Remove all groups not exist anymore + if [ -s "$__object/files/group.del" ]; then + while read -r GROUP; do + occ group:removeuser "'$GROUP'" "'$user'" + done < "$__object/files/group.del" + fi - # Add all existing groups - if [ -s "$__object/files/group.add" ]; then - while read -r GROUP; do - occ group:adduser "'$GROUP'" "'$user'" - done < "$__object/files/group.add" + # Add all existing groups + if [ -s "$__object/files/group.add" ]; then + while read -r GROUP; do + occ group:adduser "'$GROUP'" "'$user'" + done < "$__object/files/group.add" + fi fi fi diff --git a/type/__nextcloud_user/man.rst b/type/__nextcloud_user/man.rst index 3a51a98..f68a092 100644 --- a/type/__nextcloud_user/man.rst +++ b/type/__nextcloud_user/man.rst @@ -9,6 +9,10 @@ cdist-type__nextcloud_user - Setup a Nextcloud user DESCRIPTION ----------- It manages a single Nextcloud user given by the object id or parameter `--user`. +This type can create and manage most properties of the Nextcloud user. If you +only want to setup the user, but want that the user will take full control over +all settings (so cdist will not touch the user anymore), use the parameter +`--only-setup` or `--keep-*` for special parameters. REQUIRED PARAMETERS @@ -44,13 +48,21 @@ www-user webserver and cli execution. As default, `www-data` will be used. displayname - The display name the user should have. + The display name the user should have. As the display name can not be unset + or set to empty, this type will ignore the display name if this parameter + is not set. Setting the parameter to an empty string leads to an error from + the Nextcloud side. email - The email address of the Nextcloud user. + The email address of the Nextcloud user. Will be unset if no parameter + given. password - The password of the Nextcloud user. + The password of the Nextcloud user. If the password not match, the new + password will be set to the user. If no password is given, it will not + touch the current password. **A password is required for the user setup!** + If you do not want to modify the user password, set a password via this + parameter and set the parameter `--keep-password`. quota TBA. @@ -60,6 +72,40 @@ group user will be removed from every group he is in. +BOOLEAN PARAMETERS +------------------ +only-setup + Only provisioning the user if he does not exist. Do not touch the user if + he already exists (except to enforce the given state). + +keep-displayname + Do not touch the display name of the user if he is already set up. This + will avoid to delete the user-set value because it does not match with the + predefined state. If the parameter `--displayname` is set despite of this + parameter, it will be used in the user setup if he does not already exist. + +keep-email + Do not touch the email attributes of the user if he is already set up. This + will avoid to delete the user-set value because it does not match with the + predefined state. If the parameter `--email` is set despite of this + parameter, it will be used in the user setup if he does not already exist. + +keep-password + Do not touch the password if the user is already set up. This will avoid to + delete user-set passwords because they do not match with the predefined + state. If the parameter `--password` is set despite of this parameter, it + will be used in the user setup if he does not already exists. + +keep-quota + TBA. + +keep-groups + Do not touch the user groups if the user is already set up. This will avoid + to delete group assosiactions not defined via cdist. If the parameter + `--group` is set despite of this parameter, it will be used in the user + setup if he does not already exists. + + MESSAGES -------- created diff --git a/type/__nextcloud_user/parameter/boolean b/type/__nextcloud_user/parameter/boolean new file mode 100644 index 0000000..cf0a40a --- /dev/null +++ b/type/__nextcloud_user/parameter/boolean @@ -0,0 +1,6 @@ +only-setup +keep-displayname +keep-email +keep-password +keep-quota +keep-groups From 30205b38ae88f3ba89f65cce818fb9992a839cbb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 31 Oct 2020 11:44:31 +0100 Subject: [PATCH 155/332] __nextcloud_user: add --quota paramter --- type/__nextcloud_user/explorer/user | 10 ++++++++-- type/__nextcloud_user/gencode-remote | 13 ++++++++++++- type/__nextcloud_user/man.rst | 17 +++++++++++------ type/__nextcloud_user/parameter/default/quota | 1 + 4 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 type/__nextcloud_user/parameter/default/quota diff --git a/type/__nextcloud_user/explorer/user b/type/__nextcloud_user/explorer/user index ab97a1f..bf2c475 100755 --- a/type/__nextcloud_user/explorer/user +++ b/type/__nextcloud_user/explorer/user @@ -1,7 +1,10 @@ #!/bin/sh -e # __nextcloud_user/explorer/user -# Outputs the raw nextcloud command output of the given user +# Outputs the raw nextcloud command output of the given user. +# +# The output is extended by the following fields (in the same syntax): +# 1. quota_param which outputs the real quota value instead of resolved values # Parameters @@ -32,7 +35,10 @@ cd '$cloud' if php -r 'define("OC_CONSOLE",1); require_once(__DIR__."/lib/base.php"); exit(\\OC::\$server->getUserSession()->getManager()->userExists("$user") ? 0 : 1);' then - php occ --no-interaction --no-ansi --output=plain user:info '$user' + php occ --no-warnings --no-interaction --no-ansi --output=plain user:info '$user' + # also output the quota parameter + printf " - quota_parm: %s\n" \ + "\$(php occ --no-warnings --no-interaction --no-ansi user:setting '$user' files quota)" fi SU fi diff --git a/type/__nextcloud_user/gencode-remote b/type/__nextcloud_user/gencode-remote index dd0c715..bd986f5 100644 --- a/type/__nextcloud_user/gencode-remote +++ b/type/__nextcloud_user/gencode-remote @@ -12,7 +12,7 @@ occ() { # will not use -q as it supresses errors, too cat << SHELL su -s /bin/sh -l "$www_user" -- -e <<'SU' -cd '$cloud' && php occ --no-interaction --no-ansi $@ +cd '$cloud' && php occ --no-warnings --no-interaction --no-ansi $@ SU SHELL } @@ -51,6 +51,9 @@ SHELL occ user:setting -- "'$user'" settings email "'$(cat "$__object/parameter/email")'" fi + # add the set quota - the default value don't hurt if it's explicitly set + occ user:setting -- "'$user'" files quota "'$(cat "$__object/parameter/quota")'" + # save that use user will be created and no further steps are required ignore_config="yes" } @@ -193,6 +196,14 @@ SHELL fi fi + if ! [ -f "$__object/parameter/keep-quota" ]; then + # Handle the qouta + # the parameter is always set cause of the default value + quota="$(cat "$__object/parameter/quota")" + if ! match_param quota_param "$quota"; then + occ user:setting -- "'$user'" files quota "'$quota'" + fi + fi if ! [ -f "$__object/parameter/keep-groups" ]; then # Handle the user groups diff --git a/type/__nextcloud_user/man.rst b/type/__nextcloud_user/man.rst index f68a092..43aefc8 100644 --- a/type/__nextcloud_user/man.rst +++ b/type/__nextcloud_user/man.rst @@ -65,7 +65,7 @@ password parameter and set the parameter `--keep-password`. quota - TBA. + The quota the Nextcloud user have to store it data. Defaults to `default`. group Multiple group names which the Nextcloud user belongs to. If not set, the @@ -82,27 +82,32 @@ keep-displayname Do not touch the display name of the user if he is already set up. This will avoid to delete the user-set value because it does not match with the predefined state. If the parameter `--displayname` is set despite of this - parameter, it will be used in the user setup if he does not already exist. + parameter, it will only be used in the user setup if he does not already + exist. keep-email Do not touch the email attributes of the user if he is already set up. This will avoid to delete the user-set value because it does not match with the predefined state. If the parameter `--email` is set despite of this - parameter, it will be used in the user setup if he does not already exist. + parameter, it will only be used in the user setup if he does not already + exist. keep-password Do not touch the password if the user is already set up. This will avoid to delete user-set passwords because they do not match with the predefined state. If the parameter `--password` is set despite of this parameter, it - will be used in the user setup if he does not already exists. + will only be used in the user setup if he does not already exists. keep-quota - TBA. + Do not touch the user quota if he is already set up. This will avoid to + delete the configuration set by an administrator. If the parameter `--quota` + is set despite of this parameter, it will only be used in the user setup if + he does not already exist. keep-groups Do not touch the user groups if the user is already set up. This will avoid to delete group assosiactions not defined via cdist. If the parameter - `--group` is set despite of this parameter, it will be used in the user + `--group` is set despite of this parameter, it will only be used in the user setup if he does not already exists. diff --git a/type/__nextcloud_user/parameter/default/quota b/type/__nextcloud_user/parameter/default/quota new file mode 100644 index 0000000..4ad96d5 --- /dev/null +++ b/type/__nextcloud_user/parameter/default/quota @@ -0,0 +1 @@ +default From a82a880858457cf57b74b64255d72b16e694a620 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 31 Oct 2020 12:26:36 +0100 Subject: [PATCH 156/332] __nextcloud_user: finished manpage --- type/__nextcloud_user/gencode-remote | 4 ++ type/__nextcloud_user/man.rst | 59 +++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/type/__nextcloud_user/gencode-remote b/type/__nextcloud_user/gencode-remote index bd986f5..c76f74e 100644 --- a/type/__nextcloud_user/gencode-remote +++ b/type/__nextcloud_user/gencode-remote @@ -242,4 +242,8 @@ SHELL done < "$__object/files/group.add" fi fi + + + # These parameters are only set if they exist + # ... fi diff --git a/type/__nextcloud_user/man.rst b/type/__nextcloud_user/man.rst index 43aefc8..39ff38a 100644 --- a/type/__nextcloud_user/man.rst +++ b/type/__nextcloud_user/man.rst @@ -66,6 +66,17 @@ password quota The quota the Nextcloud user have to store it data. Defaults to `default`. + Following values are accepted by Nextcloud: + + default + Uses the quota set as default in Nextcloud. + + none + No quota limit set; unlimited. + + $size + The quota that should be used. Same values as set over the user + interface. First the number, then a space and then the unit like `GB`. group Multiple group names which the Nextcloud user belongs to. If not set, the @@ -128,13 +139,57 @@ removed EXAMPLES -------- -TBA. + +.. code-block:: sh + + # nextcloud base installation + __nextcloud cloud + + # setups an user, but do not touch it after it was created + require="__nextcloud/cloud" __nextcloud_user foo \ + --cloud /var/www/html/cloud/ \ + --displayname "Big Fooo" \ + --email "foo@bar.tld" \ + --password "do-not-use-this-password" \ + --group "team_a" --group "xxxx" \ + --quota "2 GB" + --only-setup + + # manages an admin user fully controlled by cdist + require="__nextcloud/cloud" __nextcloud_user bar \ + --cloud /var/www/html/cloud/ \ + --displayname "Bar" \ + --email "bar@bar.tld" \ + --password "nope_insecure" \ + --group "admin" + + # disables an user + require="__nextcloud/cloud" __nextcloud_user bb \ + --state disabled \ + --cloud /var/www/html/cloud/ \ + --displayname "byebye" \ + --password "do_not_copy" \ + --keep-email --keep-password --keep-quota --keep-groups + + # removes an user + require="__nextcloud/cloud" __nextcloud_user foobar \ + --state absent \ + --cloud /var/www/html/cloud/ + + +NOTES +----- +This type may be extended by more user settings. If you think some +configuration is missing, you are welcome to contribute! + +Sometimes, this type uses custom php code to hack into Nextcloud to gather some +information not possible to get via the `occ` command or even set a value. SEE ALSO -------- :strong:`cdist-type__nextcloud`\ (7) -:string:`cdist-type__nextcloud_app`\ (7) +:strong:`cdist-type__nextcloud_app`\ (7) AUTHORS From d0f597b9e44ed60bb00d3c09d7eaaf770e622b20 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 31 Oct 2020 14:16:31 +0100 Subject: [PATCH 157/332] __nextcloud: added manpage notes .. and some more. --- type/__nextcloud/man.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst index 9daa4ce..1996d42 100644 --- a/type/__nextcloud/man.rst +++ b/type/__nextcloud/man.rst @@ -234,8 +234,19 @@ this type and contribute it upstream! - `Nextcloud configuration reference `_ +Currently, the state of this object is always `present`. So it will always be +installed without the option to uninstall it again (`absent`). This was done +because it will not be a common demand to uninstall nextcloud again. If you +need to toggle the state, you are welcome to contirbute! + +Parameters given for the admin user which will be set up at installation time +(`--admin-*` ones) are not applied if nextcloud will not be installed. +Therefor, parameter changes are not applied to the installation. Currently not +implemented - but possible - is to use the type +:strong:`cdist-type__nextcloud_user`\ (7) to do all the later work. + Database migration is only partly supported if the database will be changed to -``mysql` or ``pgsql``, because it is supported by an upstream script. You are +``mysql`` or ``pgsql``, because it is supported by an upstream script. You are welcome to extend this type for database migrations between the same database type. For an implementation, you may use shell utilites like ``mysqldump(1)`` (be aware that this may not already be installed) or use the already installed @@ -267,6 +278,13 @@ environment. This can happen after a database migration between different database types, as it installs the database module only when it is required. +SEE ALSO +-------- +`Nextcloud documentation `_ + +:strong:`cdist-type__nextcloud_user`\ (7) + + AUTHORS ------- Matthias Stecher From 9ed1a9cbfd57b6666747971f812f9e566138053b Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 31 Oct 2020 16:25:35 +0100 Subject: [PATCH 158/332] __nextcloud_user: fix misspelled quota parameter --- type/__nextcloud_user/explorer/user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__nextcloud_user/explorer/user b/type/__nextcloud_user/explorer/user index bf2c475..face683 100755 --- a/type/__nextcloud_user/explorer/user +++ b/type/__nextcloud_user/explorer/user @@ -37,7 +37,7 @@ if php -r 'define("OC_CONSOLE",1); require_once(__DIR__."/lib/base.php"); then php occ --no-warnings --no-interaction --no-ansi --output=plain user:info '$user' # also output the quota parameter - printf " - quota_parm: %s\n" \ + printf " - quota_param: %s\n" \ "\$(php occ --no-warnings --no-interaction --no-ansi user:setting '$user' files quota)" fi SU From 7490cef49e148e68700c643242b4976d7ee15f8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 31 Oct 2020 16:47:39 +0100 Subject: [PATCH 159/332] Initial __nextcloud_app type Overall complete, more functionalitly is currently out of scope. --- type/__nextcloud_app/explorer/state | 38 +++++ type/__nextcloud_app/gencode-remote | 85 ++++++++++++ type/__nextcloud_app/man.rst | 131 ++++++++++++++++++ type/__nextcloud_app/parameter/default/state | 1 + .../parameter/default/www-user | 1 + type/__nextcloud_app/parameter/optional | 3 + type/__nextcloud_app/parameter/required | 1 + type/__nextcloud_user/man.rst | 8 +- 8 files changed, 264 insertions(+), 4 deletions(-) create mode 100755 type/__nextcloud_app/explorer/state create mode 100644 type/__nextcloud_app/gencode-remote create mode 100644 type/__nextcloud_app/man.rst create mode 100644 type/__nextcloud_app/parameter/default/state create mode 100644 type/__nextcloud_app/parameter/default/www-user create mode 100644 type/__nextcloud_app/parameter/optional create mode 100644 type/__nextcloud_app/parameter/required diff --git a/type/__nextcloud_app/explorer/state b/type/__nextcloud_app/explorer/state new file mode 100755 index 0000000..d7572ce --- /dev/null +++ b/type/__nextcloud_app/explorer/state @@ -0,0 +1,38 @@ +#!/bin/sh -e +# __nextcloud_app/explorer/state + +# Outputs the current state of the app. There are: +# - `enabled` if the app is enabled +# - `disabled` if the app is disabled +# - `absent` if the app does not exist +# - nothing if nextcloud is not installed + + +# Get the app id +appid="$__object/parameter/appid" +if [ -f "$appid" ]; then + appid="$(cat "$appid")" +else + appid="$__object_id" +fi + +# Get the installation directory +cloud="$(cat "$__object/parameter/cloud")" +www_user="$(cat "$__object/parameter/www-user")" + + +# Check if the installation directory exists +if [ -d "$cloud" ]; then + # if those files exist, everything should be fine + if [ -f "$cloud/occ" ] && [ -f "$cloud/config/config.php" ]; then + # Check if the app exists in the correct user context + su -s /bin/sh -l "$www_user" -- -e <&2 + echo "Use the type __nextcloud to ensure the installation and mark it as dependency for this type!" >&2 + exit 2 +fi + + +# Check if the state changes +if [ "$state_is" != "$state_should" ]; then + # check what to do + case "$state_should" in + enabled) + if [ "$state_is" = "disabled" ]; then + occ app:enable "'$appid'" + echo enabled >> "$__messages_out" + else + occ app:install "'$appid'" + echo installed >> "$__messages_out" + fi + ;; + + disabled) + if [ "$state_is" = "absent" ]; then + occ app:install --keep-disabled "'$appid'" + echo installed >> "$__messages_out" + else + occ app:disable "'$appid'" + echo disabled >> "$__messages_out" + fi + ;; + + present) + if [ "$state_is" = "absent" ]; then + occ app:install "'$appid'" + echo installed >> "$__messages_out" + fi + # else, everything is ok + ;; + + absent) + occ app:remove "'$appid'" + echo removed >> "$__messages_out" + ;; + esac +fi diff --git a/type/__nextcloud_app/man.rst b/type/__nextcloud_app/man.rst new file mode 100644 index 0000000..2b13c5c --- /dev/null +++ b/type/__nextcloud_app/man.rst @@ -0,0 +1,131 @@ +cdist-type__nextcloud_app(7) +============================ + +NAME +---- +cdist-type__nextcloud_app - Managese a Nextcloud app installation + + +DESCRIPTION +----------- +This types manages an app for a Nextcloud installation. For now, you can only +(un-)install or enable/disable an app. + +The object id is the appid of the app which will be managed by this type. It +will be overwritten by the parameter `--appid`. See this parameter for more +information about the appid. + + +REQUIRED PARAMETERS +------------------- +cloud + The absolute path of the Nextcloud installation. + + +OPTIONAL PARAMETERS +------------------- +state + The state of the app. Can be the following: + + present *(default)* + The app is installed. + + enabled + The app is installed and enabled. + + disabled + The app is installed, but disabled. + + absent + The app is not installed. + +appid + The appid is the uniquie identifier for an app in the Nextcloud app store. + It is required to know which app should be installed, which is expressed + via the appid. Apps who are shipped by the installation can not be removed. + Doing this will throw an error at exeuction time. + + To find the appid, you must select the app in the Nextcloud app menu or on + the app page in the Nextcloud app store. Then, examine the URL and use the + lastest part (e.g. "the filename") as appid. + +www-user + The unix user which will be used to execute Nextcloud related stuff. You + should always use the same user for all Nextcloud interactions, for the + webserver and cli execution. As default, `www-data` will be used. + + +MESSAGES +-------- +installed + The app was installed. + +enabled + The app is already installed and was enabled. + +disabled + The app is already installed and was disabled. + +removed + The app was removed. + + +EXAMPLES +-------- + +.. code-block:: sh + + # nextcloud base installation + __nextcloud cloud $args + + # install the music app + require="__nextcloud/cloud" __nextcloud_app music \ + --cloud /var/www/html/cloud/ --state enabled + + # enable a shipped app (already installed) + require="__nextcloud/cloud" __nextcloud_app files_external \ + --cloud /var/www/html/cloud/ --state enabled + + # remove some app + require="__nextcloud/cloud" __nextcloud_app drawio \ + --cloud /var/www/html/cloud/ --state absent + + +NOTES +----- +Currently, it manages just if the app is installed and enabled. Further +implementation is possible, but not done yet. This contains the management of +the app settings (via ``occ config:app:*``) and further finetuning to the +possibilities of installation and enablement (force-enable an app or restrict +enablement only to some groups). + +Special app settings could also be written as a new type which completly +handles this one app with all configuration options. + +Upgrading an Nextcloud app may be possible, but not the scope of this type. +Also, the upgrade can not be done to a given version, which results that this +type will loose the control over the state of the app. Installing the app +manually or hooking into the Nextcloud code is too unsafe and complex, in +addition it will be used rarely. Most admins would propably just update the app +via the web interface. + + +SEE ALSO +-------- +`Nextcloud app store `_ + +:strong:`cdist-type__nextcloud`\ (7) +:strong:`cdist-type__nextcloud_user`\ (7) + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2020 Matthias Stecher. +You can redistribute it and/or modify it under the terms of the GNU +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. diff --git a/type/__nextcloud_app/parameter/default/state b/type/__nextcloud_app/parameter/default/state new file mode 100644 index 0000000..e7f6134 --- /dev/null +++ b/type/__nextcloud_app/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/type/__nextcloud_app/parameter/default/www-user b/type/__nextcloud_app/parameter/default/www-user new file mode 100644 index 0000000..5bbad18 --- /dev/null +++ b/type/__nextcloud_app/parameter/default/www-user @@ -0,0 +1 @@ +www-data diff --git a/type/__nextcloud_app/parameter/optional b/type/__nextcloud_app/parameter/optional new file mode 100644 index 0000000..0f2a3eb --- /dev/null +++ b/type/__nextcloud_app/parameter/optional @@ -0,0 +1,3 @@ +state +appid +www-user diff --git a/type/__nextcloud_app/parameter/required b/type/__nextcloud_app/parameter/required new file mode 100644 index 0000000..c3de202 --- /dev/null +++ b/type/__nextcloud_app/parameter/required @@ -0,0 +1 @@ +cloud diff --git a/type/__nextcloud_user/man.rst b/type/__nextcloud_user/man.rst index 39ff38a..f0ceff9 100644 --- a/type/__nextcloud_user/man.rst +++ b/type/__nextcloud_user/man.rst @@ -142,11 +142,11 @@ EXAMPLES .. code-block:: sh - # nextcloud base installation - __nextcloud cloud + # nextcloud base installation + __nextcloud cloud $args - # setups an user, but do not touch it after it was created - require="__nextcloud/cloud" __nextcloud_user foo \ + # setups an user, but do not touch it after it was created + require="__nextcloud/cloud" __nextcloud_user foo \ --cloud /var/www/html/cloud/ \ --displayname "Big Fooo" \ --email "foo@bar.tld" \ From c24aedd4dbd9bb468bdce51cd6435d0f47f91da8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 15 Nov 2020 16:21:56 +0100 Subject: [PATCH 160/332] __nextcloud: remove --webroot parameter Because the detection of the webroot is not that good, it was removed. Now, the object id contains the full path of the nextcloud installation. This changes the installation process a bit, as it will download the tarball to /tmp rather the webroot. Also, this commit contains a bit overhaul work. --- type/__nextcloud/explorer/config | 2 +- type/__nextcloud/explorer/installdir | 16 --------- type/__nextcloud/explorer/version | 2 +- type/__nextcloud/explorer/webroot | 34 ------------------- type/__nextcloud/gencode-remote | 14 ++++---- type/__nextcloud/man.rst | 49 ++++++++++------------------ type/__nextcloud/manifest | 26 +++++++-------- type/__nextcloud/map-conf-changes.sh | 4 +-- type/__nextcloud/parameter/optional | 1 - 9 files changed, 38 insertions(+), 110 deletions(-) delete mode 100755 type/__nextcloud/explorer/installdir delete mode 100755 type/__nextcloud/explorer/webroot diff --git a/type/__nextcloud/explorer/config b/type/__nextcloud/explorer/config index f97f39d..6637ad6 100755 --- a/type/__nextcloud/explorer/config +++ b/type/__nextcloud/explorer/config @@ -5,7 +5,7 @@ # Get the installdir -installdir="$( "$__type_explorer/installdir" )" +installdir="/$__object_id" # Check if the tools are available if [ -d "$installdir" ]; then diff --git a/type/__nextcloud/explorer/installdir b/type/__nextcloud/explorer/installdir deleted file mode 100755 index 7b34f2e..0000000 --- a/type/__nextcloud/explorer/installdir +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e -# __nextcloud/explorer/installdir - -# Detects the directory nextcloud should be installed to. - - -# by parameter or auto-detection -webroot="$( "$__type_explorer/webroot" )" -if [ -z "$webroot" ]; then - echo "no installdir given and no webroot directory found" >&2 - echo "no place to install found; set it via --webroot" >&2 - exit 1 -fi - -# assemble directory with the object id -printf "%s/%s\n" "$webroot" "$__object_id" diff --git a/type/__nextcloud/explorer/version b/type/__nextcloud/explorer/version index 5a75258..1e1cda9 100755 --- a/type/__nextcloud/explorer/version +++ b/type/__nextcloud/explorer/version @@ -5,7 +5,7 @@ # Get the install directory -installdir="$( "$__type_explorer/installdir" )" +installdir="/$__object_id" # Check if the installation directory exists if [ -d "$installdir" ]; then diff --git a/type/__nextcloud/explorer/webroot b/type/__nextcloud/explorer/webroot deleted file mode 100755 index 7eab286..0000000 --- a/type/__nextcloud/explorer/webroot +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -e -# __nextcloud/explorer/webroot - -# Detects the webroot if any - - -# Just check if there is some directory and echo + exit on success -# -# Arguments: -# 1: the directory to check -check_dir() { - if [ -d "$1" ]; then - echo "$1" - exit - fi -} - - -# Check the user choice -parameter="$__object/parameter/webroot" -if [ -f "$parameter" ]; then - cat "$parameter" - exit -fi - -# Maybe checking standard webserver configs .. - -# Check if there are default directories -check_dir "/srv/www" -check_dir "/var/www/html" -check_dir "/var/www" - - -# do nothing if no webroot found diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 1850dd0..cde78ca 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -49,9 +49,8 @@ version_is="$( cat "$__object/explorer/version" )" version_should="$( cat "$__object/parameter/version" )" # the install directory -installdir="$( cat "$__object/explorer/installdir" )" -updatedir="$( dirname "$installdir" )" -tarballdir="$updatedir/.$__object_id" +installdir="/$__object_id" +tarballdir="$(dirname "$installdir")/.$(basename "$installdir")" # get used user and group user="$( cat "$__object/parameter/user" )" @@ -77,11 +76,10 @@ fi # Remove the old folder and replace it with the new one. if [ "$upgrade" ]; then cat << REMOTE -cd '$updatedir' - chown '$user':'$group' -R '$tarballdir' REMOTE + # no more changes from the user occ_maintainer_mode_on cat << REMOTE @@ -91,8 +89,6 @@ mv '$installdir/data' '$tarballdir' rm -rf '$installdir' mv '$tarballdir' '$installdir' -cd '$installdir' - REMOTE # do some maintainer stuff @@ -227,9 +223,11 @@ if [ "$data_old" ] && [ "$data_old" != "$data_new" ]; then # not be moved. occ config:system:set datadirectory --type=string --value "'$data_new'" cat << REMOTE +cd '$installdir' # only for the users safety + rm -rf '$data_new' mkdir -p '$(dirname "$data_new")' # if the parent not exists -mv '$data_old' '$data_new' +mv -T '$data_old' '$data_new' REMOTE fi diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst index 1996d42..a89a550 100644 --- a/type/__nextcloud/man.rst +++ b/type/__nextcloud/man.rst @@ -8,11 +8,9 @@ cdist-type__nextcloud - Installs and manages a nextcloud instance DESCRIPTION ----------- -This type installs, upgrades and configure a nextcloud instance. - -It installs the application in the webspace based on the object id as relative -path from the webroot. If you want to install it directly in the webroot, you -must trick out this type by setting the webroot as parent directory. +This type installs, upgrades and configure a nextcloud instance. The object +id is the absolute path for the installation directory. Nextcloud will be +installed unter that directory. REQUIRED PARAMETERS @@ -40,11 +38,6 @@ admin-password OPTIONAL PARAMETERS ------------------- -webroot - The webroot which will be used as basis for the installation. This may be - already detected by an explorer. Must be an absolute path (starting with a - slash). - mode Sets the unix file mode of the nextcloud directory. This is not inherited to child files or folders. Defaults to `755`. @@ -152,18 +145,6 @@ database-prefix ``oc_``. -WEBROOT DETECTION ------------------ -As the `object id` is the install path relatively from the webroot, it must be -known somehow. Therefor, it will try to detect a good location for it. You can -set a custom webroot via the `--webroot` parameter. As default, following -directories will be checked if they exist to be the webroot: - -1. ``/srv/www/`` -2. ``/var/www/html/`` -3. ``/var/www/`` - - MESSAGES -------- installed @@ -185,10 +166,6 @@ installed. See the parameter description of `--version` for detailed information. The problem can be fixed by bumping the version value to at least the version that is currently installed or use the parameter `--install-only`. -The type aborts if there is no webroot given as parameter and no could be -detected by the type itself. Please set the webroot via `--webroot` or extend -this type. - It may abort if the data directory can not be moved correctly. Then, the nextcloud configuration is broken and must be resolved manually: Move the data directory to the correct location or change the configuration to point to the @@ -210,20 +187,21 @@ EXAMPLES # minimal nextcloud installation with sqlite and other defaults # please only use sqlite for minimal or test installations as recommend :) - __nextcloud nextcloud --version 20.0.0 --admin-password "iaminsecure" \ + __nextcloud /var/www/html/nextcloud --version 20.0.0 \ + --admin-password "iaminsecure" \ --host localhost --host nextcloud + # installation under the webroot + __nextcloud /var/www/html/ --version 20.0.0 + --admin-password "notthatsecure" --host mycloud.example.com + # more extensive configuration - __nextcloud cloud --version 20.0.0 --admin-password "iaminsecure" \ + __nextcloud /var/www/cloud --version 20.0.0 --admin-password "iaminsecure" \ --host localhost --host nextcloud --host 192.168.1.67 \ --data-directory /var/lib/nextcloud/what \ --database-type mysql --database-host "localhost" --database-name "nextcloud" \ --database-user "test" --database-password "not-a-good-password" - # install it in the webroot /var/www/html - __nextcloud html --version 20.0.0 --admin-password "notthatsecure" \ - --webroot "/var/www" --host localhost - NOTES ----- @@ -277,6 +255,13 @@ php dependencies, but there are not recognised by the server-internal php environment. This can happen after a database migration between different database types, as it installs the database module only when it is required. +If the tarball needs to be downloaded, it will be directly downloaded into the +directory ``/tmp`` and will be unpacked to the destination for an installation +or to the same directory but prefixed with a dot for an update. It will +download it into the temp directory because it does not find a better location. +In legacy, it was downloaded to the parent directory, but this may not the best +location as the installation dir can be everywhere. + SEE ALSO -------- diff --git a/type/__nextcloud/manifest b/type/__nextcloud/manifest index 2d89f01..90508d8 100755 --- a/type/__nextcloud/manifest +++ b/type/__nextcloud/manifest @@ -76,8 +76,7 @@ user="$(cat "$__object/parameter/user")" group="$(cat "$__object/parameter/group")" # Get the installation directory -webroot="$(cat "$__object/explorer/webroot")" -installdir="$(cat "$__object/explorer/installdir")" +installdir="/$__object_id" # Set permissions after the nextcloud installation/upgrade is done # FIXME maybe less strict if some parameter is not given by the user? @@ -111,28 +110,25 @@ if [ "$version_is" ]; then fi # Set destination to a temporary directory - destination="$webroot/.$__object_id" + destination="$(dirname "$installdir")/.$(basename "$installdir")" fi else # Set destination to the real destination - destination="$webroot/$__object_id" + destination="$installdir" fi # Install/Upgrade the nextcloud version if there is a destination set. # it checks if it is necessary and intended to upgrade if [ "$destination" ] && [ "$version_is" != "$version_should" ]; then - updatedir="$( dirname "$installdir" )" + # download it to the tmp directory + # FIXME --sum is currently rather a hack, see cdist#844 + __download "/tmp/nextcloud-${version_should}.tar.bz2" \ + --url "$nextcloud_uri" \ + --sum "sha256:$(curl -sS -L "$nextcloud_sum" | awk '{print $1}')" - # Be sure the parent directory exists for installation - __directory "$updatedir" --parents - - require="__directory$updatedir" \ - __download "$updatedir/nextcloud.tar.bz2" \ - --url "$nextcloud_uri" \ - --sum "sha256:$(curl -sS -L "$nextcloud_sum" | awk '{print $1}')" # must be a required parameter?? - - require="__download$updatedir/nextcloud.tar.bz2" \ - __unpack "$updatedir/nextcloud.tar.bz2" \ + # after this, unpack it from /tmp to $destination + require="__download/tmp/nextcloud-${version_should}.tar.bz2" \ + __unpack "/tmp/nextcloud-${version_should}.tar.bz2" \ --tar-strip 1 \ --destination "$destination" fi diff --git a/type/__nextcloud/map-conf-changes.sh b/type/__nextcloud/map-conf-changes.sh index 5855761..9b6ffe3 100755 --- a/type/__nextcloud/map-conf-changes.sh +++ b/type/__nextcloud/map-conf-changes.sh @@ -270,7 +270,7 @@ fi # misc conf_array host trusted_domains -# Already set via the installer +# If already set via the installer, we don't need to do this # set default values from the nextcloud installer to do not override them if [ -z "$install" ]; then # Database to check if the type changed @@ -312,5 +312,5 @@ if [ -z "$install" ]; then esac # data-dir is handled in the gencode-remote - #conf_string data-directory datadirectory installdef "$(cat "$__object/explorer/installdir")/$__object_id/data" + #conf_string data-directory datadirectory installdef "/$__object_id/data" fi diff --git a/type/__nextcloud/parameter/optional b/type/__nextcloud/parameter/optional index cd09af4..b51ef3d 100644 --- a/type/__nextcloud/parameter/optional +++ b/type/__nextcloud/parameter/optional @@ -1,7 +1,6 @@ mode user group -webroot database-type database-host database-name From 32417aba34e6ae226b072eea1d3e408ba9fec673 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 15 Nov 2020 16:34:26 +0100 Subject: [PATCH 161/332] __nextcloud: aborts at installation with pgsql/mariadb Accidentally, there was a check for a parameter that could not exist. The block is now streamlined as other parameters got handled, too :-) --- type/__nextcloud/gencode-remote | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index cde78ca..111347c 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -122,17 +122,9 @@ if ! grep -q -F "installed = 1" "$__object/explorer/config"; then } # Database setup for mysql and pgsql db_setup() { - if ! [ -f "$__object/parameter/db-host" ]; then - echo "no hostname given! can't proceed." >&2 - exit 3 - fi + # add type and other database values occ_install_args="$occ_install_args --database '$1'" - - db_host="$__object/parameter/database-host" - if [ -f "$db_host" ]; then - occ_install_args="$occ_install_args --database-host '$(cat "$db_host")'" - fi - + occ_install_args="$occ_install_args --database-host '$(cat "$__object/parameter/database-host" || die_err)'" occ_install_args="$occ_install_args --database-name '$(cat "$__object/parameter/database-name" || die_err)'" occ_install_args="$occ_install_args --database-user '$(cat "$__object/parameter/database-user" || die_err)'" occ_install_args="$occ_install_args --database-pass '$(cat "$__object/parameter/database-password" || die_err)'" From 3fda91ca2fde5e85353d4fb0b37860e861949fbe Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 15 Nov 2020 17:40:02 +0100 Subject: [PATCH 162/332] __nextcloud: add comments about the maintenance mode ... because it not always work like you want it. --- type/__nextcloud/gencode-remote | 3 +++ type/__nextcloud/man.rst | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 111347c..00611a3 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -101,6 +101,9 @@ REMOTE # Apply some misc to the installation folder. elif [ "$install" ]; then + # Maintainer mode is not available before installation + + # Correct all file permissions of the new installation cat << REMOTE chown '$user':'$group' -R '$installdir' REMOTE diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst index a89a550..39de9e8 100644 --- a/type/__nextcloud/man.rst +++ b/type/__nextcloud/man.rst @@ -262,6 +262,16 @@ download it into the temp directory because it does not find a better location. In legacy, it was downloaded to the parent directory, but this may not the best location as the installation dir can be everywhere. +This type does not garantee to always show the maintenance mode screen because +nextcloud does not show it in every case: + +1. For fresh installations, the maintenance mode can not be set. +2. While upgrades starting at version 20, the user is promted to execute the + update manually via the webinterface instead of the maintenance screen. + +It is recommended to show an own maintanance screen via the webserver if this +is critical for you. + SEE ALSO -------- From ee650d88ed09c8189aee40199905f6f8503287f9 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 15 Nov 2020 19:14:31 +0100 Subject: [PATCH 163/332] __nextcloud_{app,user}: Updated man cause of __nextcloud changes Also, the `gencode-remote` scripts where marked executeable. --- type/__nextcloud_app/gencode-remote | 0 type/__nextcloud_app/man.rst | 17 ++++++++++++----- type/__nextcloud_user/gencode-remote | 0 type/__nextcloud_user/man.rst | 19 +++++++++++++------ 4 files changed, 25 insertions(+), 11 deletions(-) mode change 100644 => 100755 type/__nextcloud_app/gencode-remote mode change 100644 => 100755 type/__nextcloud_user/gencode-remote diff --git a/type/__nextcloud_app/gencode-remote b/type/__nextcloud_app/gencode-remote old mode 100644 new mode 100755 diff --git a/type/__nextcloud_app/man.rst b/type/__nextcloud_app/man.rst index 2b13c5c..9074b03 100644 --- a/type/__nextcloud_app/man.rst +++ b/type/__nextcloud_app/man.rst @@ -75,22 +75,29 @@ EXAMPLES .. code-block:: sh - # nextcloud base installation - __nextcloud cloud $args + # Nextcloud base installation + __nextcloud /var/www/html/cloud $args # install the music app - require="__nextcloud/cloud" __nextcloud_app music \ + require="__nextcloud/var/www/html/cloud" __nextcloud_app music \ --cloud /var/www/html/cloud/ --state enabled # enable a shipped app (already installed) - require="__nextcloud/cloud" __nextcloud_app files_external \ + require="__nextcloud/var/www/html/cloud" __nextcloud_app files_external \ --cloud /var/www/html/cloud/ --state enabled # remove some app - require="__nextcloud/cloud" __nextcloud_app drawio \ + require="__nextcloud/var/www/html/cloud" __nextcloud_app drawio \ --cloud /var/www/html/cloud/ --state absent + # Different cloud + __nextcloud /var/www/html/nextcloud $args + # but same app name + require="__nextcloud/var/www/html/nextcloud" __nextcloud_user next_music \ + --cloud /var/www/html/nextcloud/ --appid music + + NOTES ----- Currently, it manages just if the app is installed and enabled. Further diff --git a/type/__nextcloud_user/gencode-remote b/type/__nextcloud_user/gencode-remote old mode 100644 new mode 100755 diff --git a/type/__nextcloud_user/man.rst b/type/__nextcloud_user/man.rst index f0ceff9..0b303d6 100644 --- a/type/__nextcloud_user/man.rst +++ b/type/__nextcloud_user/man.rst @@ -142,11 +142,11 @@ EXAMPLES .. code-block:: sh - # nextcloud base installation - __nextcloud cloud $args + # Nextcloud base installation + __nextcloud /var/www/html/cloud $args # setups an user, but do not touch it after it was created - require="__nextcloud/cloud" __nextcloud_user foo \ + require="__nextcloud/var/www/html/cloud" __nextcloud_user foo \ --cloud /var/www/html/cloud/ \ --displayname "Big Fooo" \ --email "foo@bar.tld" \ @@ -156,7 +156,7 @@ EXAMPLES --only-setup # manages an admin user fully controlled by cdist - require="__nextcloud/cloud" __nextcloud_user bar \ + require="__nextcloud/var/www/html/cloud" __nextcloud_user bar \ --cloud /var/www/html/cloud/ \ --displayname "Bar" \ --email "bar@bar.tld" \ @@ -164,7 +164,7 @@ EXAMPLES --group "admin" # disables an user - require="__nextcloud/cloud" __nextcloud_user bb \ + require="__nextcloud/var/www/html/cloud" __nextcloud_user bb \ --state disabled \ --cloud /var/www/html/cloud/ \ --displayname "byebye" \ @@ -172,11 +172,18 @@ EXAMPLES --keep-email --keep-password --keep-quota --keep-groups # removes an user - require="__nextcloud/cloud" __nextcloud_user foobar \ + require="__nextcloud/var/www/html/cloud" __nextcloud_user foobar \ --state absent \ --cloud /var/www/html/cloud/ + # Different cloud + __nextcloud /var/www/html/nextcloud $args + # but same user name + require="__nextcloud/var/www/html/nextcloud" __nextcloud_user next_foobar \ + --cloud /var/www/html/nextcloud/ --user foobar + + NOTES ----- This type may be extended by more user settings. If you think some From 04076a75eb728b8d4ad50b31c1b0bd87a9d781a8 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 16 Nov 2020 13:57:34 +0100 Subject: [PATCH 164/332] [type/__mail_alias] man.rst: Make bugs a list --- type/__mail_alias/man.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/type/__mail_alias/man.rst b/type/__mail_alias/man.rst index d6d9742..de40512 100644 --- a/type/__mail_alias/man.rst +++ b/type/__mail_alias/man.rst @@ -48,11 +48,15 @@ EXAMPLES BUGS ---- -- Quoted strings are not parsed by this type. As a result, email addresses - containing ``,`` (commas) are treated incorrectly (they are treated as two - addresses/aliases.) - Make sure that email addresses do not contain commas. -- ``:include:`` directives in the aliases file are ignored by this type. +- Quoted strings are not parsed by this type. As a result, aliases + containing ``,`` (commas) are treated incorrectly (they are treated as + separate aliases.) + Make sure that email addresses, file names, and pipe commands do not contain + commas. +- ``:include:`` directives in the aliases file are not evaluated by this type. + They are treated like a regular alias, the values of the included file are + not expanded. + SEE ALSO -------- From 7e20d13b9ff5126580c0ac9a9231c2ebea966985 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 16 Nov 2020 13:58:05 +0100 Subject: [PATCH 165/332] [type/__mail_alias] Use explicit line variables in update_aliases.awk --- type/__mail_alias/files/update_aliases.awk | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/type/__mail_alias/files/update_aliases.awk b/type/__mail_alias/files/update_aliases.awk index 336009f..11a4c85 100644 --- a/type/__mail_alias/files/update_aliases.awk +++ b/type/__mail_alias/files/update_aliases.awk @@ -18,11 +18,11 @@ # along with cdist. If not, see . # -function getvalue(path) { +function getvalue(path, line) { # Reads the first line of the file located at path and returns it. - getline < path + getline line < path close(path) - return $0 + return line } function sepafter(f, def, _) { @@ -31,14 +31,14 @@ function sepafter(f, def, _) { return _ ? _ : def } -function write_aliases() { +function write_aliases( line) { if (aliases_written) return # print aliases line printf "%s%s", ENVIRON["__object_id"], sepafter(1, ": ") - while ((getline < aliases_should_file) > 0) { + while ((getline line < aliases_should_file) > 0) { if (aliases_written) printf ", " - printf "%s", $0 + printf "%s", line aliases_written = 1 } printf "\n" From 0932c9ccde66bdcb75e998f7a730624fdbd3068e Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 16 Nov 2020 14:10:30 +0100 Subject: [PATCH 166/332] [type/{__dma,__dma_auth,__mail_alias}] Quote things properly --- type/__dma/files/update_dma_conf.awk | 2 +- type/__dma/gencode-remote | 17 +++++++++-------- type/__dma_auth/gencode-remote | 11 ++++++----- type/__mail_alias/gencode-remote | 17 +++++++++-------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/type/__dma/files/update_dma_conf.awk b/type/__dma/files/update_dma_conf.awk index 67661fd..2f60a3d 100644 --- a/type/__dma/files/update_dma_conf.awk +++ b/type/__dma/files/update_dma_conf.awk @@ -18,7 +18,7 @@ # along with cdist. If not, see . function comment_line(line) { - # returns the position in line at which the comment'\''s text starts + # returns the position in line at which the comment's text starts # (0 if the line is not a comment) match(line, /^[ \t]*\#+[ \t]*/) return RSTART ? (RLENGTH + 1) : 0 diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index 1987106..a33388d 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -18,7 +18,8 @@ # along with cdist. If not, see . # -drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } +quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; } +drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; } CONF_PATH=/etc/dma # set in Makefile @@ -139,13 +140,13 @@ then # options are grouped by word (the first word in the line) and appended # at the end of the file. - cat <'${dma_conf}.tmp' \ -&& cat '${dma_conf}.tmp' >'${dma_conf}' -${conf_should} -EOF -rm '${dma_conf}.tmp' -CODE + cat <<-CODE + awk $(drop_awk_comments "${__type:?}/files/update_dma_conf.awk") $(quote "${dma_conf}") $(quote "${dma_conf}") <<'EOF' >$(quote "${dma_conf}.tmp") \ + && cat $(quote "${dma_conf}.tmp") >$(quote "${dma_conf}") + ${conf_should} + EOF + rm $(quote "${dma_conf}.tmp") + CODE config_updated=true echo 'config updated' >>"${__messages_out:?}" diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index aee4d7f..4279b7a 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -18,7 +18,8 @@ # along with cdist. If not, see . # -drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } +quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; } +drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; } state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") @@ -63,9 +64,9 @@ esac cat <'${auth_conf}.tmp' \ -&& cat '${auth_conf}.tmp' >'${auth_conf}' -rm -f '${auth_conf}.tmp' +awk $(drop_awk_comments "${__type:?}/files/update_dma_auth.awk") <$(quote "${auth_conf}") >$(quote "${auth_conf}.tmp") \ +&& cat $(quote "${auth_conf}.tmp") >$(quote "${auth_conf}") +rm -f $(quote "${auth_conf}.tmp") EOF diff --git a/type/__mail_alias/gencode-remote b/type/__mail_alias/gencode-remote index 3eea452..4a8f889 100755 --- a/type/__mail_alias/gencode-remote +++ b/type/__mail_alias/gencode-remote @@ -18,7 +18,8 @@ # along with cdist. If not, see . # -drop_awk_comments() { sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@"; } +quote() { printf "'%s'" "$(printf '%s' "$*" | sed -e "s/'/'\\\\''/g")"; } +drop_awk_comments() { quote "$(sed '/^[[:blank:]]*#.*$/d;/^$/d' "$@")"; } aliases_file=$(cat "${__object:?}/explorer/aliases_file") @@ -30,7 +31,7 @@ test -n "${aliases_file}" || { state_should=$(cat "${__object:?}/parameter/state") -case $state_should +case ${state_should} in (present) if cmp -s "${__object:?}/explorer/aliases" "${__object:?}/parameter/alias" @@ -65,22 +66,22 @@ in esac cat <'${aliases_file}.tmp' \ +awk $(drop_awk_comments "${__type:?}/files/update_aliases.awk") <$(quote "${aliases_file}") >$(quote "${aliases_file}.tmp") \ || { - rm -f '${aliases_file}.tmp' + rm -f $(quote "${aliases_file}.tmp") echo 'Generating new aliases file failed!' >&2 exit 1 } -if ! cmp -s '${aliases_file}' '${aliases_file}.tmp' +if ! cmp -s $(quote "${aliases_file}") $(quote "${aliases_file}.tmp") then # aliases file was modified, replace: - cat '${aliases_file}.tmp' >'${aliases_file}' + cat $(quote "${aliases_file}.tmp") >$(quote "${aliases_file}") # then, run newaliases if present ("missing" on Alpine Linux because of typo) command -v newaliases >/dev/null 2>&1 && newaliases || true fi -rm -f '${aliases_file}.tmp' +rm -f $(quote "${aliases_file}.tmp") EOF From 487574c865c0b150d83bc37bfda44c0051f91973 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 16 Nov 2020 14:15:52 +0100 Subject: [PATCH 167/332] [type/__dma] Convert AWK to loop over same file twice --- type/__dma/files/update_dma_conf.awk | 8 ++++++++ type/__dma/gencode-remote | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/type/__dma/files/update_dma_conf.awk b/type/__dma/files/update_dma_conf.awk index 2f60a3d..15ef7bf 100644 --- a/type/__dma/files/update_dma_conf.awk +++ b/type/__dma/files/update_dma_conf.awk @@ -88,6 +88,12 @@ BEGIN { FS = "\n" EQS = "[ \t]" # copied from dma/conf.c + if (ARGV[2]) exit (e=1) + + # Loop over file twice! + ARGV[2] = ARGV[1] + ARGC++ + # read the "should" state into the `conf` array. while (getline < "/dev/stdin") { word = first($0, EQS) @@ -165,6 +171,8 @@ NR > FNR { } END { + if (e) exit + # print rest of config options ( for (word in conf) print_confs(word) } diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index a33388d..fa676d4 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -141,7 +141,7 @@ then # at the end of the file. cat <<-CODE - awk $(drop_awk_comments "${__type:?}/files/update_dma_conf.awk") $(quote "${dma_conf}") $(quote "${dma_conf}") <<'EOF' >$(quote "${dma_conf}.tmp") \ + awk $(drop_awk_comments "${__type:?}/files/update_dma_conf.awk") $(quote "${dma_conf}") <<'EOF' >$(quote "${dma_conf}.tmp") \ && cat $(quote "${dma_conf}.tmp") >$(quote "${dma_conf}") ${conf_should} EOF From c0b3f792291e8ffb0988e7b21f59270616a163ff Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 16 Nov 2020 19:32:10 +0100 Subject: [PATCH 168/332] __nextcloud{,_user}: Updated manpages --- type/__nextcloud/man.rst | 3 ++- type/__nextcloud_user/man.rst | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/type/__nextcloud/man.rst b/type/__nextcloud/man.rst index 39de9e8..1e0238d 100644 --- a/type/__nextcloud/man.rst +++ b/type/__nextcloud/man.rst @@ -33,7 +33,8 @@ version admin-password The administrator password to access the nextcloud instance. Must be given - in plain text. + in plain text. This parameter has no effect if nextcloud will not be + installed. OPTIONAL PARAMETERS diff --git a/type/__nextcloud_user/man.rst b/type/__nextcloud_user/man.rst index 0b303d6..200769f 100644 --- a/type/__nextcloud_user/man.rst +++ b/type/__nextcloud_user/man.rst @@ -64,6 +64,9 @@ password If you do not want to modify the user password, set a password via this parameter and set the parameter `--keep-password`. + Note that Nextcloud will check for the security of passwords. The type + will abort if Nextcloud refuses that password! + quota The quota the Nextcloud user have to store it data. Defaults to `default`. Following values are accepted by Nextcloud: From c89bb59409a90bafead5a6dd323fd2d87ccc55db Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 20 Nov 2020 19:23:40 +0100 Subject: [PATCH 169/332] __nextcloud: add database maintainer command for upgrades After my update to 20.0.2, Nextcloud complained about missing primary key indexes. I don't know if it existed before that version, but adding it to make life easier .. as no one will propably install an older version than Nextcloud 20.0.2. --- type/__nextcloud/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 00611a3..4ead4a9 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -93,6 +93,8 @@ REMOTE # do some maintainer stuff occ upgrade + # gamble a bit with database maintainer commands + occ db:add-missing-primary-keys occ db:add-missing-columns occ db:add-missing-indices From 1b2d41a34aa60fb90b4f145ae2776c4807550a4d Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Thu, 17 Dec 2020 13:29:03 +0100 Subject: [PATCH 170/332] Initial implementation of the __pass type. --- type/__pass/gencode-local | 71 ++++++++++++++++++++++ type/__pass/man.rst | 79 +++++++++++++++++++++++++ type/__pass/parameter/boolean | 1 + type/__pass/parameter/optional | 1 + type/__pass/parameter/required | 1 + type/__pass/parameter/required_multiple | 1 + 6 files changed, 154 insertions(+) create mode 100755 type/__pass/gencode-local create mode 100644 type/__pass/man.rst create mode 100644 type/__pass/parameter/boolean create mode 100644 type/__pass/parameter/optional create mode 100644 type/__pass/parameter/required create mode 100644 type/__pass/parameter/required_multiple diff --git a/type/__pass/gencode-local b/type/__pass/gencode-local new file mode 100755 index 0000000..e9e983c --- /dev/null +++ b/type/__pass/gencode-local @@ -0,0 +1,71 @@ +#!/bin/sh -e +# +# 2020 Joachim Desroches (joachim.desroches@epfl.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# Length of generated password. +LENGTH= + +# Keep password strictly alphanumeric. +NOSYMB= + +# Check pass is installed. +command -v pass >/dev/null 2>&1 || + { + cat <<- EOF >&2 + __pass: this type requires pass installed. + See https://www.passwordstore.org/. + EOF + exit 1; + } + +# Check for optional length parameter. +if [ -f "${__object:?}/parameter/length" ]; +then + LENGTH="$(cat "${__object:?}/parameter/length")" +fi + +# Check for optional no symbols parameter. +if [ -f "${__object:?}/parameter/no-symbols" ]; +then + NOSYMB="-n" +fi + +# Load required GPG ID parameters. +set -- +while read -r id; +do + set -- "$@" "$id" +done < "${__object:?}/parameter/gpgid" + +# Load required password store location parameter. +PASSWORD_STORE_DIR="$(cat "${__object:?}/parameter/storedir")" +export PASSWORD_STORE_DIR + +# Run every time in case GPG IDs are updated. +pass init "$@" >/dev/null + +# Generate a password if it does not already exist. +if [ ! -f "${PASSWORD_STORE_DIR}/${__object_id:?}.gpg" ]; +then + # shellcheck disable=SC2086 + pass generate $NOSYMB "${__object_id:?}" $LENGTH +fi + +# Send it out to the messages. +pass "${__object_id:?}" >> "${__messages_out:?}" diff --git a/type/__pass/man.rst b/type/__pass/man.rst new file mode 100644 index 0000000..60bc6f8 --- /dev/null +++ b/type/__pass/man.rst @@ -0,0 +1,79 @@ +cdist-type__pass(7) +=================== + +NAME +---- +cdist-type__pass - Generate and use passwords using pass(1). + + +DESCRIPTION +----------- +This type allows a user to generate and query passwords stored using pass(1) on +the host machine. The password is then printed to the cdist message system, so +types depending on this one should require it. This enables an administrator to +ensure a password exists using this type and then, from another type, use it as +need be. + +This type also sets the GPG IDs used to encrypt the password store: beware that +the IDs passed in the last ran invocation of the type will be the ones set for +the store. + +REQUIRED PARAMETERS +------------------- +storedir + The host-local directory where the password store is to be found (or + created if it does not exist). + + +REQUIRED MULTIPLE PARAMETERS +---------------------------- +gpgid + The GPG IDs of the public keys used to encrypt the password store. + +OPTIONAL PARAMETERS +------------------- +length + The length of the password to be created if it does not exist. Note that if + it exists, this has no effect (and hence will not update the password, even + if the length is different from the one specified). + +BOOLEAN PARAMETERS +------------------ +no-symbols + If this parameter is set, then a newly generated password will only contain + alphanumeric characters, making it easier for typing by meatware. + + +EXAMPLES +-------- + +Assuming that __othertype takes the path of the password as an argument and +looks up in the cdist messages to find it: + +.. code-block:: sh + + __pass database/services/arandomservice + --storedir password/store/location + --gpgpid 92296965EAA1DD86A93284EF7B21E5AA32FB9810 + + require='__pass/database/services/arandomservice' \ + __othertype --password database/service/arandomservice + +-- + +SEE ALSO +-------- +`pass`\ (7) + + +AUTHORS +------- +Joachim Desroches + + +COPYING +------- +Copyright \(C) 2020 Joachim Desroches. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__pass/parameter/boolean b/type/__pass/parameter/boolean new file mode 100644 index 0000000..8be3749 --- /dev/null +++ b/type/__pass/parameter/boolean @@ -0,0 +1 @@ +no-symbols diff --git a/type/__pass/parameter/optional b/type/__pass/parameter/optional new file mode 100644 index 0000000..7f5e3b6 --- /dev/null +++ b/type/__pass/parameter/optional @@ -0,0 +1 @@ +length diff --git a/type/__pass/parameter/required b/type/__pass/parameter/required new file mode 100644 index 0000000..f2fc3a2 --- /dev/null +++ b/type/__pass/parameter/required @@ -0,0 +1 @@ +storedir diff --git a/type/__pass/parameter/required_multiple b/type/__pass/parameter/required_multiple new file mode 100644 index 0000000..bed3d4a --- /dev/null +++ b/type/__pass/parameter/required_multiple @@ -0,0 +1 @@ +gpgid From b6db7b5ca8daa8c4febfbbee0bd8ef8c23adade8 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 21 Dec 2020 19:04:53 +0100 Subject: [PATCH 171/332] __nextcloud: suppress global warnings from occ As it's the default for __nextcloud_app and __nextcloud_user, it will now suppress global warnings, too. This will be done because nobody cares about then, only about the errors generated from the executed command. --- type/__nextcloud/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 4ead4a9..83748f5 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -17,7 +17,7 @@ occ() { # will not use -q as it supresses errors, too cat << SHELL su -s /bin/sh -l "$user" -- -e < Date: Fri, 25 Dec 2020 11:44:32 +0100 Subject: [PATCH 172/332] __nextcloud: harden here-doc for su call Because the here-doc will be evaluated again by the shell, it will was changed to avoid this. Because the script does not depend on such function, it will be done in faviour of special characters. --- type/__nextcloud/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 83748f5..15ff966 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -16,7 +16,7 @@ occ() { # su creates a new shell, so it does not affect the current session # will not use -q as it supresses errors, too cat << SHELL -su -s /bin/sh -l "$user" -- -e < Date: Fri, 25 Dec 2020 20:14:04 +0100 Subject: [PATCH 173/332] __nextcloud: add db:convert-filecache-bigint command Looks like this command should be executed on upgrades, too. --- type/__nextcloud/gencode-remote | 1 + 1 file changed, 1 insertion(+) diff --git a/type/__nextcloud/gencode-remote b/type/__nextcloud/gencode-remote index 15ff966..6812c68 100755 --- a/type/__nextcloud/gencode-remote +++ b/type/__nextcloud/gencode-remote @@ -97,6 +97,7 @@ REMOTE occ db:add-missing-primary-keys occ db:add-missing-columns occ db:add-missing-indices + occ db:convert-filecache-bigint # send upgrade message printf "upgraded %s to %s\n" "$version_is" "$version_should" >> "$__messages_out" From 006987307f9a93d8297260becea880e27bceb347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 09:43:37 +0100 Subject: [PATCH 174/332] __unbound: make the --dns64-prefix flag optional --- type/__unbound/files/unbound.conf.sh | 16 ++++++++++++++-- type/__unbound/man.rst | 6 +++--- type/__unbound/manifest | 7 +++++-- type/__unbound/parameter/required | 1 - 4 files changed, 22 insertions(+), 8 deletions(-) delete mode 100644 type/__unbound/parameter/required diff --git a/type/__unbound/files/unbound.conf.sh b/type/__unbound/files/unbound.conf.sh index 2f7ecff..39443d8 100755 --- a/type/__unbound/files/unbound.conf.sh +++ b/type/__unbound/files/unbound.conf.sh @@ -1,5 +1,10 @@ #!/bin/sh +module_config="validator iterator" +if [ -n "$DNS64_PREFIX" ]; then + module_config="dns64 $module_config" +fi + generate_interface() { for i in $INTERFACES; do echo " interface: $i" @@ -26,6 +31,12 @@ generate_local_data() { done } +generate_dns64_prefix() { + if [ -n "$DNS64_PREFIX" ]; then + echo " dns64-prefix: $DNS64_PREFIX" + fi +} + cat << EOF # # THIS FILE HAS BEEN GENERATED BY CDIST, DO NOT EDIT BY HAND. @@ -506,7 +517,7 @@ $(generate_access_control) # most modules have to be listed at the beginning of the line, # except cachedb(just before iterator), and python (at the beginning, # or, just before the iterator). - module-config: "dns64 validator iterator" + module-config: "$module_config" # File with trusted keys, kept uptodate using RFC5011 probes, # initial file like trust-anchor-file, then it stores metadata. @@ -776,7 +787,8 @@ $(generate_local_data) # DNS64 prefix. Must be specified when DNS64 is use. # Enable dns64 in module-config. Used to synthesize IPv6 from IPv4. - dns64-prefix: $DNS64_PREFIX + # dns64-prefix: $DNS64_PREFIX" + $(generate_dns64_prefix) # DNS64 ignore AAAA records for these domains and use A instead. # dns64-ignore-aaaa: "example.com" diff --git a/type/__unbound/man.rst b/type/__unbound/man.rst index 316d011..05fce45 100644 --- a/type/__unbound/man.rst +++ b/type/__unbound/man.rst @@ -13,9 +13,6 @@ answers from specified upstrean DNS server. This is a singleton type. REQUIRED PARAMETERS ------------------- -dns64_prefix - IPv6 prefix used for DNS64. - forward_addr DNS servers used to lookup names, can be provided multiple times. It can be either an IPv4 or IPv6 address but no domain name. @@ -38,6 +35,9 @@ local_data Configure local data, which is served in reply to queries for it. Can be specified multiple times. +dns64_prefix + Enable DNS64 with specified prefix. + BOOLEAN PARAMETERS ------------------ disable-ip4 diff --git a/type/__unbound/manifest b/type/__unbound/manifest index 5d6b50f..53a8ab6 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -33,12 +33,15 @@ case "$os" in esac # Required parameters: -DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") -export DNS64_PREFIX FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") export FORWARD_ADDRS # Optional parameters: +if [ -f "$__object/parameter/dns64_prefix" ]; then + DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") + export DNS64_PREFIX +fi + if [ -f "$__object/parameter/interface" ]; then INTERFACES=$(cat "$__object/parameter/interface") export INTERFACES diff --git a/type/__unbound/parameter/required b/type/__unbound/parameter/required deleted file mode 100644 index 4ad232f..0000000 --- a/type/__unbound/parameter/required +++ /dev/null @@ -1 +0,0 @@ -dns64_prefix From 9f4a85901ef8d58e95924dd9f584ca6437934030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 11:04:07 +0100 Subject: [PATCH 175/332] Import html/man doc generation logic from upstream cdist --- .gitlab-ci.yml | 10 +- Makefile | 70 ++++++++++ docs/src/Makefile | 235 ++++++++++++++++++++++++++++++++++ docs/src/conf.py | 101 +++++++++++++++ docs/src/index.rst.sh | 45 +++++++ docs/src/manpage.py | 87 +++++++++++++ type/__matrix_element/man.rst | 2 +- type/__matterbridge/man.rst | 4 +- type/__netbox/man.rst | 5 +- type/__netbox_uwsgi/man.rst | 5 +- 10 files changed, 552 insertions(+), 12 deletions(-) create mode 100644 Makefile create mode 100644 docs/src/Makefile create mode 100644 docs/src/conf.py create mode 100755 docs/src/index.rst.sh create mode 100644 docs/src/manpage.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dba7864..79c532e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,20 @@ stages: - test + - doc image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest shellcheck: stage: test script: - - ./scripts/run-shellcheck.sh + - make lint manpages: stage: test script: - - ./scripts/run-manpage-checks.sh + - make check-manpages + +docs: + stage: doc + script: + - make docs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1a0cfb3 --- /dev/null +++ b/Makefile @@ -0,0 +1,70 @@ +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo "man build only man user documentation" + @echo "html build only html user documentation" + @echo "docs build both man and html user documentation" + @echo "check-manpages check for manpage in types" + @echo "lint run shellcheck on types" + @echo "check run both type manpage checks and linting" + @echo "clean clean" + +DOCS_SRC_DIR=./docs/src +TYPEDIR=./type + +SPHINXM=make -C $(DOCS_SRC_DIR) man +SPHINXH=make -C $(DOCS_SRC_DIR) html +SPHINXC=make -C $(DOCS_SRC_DIR) clean + +################################################################################ +# Manpages +# +MAN7DSTDIR=$(DOCS_SRC_DIR)/man7 + +# Use shell / ls to get complete list - $(TYPEDIR)/*/man.rst does not work +# Using ls does not work if no file with given pattern exist, so use wildcard +MANTYPESRC=$(wildcard $(TYPEDIR)/*/man.rst) +MANTYPEPREFIX=$(subst $(TYPEDIR)/,$(MAN7DSTDIR)/cdist-type,$(MANTYPESRC)) +MANTYPES=$(subst /man.rst,.rst,$(MANTYPEPREFIX)) + +# Link manpage: do not create man.html but correct named file +$(MAN7DSTDIR)/cdist-type%.rst: $(TYPEDIR)/%/man.rst + mkdir -p $(MAN7DSTDIR) + ln -sf "../../../$^" $@ + +DOCSINDEX=$(MAN7DSTDIR)/index.rst +DOCSINDEXH=$(DOCS_SRC_DIR)/index.rst.sh + +$(DOCSINDEX): $(DOCSINDEXH) + $(DOCSINDEXH) + +# Manpages: .cdist Types +DOT_CDIST_PATH=${HOME}/.cdist +DOTMAN7DSTDIR=$(MAN7DSTDIR) +DOTTYPEDIR=$(DOT_CDIST_PATH)/type + +# Link manpage: do not create man.html but correct named file +$(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst + ln -sf "$^" $@ + +man: $(MANTYPES) $(DOCSINDEX) + $(SPHINXM) + +html: $(MANTYPES) $(DOCSINDEX) + $(SPHINXH) + +docs: man html + +check-manpages: + ./scripts/run-manpage-checks.sh + +lint: + ./scripts/run-shellcheck.sh + +check: check-manpages lint + +clean: + $(SPHINXC) + rm -f docs/src/index.rst + rm -rf docs/src/man7/ + rm -rf docs/src/__pycache__/ diff --git a/docs/src/Makefile b/docs/src/Makefile new file mode 100644 index 0000000..2e9d6ce --- /dev/null +++ b/docs/src/Makefile @@ -0,0 +1,235 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +PAPER ?= +BUILDDIR ?= ../dist +# for cache, etc. +_BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) + $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(_BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " epub3 to make an epub3" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + rm -rf $(_BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/cdist-docs.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/cdist-docs.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/cdist-docs" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/cdist-docs" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b cman $(ALLSPHINXOPTS) $(BUILDDIR)/man + mkdir -p $(BUILDDIR)/man/man7 + mv -f $(BUILDDIR)/man/*.7 $(BUILDDIR)/man/man7/ + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/src/conf.py b/docs/src/conf.py new file mode 100644 index 0000000..19b2dfd --- /dev/null +++ b/docs/src/conf.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +import sys +import os +import sphinx_rtd_theme + +from datetime import date + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", ".."))) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'docs.src.manpage', + 'sphinx.ext.extlinks', +] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = ['.rst'] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'cdist-contrib' +copyright = 'cdist-contrib contributors' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. + +version = str(date.today()) +release = os.popen('git rev-parse HEAD').read() + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' +html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + +# Output file base name for HTML help builder. +htmlhelp_basename = 'cdistcontribdoc' + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +root_mandir = os.path.dirname(os.path.realpath(__file__)) +mandirs = [] +for mansubdir in ('man7',): + mandirs.append((os.path.join(root_mandir, mansubdir), mansubdir[-1])) +man_pages = [] +for mandir, section in mandirs: + for root, dirs, files in os.walk(mandir): + for fname in files: + froot, fext = os.path.splitext(fname) + if fext == '.rst': + man_page = (os.path.join('man' + str(section), froot), + froot, '', [], section) + man_pages.append(man_page) + +# man_pages = [ +# ('cdist-type', 'cdist-type', 'cdist-type documentation', +# [author], 1), +# ('man7/cdist-type__file', 'cdist-type__file', +# '', [], 1), +# ('cdist-type__directory', 'cdist-type__directory', +# 'cdist-type__directory documentation', [author], 1), +# ] + +# If true, show URL addresses after external links. +# man_show_urls = False diff --git a/docs/src/index.rst.sh b/docs/src/index.rst.sh new file mode 100755 index 0000000..28c5078 --- /dev/null +++ b/docs/src/index.rst.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +__cdist_pwd="$(pwd -P)" +__cdist_mydir="${0%/*}"; +__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)" +__cdist_myname=${0##*/}; +__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" + +filename="${__cdist_myname%.sh}" +dest="$__cdist_abs_mydir/$filename" + +cd "$__cdist_abs_mydir" + +exec > "$dest" +cat << EOF +cdist-contrib - Community maintained cdist types +================================================ + +This project extends the \`cdist \`_ configuration management +tool with community-maitained types which are either too specific to fit/be +maintained in cdist itself or were not accepted in code cdist but could still +be useful. + +Please note this project is a **rolling release**! The documentation you're +reading has been generated from the |version| state (commit |release|). +Sources are available on \`code.ungleich.ch +\`_. + + +.. toctree:: + :hidden: + +EOF + +# If there is no such file then ls prints error to stderr, +# so redirect stderr to /dev/null. +for type in $(ls man7/cdist-type__*.rst 2>/dev/null | LC_ALL=C sort); do + no_dir="${type#man7/}"; + no_type="${no_dir#cdist-type}"; + name="${no_type%.rst}"; + manref="${no_dir%.rst}" + man="${manref}(7)" + + echo " $name" "" +done diff --git a/docs/src/manpage.py b/docs/src/manpage.py new file mode 100644 index 0000000..1f8ac4f --- /dev/null +++ b/docs/src/manpage.py @@ -0,0 +1,87 @@ +import sphinx.builders.manpage +import sphinx.writers.manpage +from docutils.frontend import OptionParser +from sphinx.util.console import bold, darkgreen +from six import string_types +from docutils.io import FileOutput +from os import path +from sphinx.util.nodes import inline_all_toctrees +from sphinx import addnodes +from sphinx.util import logging + +""" + Extension based on sphinx builtin manpage. + It does not write its own .SH NAME based on config, + but leaves everything to actual reStructuredText file content. +""" + + +logger = logging.getLogger(__name__) + + +class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator): + + def header(self): + tmpl = (".TH \"%(title_upper)s\" \"%(manual_section)s\"" + " \"%(date)s\" \"%(version)s\" \"%(manual_group)s\"\n") + return tmpl % self._docinfo + + +class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): + + def __init__(self, builder): + super().__init__(builder) + self.translator_class = ( + self.builder.get_translator_class() or ManualPageTranslator) + + +class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): + + name = 'cman' + default_translator_class = ManualPageTranslator + + def write(self, *ignored): + docwriter = ManualPageWriter(self) + docsettings = OptionParser( + defaults=self.env.settings, + components=(docwriter,), + read_config_files=True).get_default_values() + + logger.info(bold('writing... '), nonl=True) + + for info in self.config.man_pages: + docname, name, description, authors, section = info + if isinstance(authors, string_types): + if authors: + authors = [authors] + else: + authors = [] + + targetname = '%s.%s' % (name, section) + logger.info(darkgreen(targetname) + ' { ', nonl=True) + destination = FileOutput( + destination_path=path.join(self.outdir, targetname), + encoding='utf-8') + + tree = self.env.get_doctree(docname) + docnames = set() + largetree = inline_all_toctrees(self, docnames, docname, tree, + darkgreen, [docname]) + logger.info('} ', nonl=True) + self.env.resolve_references(largetree, docname, self) + # remove pending_xref nodes + for pendingnode in largetree.traverse(addnodes.pending_xref): + pendingnode.replace_self(pendingnode.children) + + largetree.settings = docsettings + largetree.settings.title = name + largetree.settings.subtitle = description + largetree.settings.authors = authors + largetree.settings.section = section + + docwriter.write(largetree, destination) + logger.info("") + + +def setup(app): + app.add_builder(ManualPageBuilder) diff --git a/type/__matrix_element/man.rst b/type/__matrix_element/man.rst index c3e0d5a..05f0685 100644 --- a/type/__matrix_element/man.rst +++ b/type/__matrix_element/man.rst @@ -1,5 +1,5 @@ cdist-type__matrix_element(7) -====================== +============================= NAME ---- diff --git a/type/__matterbridge/man.rst b/type/__matterbridge/man.rst index 02eef2c..babf34b 100644 --- a/type/__matterbridge/man.rst +++ b/type/__matterbridge/man.rst @@ -33,9 +33,9 @@ EXAMPLES .. code-block:: sh - __matterbridge --version 1.16.3 --config - << EOF + __matterbridge --version 1.16.3 --config - <<- EOF [...] -EOF + EOF SEE ALSO diff --git a/type/__netbox/man.rst b/type/__netbox/man.rst index 135304c..5f78f1d 100644 --- a/type/__netbox/man.rst +++ b/type/__netbox/man.rst @@ -122,7 +122,6 @@ basepath webroot ``/``. For example, if installed at https://example.com/netbox/, set the value ``netbox/``. -http-proxy https-proxy Proxy which will be used with any HTTP request like webhooks. @@ -171,12 +170,12 @@ redis-ssl smtp-use-tls Uses TLS to connect to the SMTP email server. `See documentation - `_ + `__ for more information. smtp-use-ssl Uses implicit TLS with the SMTP email server. `See documentation - `_ + `__ for more information. login-required diff --git a/type/__netbox_uwsgi/man.rst b/type/__netbox_uwsgi/man.rst index c5e51ca..13dc6dc 100644 --- a/type/__netbox_uwsgi/man.rst +++ b/type/__netbox_uwsgi/man.rst @@ -65,10 +65,7 @@ protocol parameter. Possible values are ``uwsgi``, ``http``, ``fastcgi`` and ``scgi``. If nothing given, it defaults to ``uwsgi``. -uwsgi-bind -http-bind -fastcgi-bind -scgi-bind +scgi-bind, uwsgi-bind, http-bind, fastcgi-bind Bind the application to a specific protocol instead of implicit uwsgi via ``--bind-to``. If such parameter given, ``--bind-to`` will be ignored. Must be a UNIX/TCP socket. Can be set multiple times. From cd833363224493e8a63d7b0b02a83958ffc97663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 11:26:45 +0100 Subject: [PATCH 176/332] Use our own image for CI --- .gitlab-ci.yml | 2 +- scripts/ci-container/Dockerfile | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 scripts/ci-container/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79c532e..31505a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ stages: - test - doc -image: code.ungleich.ch:5050/ungleich-public/cdist/cdist-ci:latest +image: code.ungleich.ch:5050/ungleich-public/cdist-contrib/ci-container:latest shellcheck: stage: test diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile new file mode 100644 index 0000000..33646f4 --- /dev/null +++ b/scripts/ci-container/Dockerfile @@ -0,0 +1,7 @@ +# This image is used in the cdist-contrib CI for linting and generating the +# documentation. +FROM fedora:latest +MAINTAINER Timothée Floure + +RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ + ShellCheck From 5a05b87bebb85c30d1af4878c0b5e134737df225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 09:46:10 +0100 Subject: [PATCH 177/332] __unbound: normalize flag format --- type/__unbound/gencode-remote | 2 +- type/__unbound/man.rst | 22 +++++++-------- type/__unbound/manifest | 28 +++++++++---------- type/__unbound/parameter/boolean | 6 ++-- .../default/{rc_interface => rc-interface} | 0 type/__unbound/parameter/optional | 2 +- type/__unbound/parameter/optional_multiple | 4 +-- type/__unbound/parameter/required_multiple | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) rename type/__unbound/parameter/default/{rc_interface => rc-interface} (100%) diff --git a/type/__unbound/gencode-remote b/type/__unbound/gencode-remote index ba6d92b..7c271b7 100755 --- a/type/__unbound/gencode-remote +++ b/type/__unbound/gencode-remote @@ -2,7 +2,7 @@ UNBOUND_CERTS_DIR=/etc/unbound -if [ -f "$__object/parameter/enable_rc" ]; then +if [ -f "$__object/parameter/enable-rc" ]; then echo "unbound-control-setup -d $UNBOUND_CERTS_DIR" echo "chown unbound:unbound $UNBOUND_CERTS_DIR/*.pem $UNBOUND_CERTS_DIR/*.key" fi diff --git a/type/__unbound/man.rst b/type/__unbound/man.rst index 05fce45..350f1ea 100644 --- a/type/__unbound/man.rst +++ b/type/__unbound/man.rst @@ -23,19 +23,19 @@ interface Interface to listen on, can be provided multiple times. Defaults to '127.0.0.1' and '::1'. -access_control +access-control Controls which clients are allowed queries to the unbound service (everything but localhost is refused by default), can be provided multiple times. The format is described in unbound.conf(5). -rc_interface +rc-interface Address or path to socket used for remote control (see `--enable_control`. Defaults to `127.0.0.1`). -local_data +local-data Configure local data, which is served in reply to queries for it. Can be specified multiple times. -dns64_prefix +dns64-prefix Enable DNS64 with specified prefix. BOOLEAN PARAMETERS @@ -48,7 +48,7 @@ disable-ip6 Do not answer or issue queries over IPv6. Cannot be used alongside the `--disable-ip4` flag. -enable_rc +enable-rc Enable remote control (see `unbound-control(8)`). EXAMPLES @@ -58,12 +58,12 @@ EXAMPLES __ungleich_unbound \ --interface '::0' \ - --dns64_prefix '2a0a:e5c0:2:10::/96' \ - --forward_addr '2a0a:e5c0:2:1::5' \ - --forward_addr '2a0a:e5c0:2:1::6' \ - --access_control '::0/0 deny' \ - --access_control '2a0a:e5c0::/29 allow' \ - --access_control '2a09:2940::/29 allow' \ + --dns64-prefix '2a0a:e5c0:2:10::/96' \ + --forward-addr '2a0a:e5c0:2:1::5' \ + --forward-addr '2a0a:e5c0:2:1::6' \ + --access-control '::0/0 deny' \ + --access-control '2a0a:e5c0::/29 allow' \ + --access-control '2a09:2940::/29 allow' \ --ip6 SEE ALSO diff --git a/type/__unbound/manifest b/type/__unbound/manifest index 53a8ab6..1a23328 100755 --- a/type/__unbound/manifest +++ b/type/__unbound/manifest @@ -33,12 +33,12 @@ case "$os" in esac # Required parameters: -FORWARD_ADDRS=$(cat "$__object/parameter/forward_addr") +FORWARD_ADDRS=$(cat "$__object/parameter/forward-addr") export FORWARD_ADDRS # Optional parameters: -if [ -f "$__object/parameter/dns64_prefix" ]; then - DNS64_PREFIX=$(cat "$__object/parameter/dns64_prefix") +if [ -f "$__object/parameter/dns64-prefix" ]; then + DNS64_PREFIX=$(cat "$__object/parameter/dns64-prefix") export DNS64_PREFIX fi @@ -47,41 +47,41 @@ if [ -f "$__object/parameter/interface" ]; then export INTERFACES fi -if [ -f "$__object/parameter/access_control" ]; then - ACCESS_CONTROLS=$(cat "$__object/parameter/access_control") +if [ -f "$__object/parameter/access-control" ]; then + ACCESS_CONTROLS=$(cat "$__object/parameter/access-control") export ACCESS_CONTROLS fi -if [ -f "$__object/parameter/rc_interface" ]; then - RC_INTERFACE=$(cat "$__object/parameter/rc_interface") +if [ -f "$__object/parameter/rc-interface" ]; then + RC_INTERFACE=$(cat "$__object/parameter/rc-interface") export RC_INTERFACE fi -if [ -f "$__object/parameter/local_data" ]; then - LOCAL_DATA=$(cat "$__object/parameter/local_data") +if [ -f "$__object/parameter/local-data" ]; then + LOCAL_DATA=$(cat "$__object/parameter/local-data") export LOCAL_DATA fi # Boolean parameters: -if [ -f "$__object/parameter/disable_ip4" ] && \ - [ -f "$__object/parameter/disable_ip6" ]; then +if [ -f "$__object/parameter/disable-ip4" ] && \ + [ -f "$__object/parameter/disable-ip6" ]; then echo "--disable-ip4 and --disable-ip6 cannot be used at the same time." >&2 exit 1 fi -if [ -f "$__object/parameter/disable_ip4" ]; then +if [ -f "$__object/parameter/disable-ip4" ]; then export DO_IP4='no' else export DO_IP4='yes' fi -if [ -f "$__object/parameter/disable_ip6" ]; then +if [ -f "$__object/parameter/disable-ip6" ]; then export DO_IP6='no' else export DO_IP6='yes' fi -if [ -f "$__object/parameter/enable_rc" ]; then +if [ -f "$__object/parameter/enable-rc" ]; then export RC_ENABLE='yes' else export RC_ENABLE='no' diff --git a/type/__unbound/parameter/boolean b/type/__unbound/parameter/boolean index b6e53a1..f754fa2 100644 --- a/type/__unbound/parameter/boolean +++ b/type/__unbound/parameter/boolean @@ -1,3 +1,3 @@ -disable_ip6 -disable_ip4 -enable_rc +disable-ip6 +disable-ip4 +enable-rc diff --git a/type/__unbound/parameter/default/rc_interface b/type/__unbound/parameter/default/rc-interface similarity index 100% rename from type/__unbound/parameter/default/rc_interface rename to type/__unbound/parameter/default/rc-interface diff --git a/type/__unbound/parameter/optional b/type/__unbound/parameter/optional index 0826b6e..86a4400 100644 --- a/type/__unbound/parameter/optional +++ b/type/__unbound/parameter/optional @@ -1 +1 @@ -rc_interface +rc-interface diff --git a/type/__unbound/parameter/optional_multiple b/type/__unbound/parameter/optional_multiple index 3fe7eca..4d06e03 100644 --- a/type/__unbound/parameter/optional_multiple +++ b/type/__unbound/parameter/optional_multiple @@ -1,3 +1,3 @@ -access_control -local_data +access-control +local-data interface diff --git a/type/__unbound/parameter/required_multiple b/type/__unbound/parameter/required_multiple index 776abf5..74b99a2 100644 --- a/type/__unbound/parameter/required_multiple +++ b/type/__unbound/parameter/required_multiple @@ -1 +1 @@ -forward_addr +forward-addr From 2bedbe96870d492d7f7d6cd868c210a7ba0e4f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 10:55:25 +0100 Subject: [PATCH 178/332] Automatically deploy docs from master (at contrib.cdi.st) --- .gitlab-ci.yml | 11 +++++++++++ scripts/ci-container/Dockerfile | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 31505a5..4b61568 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,5 +16,16 @@ manpages: docs: stage: doc + only: + - master + before_script: + - eval $(ssh-agent -s) + - echo "$CD_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - mkdir -p ~/.ssh + - echo "$CD_SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts script: - make docs + - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<- EOF + put -r docs/dist/html/* + EOF diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile index 33646f4..daa0f26 100644 --- a/scripts/ci-container/Dockerfile +++ b/scripts/ci-container/Dockerfile @@ -4,4 +4,4 @@ FROM fedora:latest MAINTAINER Timothée Floure RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ - ShellCheck + ShellCheck openssh-clients From ec41ef34901bf164b5decb5fa8fa9f150c131a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:05:09 +0100 Subject: [PATCH 179/332] Patch sftp call for doc deployment in CI --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4b61568..9392cd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,4 @@ docs: - chmod 644 ~/.ssh/known_hosts script: - make docs - - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<- EOF - put -r docs/dist/html/* - EOF + - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<< "put -r docs/dist/html/*" From ec0dc30c87a1b770c647641bb3f3abaa1f735032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:34:54 +0100 Subject: [PATCH 180/332] Generate documentation homepage from README --- .gitlab-ci.yml | 2 +- README.md | 3 ++- docs/src/index.rst.sh | 21 ++++++++------------- scripts/ci-container/Dockerfile | 2 +- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9392cd9..81db798 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,5 +25,5 @@ docs: - echo "$CD_SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts script: - - make docs + - make html - sftp fnux@staticwebhosting.ungleich.ch:public_html/cdist-contrib <<< "put -r docs/dist/html/*" diff --git a/README.md b/README.md index 4231de2..4d20c22 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,8 @@ maintained in cdist itself or were not accepted in code cdist but could still be useful. This project does not have releases and is continously updated: see git history -for change log. +for change log. You will find HTML documentation at +[contrib.cdi.st](https://contrib.cdi.st). ## Using cdist-contrib diff --git a/docs/src/index.rst.sh b/docs/src/index.rst.sh index 28c5078..babc1d9 100755 --- a/docs/src/index.rst.sh +++ b/docs/src/index.rst.sh @@ -9,23 +9,18 @@ __cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" filename="${__cdist_myname%.sh}" dest="$__cdist_abs_mydir/$filename" +if ! command -v pandoc > /dev/null; then + echo "Pandoc is required to generate HTML index from README." >&2 + exit 1 +fi + cd "$__cdist_abs_mydir" exec > "$dest" + +pandoc -f markdown -t rst ../../README.md + cat << EOF -cdist-contrib - Community maintained cdist types -================================================ - -This project extends the \`cdist \`_ configuration management -tool with community-maitained types which are either too specific to fit/be -maintained in cdist itself or were not accepted in code cdist but could still -be useful. - -Please note this project is a **rolling release**! The documentation you're -reading has been generated from the |version| state (commit |release|). -Sources are available on \`code.ungleich.ch -\`_. - .. toctree:: :hidden: diff --git a/scripts/ci-container/Dockerfile b/scripts/ci-container/Dockerfile index daa0f26..9900322 100644 --- a/scripts/ci-container/Dockerfile +++ b/scripts/ci-container/Dockerfile @@ -4,4 +4,4 @@ FROM fedora:latest MAINTAINER Timothée Floure RUN dnf install -y git findutils make python3-sphinx python3-sphinx_rtd_theme \ - ShellCheck openssh-clients + ShellCheck openssh-clients pandoc From 73e31e6d1ee69181f762847e6844c77e6a0b9db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:56:37 +0100 Subject: [PATCH 181/332] Mention known external cdist resources in README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 4d20c22..d9c5e22 100644 --- a/README.md +++ b/README.md @@ -54,3 +54,11 @@ Every type in cdist-contrib must: * Have a `man.rst` documentation page. * Pass [shellcheck](http://shellcheck.net/) without errors. + +## Other resources + +Some people/organizations are known to keep some cdist types that might be of +interest to others: + +* [cdist-evilham](https://git.sr.ht/~evilham/cdist-evilham): Evilham's cdist-types +* [cdist-recycledcloud](https://code.recycled.cloud/e-Durable/cdist-recycledcloud): e-Durable SA / Recycled Cloud public types From f01f73f33d428ecc2e09bc8754d63e512862a61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:58:21 +0100 Subject: [PATCH 182/332] Remove mention to legacy mattermost room from README --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index d9c5e22..28f54db 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,11 @@ And you would run [cdist][cdist] from the same directory as follows: ## Participating in the [cdist][cdist] community -Join us on [#cdist:ungleich.ch][cdistmatrix] on matrix or on -[#cdist over mattermost][cdistmattermost]. - +Join us on [#cdist:ungleich.ch][cdistmatrix] on matrix! [cdist]: https://www.cdi.st/ [cdistconfig]: https://www.cdi.st/manual/latest/cdist-configuration.html [cdistmatrix]: https://matrix.to/#/#cdist:ungleich.ch -[cdistmattermost]: https://chat.ungleich.ch/ungleich/channels/cdist ## Contributing From f9f5c578f744cd157c138fca09ff84616cf805b6 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 11 Jan 2021 12:16:09 +0100 Subject: [PATCH 183/332] [type/__dma*] Fix shellcheck errors --- type/__dma/gencode-remote | 2 +- type/__dma/manifest | 2 +- type/__dma_auth/explorer/state | 2 +- type/__dma_auth/gencode-remote | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/type/__dma/gencode-remote b/type/__dma/gencode-remote index fa676d4..580b22e 100755 --- a/type/__dma/gencode-remote +++ b/type/__dma/gencode-remote @@ -158,7 +158,7 @@ if test -f "${__object:?}/parameter/send-test-mail" then if grep -q '^__mail_alias/root:' "${__messages_in:?}" \ || grep -q '^__dma_auth/' "${__messages_in:?}" \ - || $config_updated + || ${config_updated} then cat <<-CODE sendmail root <<'EOF' diff --git a/type/__dma/manifest b/type/__dma/manifest index 2cbd1a5..530ad09 100755 --- a/type/__dma/manifest +++ b/type/__dma/manifest @@ -21,7 +21,7 @@ os=$(cat "${__global:?}/explorer/os") # Install DMA -case $os +case ${os} in (alpine) __package dma --state present diff --git a/type/__dma_auth/explorer/state b/type/__dma_auth/explorer/state index 621e5a2..c829cd4 100755 --- a/type/__dma_auth/explorer/state +++ b/type/__dma_auth/explorer/state @@ -25,7 +25,7 @@ # different_password: a line exists but with a different password # multiple: multiple lines matching host exist (should not happen) -auth_conf=$("${__type_explorer}/auth_conf") +auth_conf=$("${__type_explorer:?}/auth_conf") test -r "${auth_conf}" || exit 0 awk -F'\n' ' diff --git a/type/__dma_auth/gencode-remote b/type/__dma_auth/gencode-remote index 4279b7a..b6a0100 100755 --- a/type/__dma_auth/gencode-remote +++ b/type/__dma_auth/gencode-remote @@ -40,7 +40,7 @@ then exit 0 fi -case $state_should +case ${state_should} in (present) test -n "${login}" || { echo '--login must be non-empty' >&2; exit 1; } From f055527735e86f5ec280d08945b56cfd08e35959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 11:52:14 +0100 Subject: [PATCH 184/332] Import evilham's jitsi types as-it --- .../prometheus-jitsi-meet-explorer-version | 7 + .../files/debconf_settings.sh | 56 ++ type/__evilham_jitsi_meet/files/ufw | 10 + type/__evilham_jitsi_meet/gencode-remote | 5 + type/__evilham_jitsi_meet/man.rst | 84 +++ type/__evilham_jitsi_meet/manifest | 197 +++++ type/__evilham_jitsi_meet/parameter/boolean | 1 + .../parameter/default/turn-server | 0 type/__evilham_jitsi_meet/parameter/optional | 2 + type/__evilham_jitsi_meet/singleton | 0 .../files/config.js.sh | 700 ++++++++++++++++++ .../files/config.js.sh.orig | 694 +++++++++++++++++ .../files/nginx.sh | 156 ++++ .../files/nginx.sh.orig | 139 ++++ .../gencode-remote | 5 + type/__evilham_jitsi_meet_domain/man.rst | 151 ++++ type/__evilham_jitsi_meet_domain/manifest | 88 +++ .../parameter/boolean | 2 + .../parameter/default/branding-index | 0 .../parameter/default/branding-json | 0 .../parameter/default/branding-watermark | 0 .../parameter/default/channel-last-n | 1 + .../parameter/default/default-language | 1 + .../parameter/default/notice-message | 0 .../parameter/default/start-video-muted | 1 + .../parameter/default/turn-server | 0 .../parameter/default/video-constraints | 0 .../parameter/optional | 9 + .../parameter/required | 1 + 29 files changed, 2310 insertions(+) create mode 100755 type/__evilham_jitsi_meet/explorer/prometheus-jitsi-meet-explorer-version create mode 100644 type/__evilham_jitsi_meet/files/debconf_settings.sh create mode 100644 type/__evilham_jitsi_meet/files/ufw create mode 100755 type/__evilham_jitsi_meet/gencode-remote create mode 100644 type/__evilham_jitsi_meet/man.rst create mode 100755 type/__evilham_jitsi_meet/manifest create mode 100644 type/__evilham_jitsi_meet/parameter/boolean create mode 100644 type/__evilham_jitsi_meet/parameter/default/turn-server create mode 100644 type/__evilham_jitsi_meet/parameter/optional create mode 100644 type/__evilham_jitsi_meet/singleton create mode 100644 type/__evilham_jitsi_meet_domain/files/config.js.sh create mode 100644 type/__evilham_jitsi_meet_domain/files/config.js.sh.orig create mode 100644 type/__evilham_jitsi_meet_domain/files/nginx.sh create mode 100644 type/__evilham_jitsi_meet_domain/files/nginx.sh.orig create mode 100755 type/__evilham_jitsi_meet_domain/gencode-remote create mode 100644 type/__evilham_jitsi_meet_domain/man.rst create mode 100755 type/__evilham_jitsi_meet_domain/manifest create mode 100644 type/__evilham_jitsi_meet_domain/parameter/boolean create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/branding-index create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/branding-json create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/branding-watermark create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/channel-last-n create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/default-language create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/notice-message create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/start-video-muted create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/turn-server create mode 100644 type/__evilham_jitsi_meet_domain/parameter/default/video-constraints create mode 100644 type/__evilham_jitsi_meet_domain/parameter/optional create mode 100644 type/__evilham_jitsi_meet_domain/parameter/required diff --git a/type/__evilham_jitsi_meet/explorer/prometheus-jitsi-meet-explorer-version b/type/__evilham_jitsi_meet/explorer/prometheus-jitsi-meet-explorer-version new file mode 100755 index 0000000..b1cec48 --- /dev/null +++ b/type/__evilham_jitsi_meet/explorer/prometheus-jitsi-meet-explorer-version @@ -0,0 +1,7 @@ +#!/bin/sh -e + +EXPORTER_VERSION_FILE="/usr/local/bin/.prometheus-jitsi-meet-exporter.cdist.version" + +if [ -f "${EXPORTER_VERSION_FILE}" ]; then + cat "${EXPORTER_VERSION_FILE}" +fi diff --git a/type/__evilham_jitsi_meet/files/debconf_settings.sh b/type/__evilham_jitsi_meet/files/debconf_settings.sh new file mode 100644 index 0000000..9e358f0 --- /dev/null +++ b/type/__evilham_jitsi_meet/files/debconf_settings.sh @@ -0,0 +1,56 @@ +#!/bin/sh -e + +# This can be obtained with debconf-get-selections on a host with jitsi +# (and also analysing the deb-src) +if false; then + # We are currently not using these, just here as documentation + DEBCONF_SETTINGS="$(cat < + + +COPYING +------- +Copyright \(C) 2020 Evilham. diff --git a/type/__evilham_jitsi_meet/manifest b/type/__evilham_jitsi_meet/manifest new file mode 100755 index 0000000..83d9dab --- /dev/null +++ b/type/__evilham_jitsi_meet/manifest @@ -0,0 +1,197 @@ +#!/bin/sh -e + +os="$(cat "${__global}/explorer/os")" +init="$(cat "${__global}/explorer/init")" +case "${os}" in + devuan|debian) + ;; + *) + echo "Your OS '${os}' is currently not supported." > /dev/stderr + exit 1 + ;; +esac + + +JITSI_HOST="${__target_host}" +TURN_SERVER="$(cat "${__object}/parameter/turn-server")" +TURN_SECRET="$(cat "${__object}/parameter/turn-secret")" + +if [ -z "${TURN_SERVER}" ]; then + TURN_SERVER="${JITSI_HOST}" +fi + +PROMETHEUS_JITSI_EXPORTER_IS_VERSION="$(cat "${__object}/explorer/prometheus-jitsi-meet-explorer-version")" + +# The rest is loosely based on Jitsi's documentation +# https://jitsi.github.io/handbook/docs/devops-guide/devops-guide-quickstart + +# Setup repositories +## First the signing keys +__package gnupg2 +require="__package/gnupg2" __apt_key_uri jitsi_meet \ + --name 'Jitsi ' \ + --uri https://download.jitsi.org/jitsi-key.gpg.key \ + --state present +## Now the repositories (they are a tad weird, so distribution is 'stable/') +require="__ufw __apt_key_uri/jitsi_meet" __apt_source jitsi_meet \ + --uri 'https://download.jitsi.org' \ + --distribution 'stable/' \ + --state present +## Ensure apt cache is up-to-date +require="__apt_source/jitsi_meet" __apt_update_index + +export require="${require} __apt_source/jitsi_meet __apt_update_index" + +# Pre-feed debconf settings, so Jitsi's installation has a good config +. "${__type}/files/debconf_settings.sh" # This defines DEBCONF_SETTINGS +__debconf_set_selections jitsi_meet --file - <&1 +EOF + + export require="__evilham_runit_service/prometheus-jitsi-meet-exporter" + JITSI_MEET_EXPORTER_SERVICE="sv %s prometheus-jitsi-meet-exporter" + ;; + systemd) + __systemd_unit prometheus-jitsi-meet-exporter.service \ + --source "-" \ + --enablement-state "enabled" <${JITSI_HOST}' + }, + + // BOSH URL. FIXME: use XEP-0156 to discover it. + bosh: '//${JITSI_HOST}/http-bind', + + // Websocket URL + // websocket: 'wss://${JITSI_HOST}/xmpp-websocket', + + // The name of client node advertised in XEP-0115 'c' stanza + clientNode: 'http://jitsi.org/jitsimeet', + + // The real JID of focus participant - can be overridden here + // Do not change username - FIXME: Make focus username configurable + // https://github.com/jitsi/jitsi-meet/issues/7376 + // focusUserJid: 'focus@auth.${JITSI_HOST}', + + + // Testing / experimental features. + // + + testing: { + // Disables the End to End Encryption feature. Useful for debugging + // issues related to insertable streams. + // disableE2EE: false, + + // P2P test mode disables automatic switching to P2P when there are 2 + // participants in the conference. + p2pTestMode: false + + // Enables the test specific features consumed by jitsi-meet-torture + // testMode: false + + // Disables the auto-play behavior of *all* newly created video element. + // This is useful when the client runs on a host with limited resources. + // noAutoPlayVideo: false + + // Enable / disable 500 Kbps bitrate cap on desktop tracks. When enabled, + // simulcast is turned off for the desktop share. If presenter is turned + // on while screensharing is in progress, the max bitrate is automatically + // adjusted to 2.5 Mbps. This takes a value between 0 and 1 which determines + // the probability for this to be enabled. + // capScreenshareBitrate: 1 // 0 to disable + + // Enable callstats only for a percentage of users. + // This takes a value between 0 and 100 which determines the probability for + // the callstats to be enabled. + // callStatsThreshold: 5 // enable callstats for 5% of the users. + }, + + // Disables ICE/UDP by filtering out local and remote UDP candidates in + // signalling. + // webrtcIceUdpDisable: false, + + // Disables ICE/TCP by filtering out local and remote TCP candidates in + // signalling. + // webrtcIceTcpDisable: false, + + + // Media + // + + // Audio + + // Disable measuring of audio levels. + disableAudioLevels: $(if [ -n "${DISABLE_AUDIO_LEVELS}" ]; then printf "true"; else printf "false"; fi), + // audioLevelsInterval: 200, + + // Enabling this will run the lib-jitsi-meet no audio detection module which + // will notify the user if the current selected microphone has no audio + // input and will suggest another valid device if one is present. + enableNoAudioDetection: true, + + // Enabling this will run the lib-jitsi-meet noise detection module which will + // notify the user if there is noise, other than voice, coming from the current + // selected microphone. The purpose it to let the user know that the input could + // be potentially unpleasant for other meeting participants. + enableNoisyMicDetection: true, + + // Start the conference in audio only mode (no video is being received nor + // sent). + // startAudioOnly: false, + + // Every participant after the Nth will start audio muted. + // startAudioMuted: 10, + + // Start calls with audio muted. Unlike the option above, this one is only + // applied locally. FIXME: having these 2 options is confusing. + // startWithAudioMuted: false, + + // Enabling it (with #params) will disable local audio output of remote + // participants and to enable it back a reload is needed. + // startSilent: false + + // Sets the preferred target bitrate for the Opus audio codec by setting its + // 'maxaveragebitrate' parameter. Currently not available in p2p mode. + // Valid values are in the range 6000 to 510000 + // opusMaxAverageBitrate: 20000, + + // Enables redundancy for Opus + // enableOpusRed: false + + // Video + + // Sets the preferred resolution (height) for local video. Defaults to 720. + // resolution: 720, + + // How many participants while in the tile view mode, before the receiving video quality is reduced from HD to SD. + // Use -1 to disable. + // maxFullResolutionParticipants: 2, + + // w3c spec-compliant video constraints to use for video capture. Currently + // used by browsers that return true from lib-jitsi-meet's + // util#browser#usesNewGumFlow. The constraints are independent from + // this config's resolution value. Defaults to requesting an ideal + // resolution of 720p. + // constraints: { + // video: { + // height: { + // ideal: 720, + // max: 720, + // min: 240 + // } + // } + // }, +$(if [ -n "${VIDEO_CONSTRAINTS}" ]; then echo "${VIDEO_CONSTRAINTS},"; fi) + + // Enable / disable simulcast support. + // disableSimulcast: false, + + // Enable / disable layer suspension. If enabled, endpoints whose HD + // layers are not in use will be suspended (no longer sent) until they + // are requested again. + // enableLayerSuspension: false, + + // Every participant after the Nth will start video muted. + startVideoMuted: ${START_VIDEO_MUTED}, + + // Start calls with video muted. Unlike the option above, this one is only + // applied locally. FIXME: having these 2 options is confusing. + // startWithVideoMuted: false, + + // If set to true, prefer to use the H.264 video codec (if supported). + // Note that it's not recommended to do this because simulcast is not + // supported when using H.264. For 1-to-1 calls this setting is enabled by + // default and can be toggled in the p2p section. + // This option has been deprecated, use preferredCodec under videoQuality section instead. + // preferH264: true, + + // If set to true, disable H.264 video codec by stripping it out of the + // SDP. + // disableH264: false, + + // Desktop sharing + + // Optional desktop sharing frame rate options. Default value: min:5, max:5. + // desktopSharingFrameRate: { + // min: 5, + // max: 5 + // }, + + // Try to start calls with screen-sharing instead of camera video. + // startScreenSharing: false, + + // Recording + + // Whether to enable file recording or not. + // fileRecordingsEnabled: false, + // Enable the dropbox integration. + // dropbox: { + // appKey: '' // Specify your app key here. + // // A URL to redirect the user to, after authenticating + // // by default uses: + // // 'https://${JITSI_HOST}/static/oauth.html' + // redirectURI: + // 'https://${JITSI_HOST}/subfolder/static/oauth.html' + // }, + // When integrations like dropbox are enabled only that will be shown, + // by enabling fileRecordingsServiceEnabled, we show both the integrations + // and the generic recording service (its configuration and storage type + // depends on jibri configuration) + // fileRecordingsServiceEnabled: false, + // Whether to show the possibility to share file recording with other people + // (e.g. meeting participants), based on the actual implementation + // on the backend. + // fileRecordingsServiceSharingEnabled: false, + + // Whether to enable live streaming or not. + // liveStreamingEnabled: false, + + // Transcription (in interface_config, + // subtitles and buttons can be configured) + // transcribingEnabled: false, + + // Enables automatic turning on captions when recording is started + // autoCaptionOnRecord: false, + + // Misc + + // Default value for the channel "last N" attribute. -1 for unlimited. + channelLastN: ${CHANNEL_LAST_N}, + + // Provides a way to use different "last N" values based on the number of participants in the conference. + // The keys in an Object represent number of participants and the values are "last N" to be used when number of + // participants gets to or above the number. + // + // For the given example mapping, "last N" will be set to 20 as long as there are at least 5, but less than + // 29 participants in the call and it will be lowered to 15 when the 30th participant joins. The 'channelLastN' + // will be used as default until the first threshold is reached. + // + // lastNLimits: { + // 5: 20, + // 30: 15, + // 50: 10, + // 70: 5, + // 90: 2 + // }, + + // Specify the settings for video quality optimizations on the client. + // videoQuality: { + // // Provides a way to prevent a video codec from being negotiated on the JVB connection. The codec specified + // // here will be removed from the list of codecs present in the SDP answer generated by the client. If the + // // same codec is specified for both the disabled and preferred option, the disable settings will prevail. + // // Note that 'VP8' cannot be disabled since it's a mandatory codec, the setting will be ignored in this case. + // disabledCodec: 'H264', + // + // // Provides a way to set a preferred video codec for the JVB connection. If 'H264' is specified here, + // // simulcast will be automatically disabled since JVB doesn't support H264 simulcast yet. This will only + // // rearrange the the preference order of the codecs in the SDP answer generated by the browser only if the + // // preferred codec specified here is present. Please ensure that the JVB offers the specified codec for this + // // to take effect. + // preferredCodec: 'VP8', + // + // // Provides a way to configure the maximum bitrates that will be enforced on the simulcast streams for + // // video tracks. The keys in the object represent the type of the stream (LD, SD or HD) and the values + // // are the max.bitrates to be set on that particular type of stream. The actual send may vary based on + // // the available bandwidth calculated by the browser, but it will be capped by the values specified here. + // // This is currently not implemented on app based clients on mobile. + // maxBitratesVideo: { + // low: 200000, + // standard: 500000, + // high: 1500000 + // }, + // + // // The options can be used to override default thresholds of video thumbnail heights corresponding to + // // the video quality levels used in the application. At the time of this writing the allowed levels are: + // // 'low' - for the low quality level (180p at the time of this writing) + // // 'standard' - for the medium quality level (360p) + // // 'high' - for the high quality level (720p) + // // The keys should be positive numbers which represent the minimal thumbnail height for the quality level. + // // + // // With the default config value below the application will use 'low' quality until the thumbnails are + // // at least 360 pixels tall. If the thumbnail height reaches 720 pixels then the application will switch to + // // the high quality. + // minHeightForQualityLvl: { + // 360: 'standard, + // 720: 'high' + // } + // }, + + // // Options for the recording limit notification. + // recordingLimit: { + // + // // The recording limit in minutes. Note: This number appears in the notification text + // // but doesn't enforce the actual recording time limit. This should be configured in + // // jibri! + // limit: 60, + // + // // The name of the app with unlimited recordings. + // appName: 'Unlimited recordings APP', + // + // // The URL of the app with unlimited recordings. + // appURL: 'https://unlimited.recordings.app.com/' + // }, + + // Disables or enables RTX (RFC 4588) (defaults to false). + // disableRtx: false, + + // Disables or enables TCC (the default is in Jicofo and set to true) + // (draft-holmer-rmcat-transport-wide-cc-extensions-01). This setting + // affects congestion control, it practically enables send-side bandwidth + // estimations. + // enableTcc: true, + + // Disables or enables REMB (the default is in Jicofo and set to false) + // (draft-alvestrand-rmcat-remb-03). This setting affects congestion + // control, it practically enables recv-side bandwidth estimations. When + // both TCC and REMB are enabled, TCC takes precedence. When both are + // disabled, then bandwidth estimations are disabled. + // enableRemb: false, + + // Enables ICE restart logic in LJM and displays the page reload overlay on + // ICE failure. Current disabled by default because it's causing issues with + // signaling when Octo is enabled. Also when we do an "ICE restart"(which is + // not a real ICE restart), the client maintains the TCC sequence number + // counter, but the bridge resets it. The bridge sends media packets with + // TCC sequence numbers starting from 0. + // enableIceRestart: false, + + // Defines the minimum number of participants to start a call (the default + // is set in Jicofo and set to 2). + // minParticipants: 2, + + // Use TURN/UDP servers for the jitsi-videobridge connection (by default + // we filter out TURN/UDP because it is usually not needed since the + // bridge itself is reachable via UDP) + // useTurnUdp: false + + // Enables / disables a data communication channel with the Videobridge. + // Values can be 'datachannel', 'websocket', true (treat it as + // 'datachannel'), undefined (treat it as 'datachannel') and false (don't + // open any channel). + // openBridgeChannel: true, + openBridgeChannel: 'websocket', + + + // UI + // + + // Hides lobby button + // hideLobbyButton: false, + + // Require users to always specify a display name. + // requireDisplayName: true, + + // Whether to use a welcome page or not. In case it's false a random room + // will be joined when no room is specified. + enableWelcomePage: true, + + // Enabling the close page will ignore the welcome page redirection when + // a call is hangup. + // enableClosePage: false, + + // Disable hiding of remote thumbnails when in a 1-on-1 conference call. + // disable1On1Mode: false, + + // Default language for the user interface. + defaultLanguage: '${DEFAULT_LANGUAGE}', + + // If true all users without a token will be considered guests and all users + // with token will be considered non-guests. Only guests will be allowed to + // edit their profile. + enableUserRolesBasedOnToken: false, + + // Whether or not some features are checked based on token. + // enableFeaturesBasedOnToken: false, + + // Enable lock room for all moderators, even when userRolesBasedOnToken is enabled and participants are guests. + // lockRoomGuestEnabled: false, + + // When enabled the password used for locking a room is restricted to up to the number of digits specified + // roomPasswordNumberOfDigits: 10, + // default: roomPasswordNumberOfDigits: false, + + // Message to show the users. Example: 'The service will be down for + // maintenance at 01:00 AM GMT, + noticeMessage: '${NOTICE_MESSAGE}', + + // Enables calendar integration, depends on googleApiApplicationClientID + // and microsoftApiApplicationClientID + // enableCalendarIntegration: false, + + // When 'true', it shows an intermediate page before joining, where the user can configure their devices. + // prejoinPageEnabled: false, + + // If true, shows the unsafe room name warning label when a room name is + // deemed unsafe (due to the simplicity in the name) and a password is not + // set or the lobby is not enabled. + // enableInsecureRoomNameWarning: false, + + // Whether to automatically copy invitation URL after creating a room. + // Document should be focused for this option to work + // enableAutomaticUrlCopy: false, + + // Stats + // + + // Whether to enable stats collection or not in the TraceablePeerConnection. + // This can be useful for debugging purposes (post-processing/analysis of + // the webrtc stats) as it is done in the jitsi-meet-torture bandwidth + // estimation tests. + // gatherStats: false, + + // The interval at which PeerConnection.getStats() is called. Defaults to 10000 + // pcStatsInterval: 10000, + + // To enable sending statistics to callstats.io you must provide the + // Application ID and Secret. + // callStatsID: '', + // callStatsSecret: '', + + // Enables sending participants' display names to callstats + // enableDisplayNameInStats: false, + + // Enables sending participants' emails (if available) to callstats and other analytics + // enableEmailInStats: false, + + // Privacy + // + + // If third party requests are disabled, no other server will be contacted. + // This means avatars will be locally generated and callstats integration + // will not function. + disableThirdPartyRequests: $(if [ -z "${ENABLE_THIRD_PARTY_REQUESTS}" ]; then printf "true"; else printf "false"; fi), + + + // Peer-To-Peer mode: used (if enabled) when there are just 2 participants. + // + + p2p: { + // Enables peer to peer mode. When enabled the system will try to + // establish a direct connection when there are exactly 2 participants + // in the room. If that succeeds the conference will stop sending data + // through the JVB and use the peer to peer connection instead. When a + // 3rd participant joins the conference will be moved back to the JVB + // connection. + enabled: true, + + // The STUN servers that will be used in the peer to peer connections + stunServers: [ + + { urls: 'stun:${TURN_SERVER}:443' } + ] + + // Sets the ICE transport policy for the p2p connection. At the time + // of this writing the list of possible values are 'all' and 'relay', + // but that is subject to change in the future. The enum is defined in + // the WebRTC standard: + // https://www.w3.org/TR/webrtc/#rtcicetransportpolicy-enum. + // If not set, the effective value is 'all'. + // iceTransportPolicy: 'all', + + // If set to true, it will prefer to use H.264 for P2P calls (if H.264 + // is supported). This setting is deprecated, use preferredCodec instead. + // preferH264: true + + // Provides a way to set the video codec preference on the p2p connection. Acceptable + // codec values are 'VP8', 'VP9' and 'H264'. + // preferredCodec: 'H264', + + // If set to true, disable H.264 video codec by stripping it out of the + // SDP. This setting is deprecated, use disabledCodec instead. + // disableH264: false, + + // Provides a way to prevent a video codec from being negotiated on the p2p connection. + // disabledCodec: '', + + // How long we're going to wait, before going back to P2P after the 3rd + // participant has left the conference (to filter out page reload). + // backToP2PDelay: 5 + }, + + analytics: { + // The Google Analytics Tracking ID: + // googleAnalyticsTrackingId: 'your-tracking-id-UA-123456-1' + + // Matomo configuration: + // matomoEndpoint: 'https://your-matomo-endpoint/', + // matomoSiteID: '42', + + // The Amplitude APP Key: + // amplitudeAPPKey: '' + + // Configuration for the rtcstats server: + // By enabling rtcstats server every time a conference is joined the rtcstats + // module connects to the provided rtcstatsEndpoint and sends statistics regarding + // PeerConnection states along with getStats metrics polled at the specified + // interval. + // rtcstatsEnabled: true, + + // In order to enable rtcstats one needs to provide a endpoint url. + // rtcstatsEndpoint: wss://rtcstats-server-pilot.jitsi.net/, + + // The interval at which rtcstats will poll getStats, defaults to 1000ms. + // If the value is set to 0 getStats won't be polled and the rtcstats client + // will only send data related to RTCPeerConnection events. + // rtcstatsPolIInterval: 1000 + + // Array of script URLs to load as lib-jitsi-meet "analytics handlers". + // scriptURLs: [ + // "libs/analytics-ga.min.js", // google-analytics + // "https://example.com/my-custom-analytics.js" + // ], + }, + + // Logs that should go be passed through the 'log' event if a handler is defined for it + // apiLogLevels: ['warn', 'log', 'error', 'info', 'debug'], + + // Information about the jitsi-meet instance we are connecting to, including + // the user region as seen by the server. + deploymentInfo: { + // shard: "shard1", + // region: "europe", + // userRegion: "asia" + }, + + // Decides whether the start/stop recording audio notifications should play on record. + // disableRecordAudioNotification: false, + + // Information for the chrome extension banner + // chromeExtensionBanner: { + // // The chrome extension to be installed address + // url: 'https://chrome.google.com/webstore/detail/jitsi-meetings/kglhbbefdnlheedjiejgomgmfplipfeb', + + // // Extensions info which allows checking if they are installed or not + // chromeExtensionsInfo: [ + // { + // id: 'kglhbbefdnlheedjiejgomgmfplipfeb', + // path: 'jitsi-logo-48x48.png' + // } + // ] + // }, + + // Local Recording + // + + // localRecording: { + // Enables local recording. + // Additionally, 'localrecording' (all lowercase) needs to be added to + // TOOLBAR_BUTTONS in interface_config.js for the Local Recording + // button to show up on the toolbar. + // + // enabled: true, + // + + // The recording format, can be one of 'ogg', 'flac' or 'wav'. + // format: 'flac' + // + + // }, + + // Options related to end-to-end (participant to participant) ping. + // e2eping: { + // // The interval in milliseconds at which pings will be sent. + // // Defaults to 10000, set to <= 0 to disable. + // pingInterval: 10000, + // + // // The interval in milliseconds at which analytics events + // // with the measured RTT will be sent. Defaults to 60000, set + // // to <= 0 to disable. + // analyticsInterval: 60000, + // }, + + // If set, will attempt to use the provided video input device label when + // triggering a screenshare, instead of proceeding through the normal flow + // for obtaining a desktop stream. + // NOTE: This option is experimental and is currently intended for internal + // use only. + // _desktopSharingSourceDevice: 'sample-id-or-label', + + // If true, any checks to handoff to another application will be prevented + // and instead the app will continue to display in the current browser. + // disableDeepLinking: false, + + // A property to disable the right click context menu for localVideo + // the menu has option to flip the locally seen video for local presentations + // disableLocalVideoFlip: false, + + // Mainly privacy related settings + + // Disables all invite functions from the app (share, invite, dial out...etc) + // disableInviteFunctions: true, + + // Disables storing the room name to the recents list + // doNotStoreRoom: true, + + // Deployment specific URLs. + // deploymentUrls: { + // // If specified a 'Help' button will be displayed in the overflow menu with a link to the specified URL for + // // user documentation. + // userDocumentationURL: 'https://docs.example.com/video-meetings.html', + // // If specified a 'Download our apps' button will be displayed in the overflow menu with a link + // // to the specified URL for an app download page. + // downloadAppsUrl: 'https://docs.example.com/our-apps.html' + // }, + + // Options related to the remote participant menu. + // remoteVideoMenu: { + // // If set to true the 'Kick out' button will be disabled. + // disableKick: true + // }, + + // If set to true all muting operations of remote participants will be disabled. + // disableRemoteMute: true, + + /** + External API url used to receive branding specific information. + If there is no url set or there are missing fields, the defaults are applied. + None of the fields are mandatory and the response must have the shape: + { + // The hex value for the colour used as background + backgroundColor: '#fff', + // The url for the image used as background + backgroundImageUrl: 'https://example.com/background-img.png', + // The anchor url used when clicking the logo image + logoClickUrl: 'https://example-company.org', + // The url used for the image used as logo + logoImageUrl: 'https://example.com/logo-img.png' + } + */ + brandingDataUrl: "$(if [ -n "${BRANDING_JSON}" ]; then printf "/branding.json"; fi)", + + // The URL of the moderated rooms microservice, if available. If it + // is present, a link to the service will be rendered on the welcome page, + // otherwise the app doesn't render it. + // moderatedRoomServiceUrl: 'https://moderated.${JITSI_HOST}', + + // List of undocumented settings used in jitsi-meet + /** + _immediateReloadThreshold + debug + debugAudioLevels + deploymentInfo + dialInConfCodeUrl + dialInNumbersUrl + dialOutAuthUrl + dialOutCodesUrl + disableRemoteControl + displayJids + etherpad_base + externalConnectUrl + firefox_fake_device + googleApiApplicationClientID + iAmRecorder + iAmSipGateway + microsoftApiApplicationClientID + peopleSearchQueryTypes + peopleSearchUrl + requireDisplayName + tokenAuthUrl + */ + + /** + * This property can be used to alter the generated meeting invite links (in combination with a branding domain + * which is retrieved internally by jitsi meet) (e.g. https://meet.jit.si/someMeeting + * can become https://brandedDomain/roomAlias) + */ + // brandingRoomAlias: null, + + // List of undocumented settings used in lib-jitsi-meet + /** + _peerConnStatusOutOfLastNTimeout + _peerConnStatusRtcMuteTimeout + abTesting + avgRtpStatsN + callStatsConfIDNamespace + callStatsCustomScriptUrl + desktopSharingSources + disableAEC + disableAGC + disableAP + disableHPF + disableNS + enableLipSync + enableTalkWhileMuted + forceJVB121Ratio + hiddenDomain + ignoreStartMuted + nick + startBitrate + */ + + + // Allow all above example options to include a trailing comma and + // prevent fear when commenting out the last value. + makeJsonParserHappy: 'even if last key had a trailing comma' + + // no configuration value should follow this line. +}; + +/* eslint-enable no-unused-vars, no-var */ +EOF +)" diff --git a/type/__evilham_jitsi_meet_domain/files/config.js.sh.orig b/type/__evilham_jitsi_meet_domain/files/config.js.sh.orig new file mode 100644 index 0000000..da2bff5 --- /dev/null +++ b/type/__evilham_jitsi_meet_domain/files/config.js.sh.orig @@ -0,0 +1,694 @@ +/* eslint-disable no-unused-vars, no-var */ + +var config = { + // Connection + // + + hosts: { + // XMPP domain. + domain: 'jitsi-meet.example.org', + + // When using authentication, domain for guest users. + // anonymousdomain: 'guest.example.com', + + // Domain for authenticated users. Defaults to . + // authdomain: 'jitsi-meet.example.org', + + // Call control component (Jigasi). + // call_control: 'callcontrol.jitsi-meet.example.org', + + // Focus component domain. Defaults to focus.. + // focus: 'focus.jitsi-meet.example.org', + + // XMPP MUC domain. FIXME: use XEP-0030 to discover it. + muc: 'conference.jitsi-meet.example.org' + }, + + // BOSH URL. FIXME: use XEP-0156 to discover it. + bosh: '//jitsi-meet.example.org/http-bind', + + // Websocket URL + // websocket: 'wss://jitsi-meet.example.org/xmpp-websocket', + + // The name of client node advertised in XEP-0115 'c' stanza + clientNode: 'http://jitsi.org/jitsimeet', + + // The real JID of focus participant - can be overridden here + // Do not change username - FIXME: Make focus username configurable + // https://github.com/jitsi/jitsi-meet/issues/7376 + // focusUserJid: 'focus@auth.jitsi-meet.example.org', + + + // Testing / experimental features. + // + + testing: { + // Disables the End to End Encryption feature. Useful for debugging + // issues related to insertable streams. + // disableE2EE: false, + + // P2P test mode disables automatic switching to P2P when there are 2 + // participants in the conference. + p2pTestMode: false + + // Enables the test specific features consumed by jitsi-meet-torture + // testMode: false + + // Disables the auto-play behavior of *all* newly created video element. + // This is useful when the client runs on a host with limited resources. + // noAutoPlayVideo: false + + // Enable / disable 500 Kbps bitrate cap on desktop tracks. When enabled, + // simulcast is turned off for the desktop share. If presenter is turned + // on while screensharing is in progress, the max bitrate is automatically + // adjusted to 2.5 Mbps. This takes a value between 0 and 1 which determines + // the probability for this to be enabled. + // capScreenshareBitrate: 1 // 0 to disable + + // Enable callstats only for a percentage of users. + // This takes a value between 0 and 100 which determines the probability for + // the callstats to be enabled. + // callStatsThreshold: 5 // enable callstats for 5% of the users. + }, + + // Disables ICE/UDP by filtering out local and remote UDP candidates in + // signalling. + // webrtcIceUdpDisable: false, + + // Disables ICE/TCP by filtering out local and remote TCP candidates in + // signalling. + // webrtcIceTcpDisable: false, + + + // Media + // + + // Audio + + // Disable measuring of audio levels. + // disableAudioLevels: false, + // audioLevelsInterval: 200, + + // Enabling this will run the lib-jitsi-meet no audio detection module which + // will notify the user if the current selected microphone has no audio + // input and will suggest another valid device if one is present. + enableNoAudioDetection: true, + + // Enabling this will run the lib-jitsi-meet noise detection module which will + // notify the user if there is noise, other than voice, coming from the current + // selected microphone. The purpose it to let the user know that the input could + // be potentially unpleasant for other meeting participants. + enableNoisyMicDetection: true, + + // Start the conference in audio only mode (no video is being received nor + // sent). + // startAudioOnly: false, + + // Every participant after the Nth will start audio muted. + // startAudioMuted: 10, + + // Start calls with audio muted. Unlike the option above, this one is only + // applied locally. FIXME: having these 2 options is confusing. + // startWithAudioMuted: false, + + // Enabling it (with #params) will disable local audio output of remote + // participants and to enable it back a reload is needed. + // startSilent: false + + // Sets the preferred target bitrate for the Opus audio codec by setting its + // 'maxaveragebitrate' parameter. Currently not available in p2p mode. + // Valid values are in the range 6000 to 510000 + // opusMaxAverageBitrate: 20000, + + // Enables redundancy for Opus + // enableOpusRed: false + + // Video + + // Sets the preferred resolution (height) for local video. Defaults to 720. + // resolution: 720, + + // How many participants while in the tile view mode, before the receiving video quality is reduced from HD to SD. + // Use -1 to disable. + // maxFullResolutionParticipants: 2, + + // w3c spec-compliant video constraints to use for video capture. Currently + // used by browsers that return true from lib-jitsi-meet's + // util#browser#usesNewGumFlow. The constraints are independent from + // this config's resolution value. Defaults to requesting an ideal + // resolution of 720p. + // constraints: { + // video: { + // height: { + // ideal: 720, + // max: 720, + // min: 240 + // } + // } + // }, + + // Enable / disable simulcast support. + // disableSimulcast: false, + + // Enable / disable layer suspension. If enabled, endpoints whose HD + // layers are not in use will be suspended (no longer sent) until they + // are requested again. + // enableLayerSuspension: false, + + // Every participant after the Nth will start video muted. + // startVideoMuted: 10, + + // Start calls with video muted. Unlike the option above, this one is only + // applied locally. FIXME: having these 2 options is confusing. + // startWithVideoMuted: false, + + // If set to true, prefer to use the H.264 video codec (if supported). + // Note that it's not recommended to do this because simulcast is not + // supported when using H.264. For 1-to-1 calls this setting is enabled by + // default and can be toggled in the p2p section. + // This option has been deprecated, use preferredCodec under videoQuality section instead. + // preferH264: true, + + // If set to true, disable H.264 video codec by stripping it out of the + // SDP. + // disableH264: false, + + // Desktop sharing + + // Optional desktop sharing frame rate options. Default value: min:5, max:5. + // desktopSharingFrameRate: { + // min: 5, + // max: 5 + // }, + + // Try to start calls with screen-sharing instead of camera video. + // startScreenSharing: false, + + // Recording + + // Whether to enable file recording or not. + // fileRecordingsEnabled: false, + // Enable the dropbox integration. + // dropbox: { + // appKey: '' // Specify your app key here. + // // A URL to redirect the user to, after authenticating + // // by default uses: + // // 'https://jitsi-meet.example.org/static/oauth.html' + // redirectURI: + // 'https://jitsi-meet.example.org/subfolder/static/oauth.html' + // }, + // When integrations like dropbox are enabled only that will be shown, + // by enabling fileRecordingsServiceEnabled, we show both the integrations + // and the generic recording service (its configuration and storage type + // depends on jibri configuration) + // fileRecordingsServiceEnabled: false, + // Whether to show the possibility to share file recording with other people + // (e.g. meeting participants), based on the actual implementation + // on the backend. + // fileRecordingsServiceSharingEnabled: false, + + // Whether to enable live streaming or not. + // liveStreamingEnabled: false, + + // Transcription (in interface_config, + // subtitles and buttons can be configured) + // transcribingEnabled: false, + + // Enables automatic turning on captions when recording is started + // autoCaptionOnRecord: false, + + // Misc + + // Default value for the channel "last N" attribute. -1 for unlimited. + channelLastN: -1, + + // Provides a way to use different "last N" values based on the number of participants in the conference. + // The keys in an Object represent number of participants and the values are "last N" to be used when number of + // participants gets to or above the number. + // + // For the given example mapping, "last N" will be set to 20 as long as there are at least 5, but less than + // 29 participants in the call and it will be lowered to 15 when the 30th participant joins. The 'channelLastN' + // will be used as default until the first threshold is reached. + // + // lastNLimits: { + // 5: 20, + // 30: 15, + // 50: 10, + // 70: 5, + // 90: 2 + // }, + + // Specify the settings for video quality optimizations on the client. + // videoQuality: { + // // Provides a way to prevent a video codec from being negotiated on the JVB connection. The codec specified + // // here will be removed from the list of codecs present in the SDP answer generated by the client. If the + // // same codec is specified for both the disabled and preferred option, the disable settings will prevail. + // // Note that 'VP8' cannot be disabled since it's a mandatory codec, the setting will be ignored in this case. + // disabledCodec: 'H264', + // + // // Provides a way to set a preferred video codec for the JVB connection. If 'H264' is specified here, + // // simulcast will be automatically disabled since JVB doesn't support H264 simulcast yet. This will only + // // rearrange the the preference order of the codecs in the SDP answer generated by the browser only if the + // // preferred codec specified here is present. Please ensure that the JVB offers the specified codec for this + // // to take effect. + // preferredCodec: 'VP8', + // + // // Provides a way to configure the maximum bitrates that will be enforced on the simulcast streams for + // // video tracks. The keys in the object represent the type of the stream (LD, SD or HD) and the values + // // are the max.bitrates to be set on that particular type of stream. The actual send may vary based on + // // the available bandwidth calculated by the browser, but it will be capped by the values specified here. + // // This is currently not implemented on app based clients on mobile. + // maxBitratesVideo: { + // low: 200000, + // standard: 500000, + // high: 1500000 + // }, + // + // // The options can be used to override default thresholds of video thumbnail heights corresponding to + // // the video quality levels used in the application. At the time of this writing the allowed levels are: + // // 'low' - for the low quality level (180p at the time of this writing) + // // 'standard' - for the medium quality level (360p) + // // 'high' - for the high quality level (720p) + // // The keys should be positive numbers which represent the minimal thumbnail height for the quality level. + // // + // // With the default config value below the application will use 'low' quality until the thumbnails are + // // at least 360 pixels tall. If the thumbnail height reaches 720 pixels then the application will switch to + // // the high quality. + // minHeightForQualityLvl: { + // 360: 'standard, + // 720: 'high' + // } + // }, + + // // Options for the recording limit notification. + // recordingLimit: { + // + // // The recording limit in minutes. Note: This number appears in the notification text + // // but doesn't enforce the actual recording time limit. This should be configured in + // // jibri! + // limit: 60, + // + // // The name of the app with unlimited recordings. + // appName: 'Unlimited recordings APP', + // + // // The URL of the app with unlimited recordings. + // appURL: 'https://unlimited.recordings.app.com/' + // }, + + // Disables or enables RTX (RFC 4588) (defaults to false). + // disableRtx: false, + + // Disables or enables TCC (the default is in Jicofo and set to true) + // (draft-holmer-rmcat-transport-wide-cc-extensions-01). This setting + // affects congestion control, it practically enables send-side bandwidth + // estimations. + // enableTcc: true, + + // Disables or enables REMB (the default is in Jicofo and set to false) + // (draft-alvestrand-rmcat-remb-03). This setting affects congestion + // control, it practically enables recv-side bandwidth estimations. When + // both TCC and REMB are enabled, TCC takes precedence. When both are + // disabled, then bandwidth estimations are disabled. + // enableRemb: false, + + // Enables ICE restart logic in LJM and displays the page reload overlay on + // ICE failure. Current disabled by default because it's causing issues with + // signaling when Octo is enabled. Also when we do an "ICE restart"(which is + // not a real ICE restart), the client maintains the TCC sequence number + // counter, but the bridge resets it. The bridge sends media packets with + // TCC sequence numbers starting from 0. + // enableIceRestart: false, + + // Defines the minimum number of participants to start a call (the default + // is set in Jicofo and set to 2). + // minParticipants: 2, + + // Use TURN/UDP servers for the jitsi-videobridge connection (by default + // we filter out TURN/UDP because it is usually not needed since the + // bridge itself is reachable via UDP) + // useTurnUdp: false + + // Enables / disables a data communication channel with the Videobridge. + // Values can be 'datachannel', 'websocket', true (treat it as + // 'datachannel'), undefined (treat it as 'datachannel') and false (don't + // open any channel). + // openBridgeChannel: true, + openBridgeChannel: 'websocket', + + + // UI + // + + // Hides lobby button + // hideLobbyButton: false, + + // Require users to always specify a display name. + // requireDisplayName: true, + + // Whether to use a welcome page or not. In case it's false a random room + // will be joined when no room is specified. + enableWelcomePage: true, + + // Enabling the close page will ignore the welcome page redirection when + // a call is hangup. + // enableClosePage: false, + + // Disable hiding of remote thumbnails when in a 1-on-1 conference call. + // disable1On1Mode: false, + + // Default language for the user interface. + // defaultLanguage: 'en', + + // If true all users without a token will be considered guests and all users + // with token will be considered non-guests. Only guests will be allowed to + // edit their profile. + enableUserRolesBasedOnToken: false, + + // Whether or not some features are checked based on token. + // enableFeaturesBasedOnToken: false, + + // Enable lock room for all moderators, even when userRolesBasedOnToken is enabled and participants are guests. + // lockRoomGuestEnabled: false, + + // When enabled the password used for locking a room is restricted to up to the number of digits specified + // roomPasswordNumberOfDigits: 10, + // default: roomPasswordNumberOfDigits: false, + + // Message to show the users. Example: 'The service will be down for + // maintenance at 01:00 AM GMT, + // noticeMessage: '', + + // Enables calendar integration, depends on googleApiApplicationClientID + // and microsoftApiApplicationClientID + // enableCalendarIntegration: false, + + // When 'true', it shows an intermediate page before joining, where the user can configure their devices. + // prejoinPageEnabled: false, + + // If true, shows the unsafe room name warning label when a room name is + // deemed unsafe (due to the simplicity in the name) and a password is not + // set or the lobby is not enabled. + // enableInsecureRoomNameWarning: false, + + // Whether to automatically copy invitation URL after creating a room. + // Document should be focused for this option to work + // enableAutomaticUrlCopy: false, + + // Stats + // + + // Whether to enable stats collection or not in the TraceablePeerConnection. + // This can be useful for debugging purposes (post-processing/analysis of + // the webrtc stats) as it is done in the jitsi-meet-torture bandwidth + // estimation tests. + // gatherStats: false, + + // The interval at which PeerConnection.getStats() is called. Defaults to 10000 + // pcStatsInterval: 10000, + + // To enable sending statistics to callstats.io you must provide the + // Application ID and Secret. + // callStatsID: '', + // callStatsSecret: '', + + // Enables sending participants' display names to callstats + // enableDisplayNameInStats: false, + + // Enables sending participants' emails (if available) to callstats and other analytics + // enableEmailInStats: false, + + // Privacy + // + + // If third party requests are disabled, no other server will be contacted. + // This means avatars will be locally generated and callstats integration + // will not function. + // disableThirdPartyRequests: false, + + + // Peer-To-Peer mode: used (if enabled) when there are just 2 participants. + // + + p2p: { + // Enables peer to peer mode. When enabled the system will try to + // establish a direct connection when there are exactly 2 participants + // in the room. If that succeeds the conference will stop sending data + // through the JVB and use the peer to peer connection instead. When a + // 3rd participant joins the conference will be moved back to the JVB + // connection. + enabled: true, + + // The STUN servers that will be used in the peer to peer connections + stunServers: [ + + // { urls: 'stun:jitsi-meet.example.org:3478' }, + { urls: 'stun:meet-jit-si-turnrelay.jitsi.net:443' } + ] + + // Sets the ICE transport policy for the p2p connection. At the time + // of this writing the list of possible values are 'all' and 'relay', + // but that is subject to change in the future. The enum is defined in + // the WebRTC standard: + // https://www.w3.org/TR/webrtc/#rtcicetransportpolicy-enum. + // If not set, the effective value is 'all'. + // iceTransportPolicy: 'all', + + // If set to true, it will prefer to use H.264 for P2P calls (if H.264 + // is supported). This setting is deprecated, use preferredCodec instead. + // preferH264: true + + // Provides a way to set the video codec preference on the p2p connection. Acceptable + // codec values are 'VP8', 'VP9' and 'H264'. + // preferredCodec: 'H264', + + // If set to true, disable H.264 video codec by stripping it out of the + // SDP. This setting is deprecated, use disabledCodec instead. + // disableH264: false, + + // Provides a way to prevent a video codec from being negotiated on the p2p connection. + // disabledCodec: '', + + // How long we're going to wait, before going back to P2P after the 3rd + // participant has left the conference (to filter out page reload). + // backToP2PDelay: 5 + }, + + analytics: { + // The Google Analytics Tracking ID: + // googleAnalyticsTrackingId: 'your-tracking-id-UA-123456-1' + + // Matomo configuration: + // matomoEndpoint: 'https://your-matomo-endpoint/', + // matomoSiteID: '42', + + // The Amplitude APP Key: + // amplitudeAPPKey: '' + + // Configuration for the rtcstats server: + // By enabling rtcstats server every time a conference is joined the rtcstats + // module connects to the provided rtcstatsEndpoint and sends statistics regarding + // PeerConnection states along with getStats metrics polled at the specified + // interval. + // rtcstatsEnabled: true, + + // In order to enable rtcstats one needs to provide a endpoint url. + // rtcstatsEndpoint: wss://rtcstats-server-pilot.jitsi.net/, + + // The interval at which rtcstats will poll getStats, defaults to 1000ms. + // If the value is set to 0 getStats won't be polled and the rtcstats client + // will only send data related to RTCPeerConnection events. + // rtcstatsPolIInterval: 1000 + + // Array of script URLs to load as lib-jitsi-meet "analytics handlers". + // scriptURLs: [ + // "libs/analytics-ga.min.js", // google-analytics + // "https://example.com/my-custom-analytics.js" + // ], + }, + + // Logs that should go be passed through the 'log' event if a handler is defined for it + // apiLogLevels: ['warn', 'log', 'error', 'info', 'debug'], + + // Information about the jitsi-meet instance we are connecting to, including + // the user region as seen by the server. + deploymentInfo: { + // shard: "shard1", + // region: "europe", + // userRegion: "asia" + }, + + // Decides whether the start/stop recording audio notifications should play on record. + // disableRecordAudioNotification: false, + + // Information for the chrome extension banner + // chromeExtensionBanner: { + // // The chrome extension to be installed address + // url: 'https://chrome.google.com/webstore/detail/jitsi-meetings/kglhbbefdnlheedjiejgomgmfplipfeb', + + // // Extensions info which allows checking if they are installed or not + // chromeExtensionsInfo: [ + // { + // id: 'kglhbbefdnlheedjiejgomgmfplipfeb', + // path: 'jitsi-logo-48x48.png' + // } + // ] + // }, + + // Local Recording + // + + // localRecording: { + // Enables local recording. + // Additionally, 'localrecording' (all lowercase) needs to be added to + // TOOLBAR_BUTTONS in interface_config.js for the Local Recording + // button to show up on the toolbar. + // + // enabled: true, + // + + // The recording format, can be one of 'ogg', 'flac' or 'wav'. + // format: 'flac' + // + + // }, + + // Options related to end-to-end (participant to participant) ping. + // e2eping: { + // // The interval in milliseconds at which pings will be sent. + // // Defaults to 10000, set to <= 0 to disable. + // pingInterval: 10000, + // + // // The interval in milliseconds at which analytics events + // // with the measured RTT will be sent. Defaults to 60000, set + // // to <= 0 to disable. + // analyticsInterval: 60000, + // }, + + // If set, will attempt to use the provided video input device label when + // triggering a screenshare, instead of proceeding through the normal flow + // for obtaining a desktop stream. + // NOTE: This option is experimental and is currently intended for internal + // use only. + // _desktopSharingSourceDevice: 'sample-id-or-label', + + // If true, any checks to handoff to another application will be prevented + // and instead the app will continue to display in the current browser. + // disableDeepLinking: false, + + // A property to disable the right click context menu for localVideo + // the menu has option to flip the locally seen video for local presentations + // disableLocalVideoFlip: false, + + // Mainly privacy related settings + + // Disables all invite functions from the app (share, invite, dial out...etc) + // disableInviteFunctions: true, + + // Disables storing the room name to the recents list + // doNotStoreRoom: true, + + // Deployment specific URLs. + // deploymentUrls: { + // // If specified a 'Help' button will be displayed in the overflow menu with a link to the specified URL for + // // user documentation. + // userDocumentationURL: 'https://docs.example.com/video-meetings.html', + // // If specified a 'Download our apps' button will be displayed in the overflow menu with a link + // // to the specified URL for an app download page. + // downloadAppsUrl: 'https://docs.example.com/our-apps.html' + // }, + + // Options related to the remote participant menu. + // remoteVideoMenu: { + // // If set to true the 'Kick out' button will be disabled. + // disableKick: true + // }, + + // If set to true all muting operations of remote participants will be disabled. + // disableRemoteMute: true, + + /** + External API url used to receive branding specific information. + If there is no url set or there are missing fields, the defaults are applied. + None of the fields are mandatory and the response must have the shape: + { + // The hex value for the colour used as background + backgroundColor: '#fff', + // The url for the image used as background + backgroundImageUrl: 'https://example.com/background-img.png', + // The anchor url used when clicking the logo image + logoClickUrl: 'https://example-company.org', + // The url used for the image used as logo + logoImageUrl: 'https://example.com/logo-img.png' + } + */ + // brandingDataUrl: '', + + // The URL of the moderated rooms microservice, if available. If it + // is present, a link to the service will be rendered on the welcome page, + // otherwise the app doesn't render it. + // moderatedRoomServiceUrl: 'https://moderated.jitsi-meet.example.org', + + // List of undocumented settings used in jitsi-meet + /** + _immediateReloadThreshold + debug + debugAudioLevels + deploymentInfo + dialInConfCodeUrl + dialInNumbersUrl + dialOutAuthUrl + dialOutCodesUrl + disableRemoteControl + displayJids + etherpad_base + externalConnectUrl + firefox_fake_device + googleApiApplicationClientID + iAmRecorder + iAmSipGateway + microsoftApiApplicationClientID + peopleSearchQueryTypes + peopleSearchUrl + requireDisplayName + tokenAuthUrl + */ + + /** + * This property can be used to alter the generated meeting invite links (in combination with a branding domain + * which is retrieved internally by jitsi meet) (e.g. https://meet.jit.si/someMeeting + * can become https://brandedDomain/roomAlias) + */ + // brandingRoomAlias: null, + + // List of undocumented settings used in lib-jitsi-meet + /** + _peerConnStatusOutOfLastNTimeout + _peerConnStatusRtcMuteTimeout + abTesting + avgRtpStatsN + callStatsConfIDNamespace + callStatsCustomScriptUrl + desktopSharingSources + disableAEC + disableAGC + disableAP + disableHPF + disableNS + enableLipSync + enableTalkWhileMuted + forceJVB121Ratio + hiddenDomain + ignoreStartMuted + nick + startBitrate + */ + + + // Allow all above example options to include a trailing comma and + // prevent fear when commenting out the last value. + makeJsonParserHappy: 'even if last key had a trailing comma' + + // no configuration value should follow this line. +}; + +/* eslint-enable no-unused-vars, no-var */ diff --git a/type/__evilham_jitsi_meet_domain/files/nginx.sh b/type/__evilham_jitsi_meet_domain/files/nginx.sh new file mode 100644 index 0000000..bb300fd --- /dev/null +++ b/type/__evilham_jitsi_meet_domain/files/nginx.sh @@ -0,0 +1,156 @@ +#!/bin/sh -e + +# shellcheck disable=SC2034 # This is intended to be included +JITSI_NGINX_CONFIG="$(cat < + + +COPYING +------- +Copyright \(C) 2020 Evilham. diff --git a/type/__evilham_jitsi_meet_domain/manifest b/type/__evilham_jitsi_meet_domain/manifest new file mode 100755 index 0000000..8d28256 --- /dev/null +++ b/type/__evilham_jitsi_meet_domain/manifest @@ -0,0 +1,88 @@ +#!/bin/sh -e + +os="$(cat "${__global}/explorer/os")" +case "${os}" in + devuan|debian) + ;; + *) + echo "Your OS '${os}' is currently not supported." > /dev/stderr + exit 1 + ;; +esac + +DOMAIN="${__object_id}" +ADMIN_EMAIL="$(cat "${__object}/parameter/admin-email")" +CHANNEL_LAST_N="$(cat "${__object}/parameter/channel-last-n")" +DEFAULT_LANGUAGE="$(cat "${__object}/parameter/default-language")" +NOTICE_MESSAGE="$(cat "${__object}/parameter/notice-message")" +START_VIDEO_MUTED="$(cat "${__object}/parameter/start-video-muted")" +TURN_SERVER="$(cat "${__object}/parameter/turn-server")" +VIDEO_CONSTRAINTS="$(cat "${__object}/parameter/video-constraints")" +BRANDING_INDEX="$(cat "${__object}/parameter/branding-index")" +BRANDING_JSON="$(cat "${__object}/parameter/branding-json")" +BRANDING_WATERMARK="$(cat "${__object}/parameter/branding-watermark")" + +if [ -f "${__object}/parameter/enable-third-party-requests" ]; then + ENABLE_THIRD_PARTY_REQUESTS="YES" +fi +if [ -f "${__object}/parameter/disable-audio-levels" ]; then + DISABLE_AUDIO_LEVELS="YES" +fi + +if [ -z "${TURN_SERVER}" ]; then + TURN_SERVER="${__target_host}" +fi +if [ -z "${JITSI_HOST}" ]; then + JITSI_HOST="${__target_host}" +fi + +# +# Deal with certbot +# +# use object id as domain +__letsencrypt_cert "${DOMAIN}" \ + --admin-email "${ADMIN_EMAIL}" \ + --automatic-renewal \ + --renew-hook "service nginx reload" \ + --webroot /usr/share/jitsi-meet + +# Create virtualhost for nginx +. "${__type}/files/nginx.sh" # This defines JITSI_NGINX_CONFIG +require="__letsencrypt_cert/${DOMAIN}" __file \ + "/etc/nginx/sites-enabled/${DOMAIN}.conf" \ + --mode 0644 --source "-" < Date: Mon, 11 Jan 2021 12:01:04 +0100 Subject: [PATCH 185/332] 'unbrand' evilham's jitsi types --- .../prometheus-jitsi-meet-explorer-version | 0 .../files/debconf_settings.sh | 0 .../files/ufw | 0 .../gencode-remote | 0 .../man.rst | 14 +++++++------- .../manifest | 0 .../parameter/boolean | 0 .../parameter/default/turn-server | 0 .../parameter/optional | 0 .../singleton | 0 .../files/config.js.sh | 0 .../files/config.js.sh.orig | 0 .../files/nginx.sh | 0 .../files/nginx.sh.orig | 0 .../gencode-remote | 0 .../man.rst | 17 ++++++++--------- .../manifest | 0 .../parameter/boolean | 0 .../parameter/default/branding-index | 0 .../parameter/default/branding-json | 0 .../parameter/default/branding-watermark | 0 .../parameter/default/channel-last-n | 0 .../parameter/default/default-language | 0 .../parameter/default/notice-message | 0 .../parameter/default/start-video-muted | 0 .../parameter/default/turn-server | 0 .../parameter/default/video-constraints | 0 .../parameter/optional | 0 .../parameter/required | 0 29 files changed, 15 insertions(+), 16 deletions(-) rename type/{__evilham_jitsi_meet => __jitsi_meet}/explorer/prometheus-jitsi-meet-explorer-version (100%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/files/debconf_settings.sh (100%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/files/ufw (100%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/gencode-remote (100%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/man.rst (82%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/manifest (100%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/parameter/boolean (100%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/parameter/default/turn-server (100%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/parameter/optional (100%) rename type/{__evilham_jitsi_meet => __jitsi_meet}/singleton (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/files/config.js.sh (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/files/config.js.sh.orig (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/files/nginx.sh (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/files/nginx.sh.orig (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/gencode-remote (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/man.rst (87%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/manifest (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/boolean (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/branding-index (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/branding-json (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/branding-watermark (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/channel-last-n (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/default-language (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/notice-message (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/start-video-muted (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/turn-server (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/default/video-constraints (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/optional (100%) rename type/{__evilham_jitsi_meet_domain => __jitsi_meet_domain}/parameter/required (100%) diff --git a/type/__evilham_jitsi_meet/explorer/prometheus-jitsi-meet-explorer-version b/type/__jitsi_meet/explorer/prometheus-jitsi-meet-explorer-version similarity index 100% rename from type/__evilham_jitsi_meet/explorer/prometheus-jitsi-meet-explorer-version rename to type/__jitsi_meet/explorer/prometheus-jitsi-meet-explorer-version diff --git a/type/__evilham_jitsi_meet/files/debconf_settings.sh b/type/__jitsi_meet/files/debconf_settings.sh similarity index 100% rename from type/__evilham_jitsi_meet/files/debconf_settings.sh rename to type/__jitsi_meet/files/debconf_settings.sh diff --git a/type/__evilham_jitsi_meet/files/ufw b/type/__jitsi_meet/files/ufw similarity index 100% rename from type/__evilham_jitsi_meet/files/ufw rename to type/__jitsi_meet/files/ufw diff --git a/type/__evilham_jitsi_meet/gencode-remote b/type/__jitsi_meet/gencode-remote similarity index 100% rename from type/__evilham_jitsi_meet/gencode-remote rename to type/__jitsi_meet/gencode-remote diff --git a/type/__evilham_jitsi_meet/man.rst b/type/__jitsi_meet/man.rst similarity index 82% rename from type/__evilham_jitsi_meet/man.rst rename to type/__jitsi_meet/man.rst index b97130b..3d02346 100644 --- a/type/__evilham_jitsi_meet/man.rst +++ b/type/__jitsi_meet/man.rst @@ -1,10 +1,10 @@ -cdist-type__evilham_jitsi_meet(7) +cdist-type__jitsi_meet(7) ================================= NAME ---- -cdist-type__evilham_jitsi_meet - Setup the server-side of Jitsi-Meet. +cdist-type__jitsi_meet - Setup the server-side of Jitsi-Meet. DESCRIPTION @@ -17,8 +17,8 @@ eXO's notes for installing and managing Jitsi Meet instances. This type also sets up nginx in a way that is compatible with `__letsencrypt_cert` and assumes that it will only serve Jitsi instances. -You will also need the `__evilham_jitsi_meet_domain` type in order to finish -setting up the web frontend (including TLS certificates) and its settings. +You will also need the `__jitsi_meet_domain` type in order to finish setting up +the web frontend (including TLS certificates) and its settings. You may want to use the `files/ufw` example manifest for a `__ufw`-based firewall compatible with this type. @@ -60,17 +60,17 @@ EXAMPLES .. code-block:: sh # Setup the firewall - . "${__global}/type/__evilham_jitsi_meet/files/ufw" + . "${__global}/type/__jitsi_meet/files/ufw" export require="__ufw" # Setup Jitsi on this host - __evilham_jitsi_meet \ + __jitsi_meet \ --turn-server "turn.exo.cat" \ --turn-secret "WeNeedGoodSecurity" SEE ALSO -------- -- `__evilham_jitsi_meet_domain` +- `__jitsi_meet_domain` diff --git a/type/__evilham_jitsi_meet/manifest b/type/__jitsi_meet/manifest similarity index 100% rename from type/__evilham_jitsi_meet/manifest rename to type/__jitsi_meet/manifest diff --git a/type/__evilham_jitsi_meet/parameter/boolean b/type/__jitsi_meet/parameter/boolean similarity index 100% rename from type/__evilham_jitsi_meet/parameter/boolean rename to type/__jitsi_meet/parameter/boolean diff --git a/type/__evilham_jitsi_meet/parameter/default/turn-server b/type/__jitsi_meet/parameter/default/turn-server similarity index 100% rename from type/__evilham_jitsi_meet/parameter/default/turn-server rename to type/__jitsi_meet/parameter/default/turn-server diff --git a/type/__evilham_jitsi_meet/parameter/optional b/type/__jitsi_meet/parameter/optional similarity index 100% rename from type/__evilham_jitsi_meet/parameter/optional rename to type/__jitsi_meet/parameter/optional diff --git a/type/__evilham_jitsi_meet/singleton b/type/__jitsi_meet/singleton similarity index 100% rename from type/__evilham_jitsi_meet/singleton rename to type/__jitsi_meet/singleton diff --git a/type/__evilham_jitsi_meet_domain/files/config.js.sh b/type/__jitsi_meet_domain/files/config.js.sh similarity index 100% rename from type/__evilham_jitsi_meet_domain/files/config.js.sh rename to type/__jitsi_meet_domain/files/config.js.sh diff --git a/type/__evilham_jitsi_meet_domain/files/config.js.sh.orig b/type/__jitsi_meet_domain/files/config.js.sh.orig similarity index 100% rename from type/__evilham_jitsi_meet_domain/files/config.js.sh.orig rename to type/__jitsi_meet_domain/files/config.js.sh.orig diff --git a/type/__evilham_jitsi_meet_domain/files/nginx.sh b/type/__jitsi_meet_domain/files/nginx.sh similarity index 100% rename from type/__evilham_jitsi_meet_domain/files/nginx.sh rename to type/__jitsi_meet_domain/files/nginx.sh diff --git a/type/__evilham_jitsi_meet_domain/files/nginx.sh.orig b/type/__jitsi_meet_domain/files/nginx.sh.orig similarity index 100% rename from type/__evilham_jitsi_meet_domain/files/nginx.sh.orig rename to type/__jitsi_meet_domain/files/nginx.sh.orig diff --git a/type/__evilham_jitsi_meet_domain/gencode-remote b/type/__jitsi_meet_domain/gencode-remote similarity index 100% rename from type/__evilham_jitsi_meet_domain/gencode-remote rename to type/__jitsi_meet_domain/gencode-remote diff --git a/type/__evilham_jitsi_meet_domain/man.rst b/type/__jitsi_meet_domain/man.rst similarity index 87% rename from type/__evilham_jitsi_meet_domain/man.rst rename to type/__jitsi_meet_domain/man.rst index ea997f4..862701a 100644 --- a/type/__evilham_jitsi_meet_domain/man.rst +++ b/type/__jitsi_meet_domain/man.rst @@ -1,10 +1,10 @@ -cdist-type__evilham_jitsi_meet_domain(7) +cdist-type__jitsi_meet_domain(7) ======================================== NAME ---- -cdist-type__evilham_jitsi_meet_domain - Setup a frontend for Jitsi-Meet. +cdist-type__jitsi_meet_domain - Setup a frontend for Jitsi-Meet. DESCRIPTION @@ -20,9 +20,8 @@ This is due to the underlying XMPP and signaling rooms being common. There might be a way to perform tricks on the Nginx-side to avoid this, but time is lacking :-). -This assumes `__evilham_jitsi_meet` has already been ran on the target host, -and, amongst others, that Jitsi was set up with `__target_host` as the Jitsi -domain. +This assumes `__jitsi_meet` has already been ran on the target host, and, +amongst others, that Jitsi was set up with `__target_host` as the Jitsi domain. This type will take care of TLS settings, branding and client-side configuration for Jitsi. @@ -73,13 +72,13 @@ video-constraints this config's resolution value. Defaults to requesting an ideal resolution of 720p. It must not have a trailing comma, see `constraints` in - `__evilham_jitsi_meet_domain/files/config.js.sh`. + `__jitsi_meet_domain/files/config.js.sh`. branding-json Path to a JSON file that will be served as the `brandingDataUrl`. For information on the format see `brandingDataUrl` in - `__evilham_jitsi_meet_domain/files/config.js.sh`. + `__jitsi_meet_domain/files/config.js.sh`. If not set, no branding will be set up. @@ -114,7 +113,7 @@ EXAMPLES .. code-block:: sh # Setup a Jitsi frontend for jitsi.exo.cat - __evilham_jitsi_meet_domain "jitsi.exo.cat" \ + __jitsi_meet_domain "jitsi.exo.cat" \ --admin-email "info@exo.cat" \ --turn-server "turn.exo.cat" \ --notice-message "Hola!" \ @@ -137,7 +136,7 @@ EXAMPLES SEE ALSO -------- -- `__evilham_jitsi_meet` +- `__jitsi_meet` diff --git a/type/__evilham_jitsi_meet_domain/manifest b/type/__jitsi_meet_domain/manifest similarity index 100% rename from type/__evilham_jitsi_meet_domain/manifest rename to type/__jitsi_meet_domain/manifest diff --git a/type/__evilham_jitsi_meet_domain/parameter/boolean b/type/__jitsi_meet_domain/parameter/boolean similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/boolean rename to type/__jitsi_meet_domain/parameter/boolean diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/branding-index b/type/__jitsi_meet_domain/parameter/default/branding-index similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/branding-index rename to type/__jitsi_meet_domain/parameter/default/branding-index diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/branding-json b/type/__jitsi_meet_domain/parameter/default/branding-json similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/branding-json rename to type/__jitsi_meet_domain/parameter/default/branding-json diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/branding-watermark b/type/__jitsi_meet_domain/parameter/default/branding-watermark similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/branding-watermark rename to type/__jitsi_meet_domain/parameter/default/branding-watermark diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/channel-last-n b/type/__jitsi_meet_domain/parameter/default/channel-last-n similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/channel-last-n rename to type/__jitsi_meet_domain/parameter/default/channel-last-n diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/default-language b/type/__jitsi_meet_domain/parameter/default/default-language similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/default-language rename to type/__jitsi_meet_domain/parameter/default/default-language diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/notice-message b/type/__jitsi_meet_domain/parameter/default/notice-message similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/notice-message rename to type/__jitsi_meet_domain/parameter/default/notice-message diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/start-video-muted b/type/__jitsi_meet_domain/parameter/default/start-video-muted similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/start-video-muted rename to type/__jitsi_meet_domain/parameter/default/start-video-muted diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/turn-server b/type/__jitsi_meet_domain/parameter/default/turn-server similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/turn-server rename to type/__jitsi_meet_domain/parameter/default/turn-server diff --git a/type/__evilham_jitsi_meet_domain/parameter/default/video-constraints b/type/__jitsi_meet_domain/parameter/default/video-constraints similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/default/video-constraints rename to type/__jitsi_meet_domain/parameter/default/video-constraints diff --git a/type/__evilham_jitsi_meet_domain/parameter/optional b/type/__jitsi_meet_domain/parameter/optional similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/optional rename to type/__jitsi_meet_domain/parameter/optional diff --git a/type/__evilham_jitsi_meet_domain/parameter/required b/type/__jitsi_meet_domain/parameter/required similarity index 100% rename from type/__evilham_jitsi_meet_domain/parameter/required rename to type/__jitsi_meet_domain/parameter/required From 367cbde75b81373d125ea128daaab5e1de8b38c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 Jan 2021 14:10:55 +0100 Subject: [PATCH 186/332] Import evilham's runit types as it --- type/__evilham_runit/explorer/runit-etc | 7 +++ type/__evilham_runit/explorer/svdir-exists | 7 +++ type/__evilham_runit/gencode-remote | 21 +++++++ type/__evilham_runit/man.rst | 46 +++++++++++++++ type/__evilham_runit/manifest | 10 ++++ type/__evilham_runit/singleton | 0 type/__evilham_runit_service/man.rst | 58 +++++++++++++++++++ type/__evilham_runit_service/manifest | 33 +++++++++++ .../__evilham_runit_service/parameter/boolean | 1 + .../parameter/required | 1 + 10 files changed, 184 insertions(+) create mode 100755 type/__evilham_runit/explorer/runit-etc create mode 100755 type/__evilham_runit/explorer/svdir-exists create mode 100755 type/__evilham_runit/gencode-remote create mode 100644 type/__evilham_runit/man.rst create mode 100755 type/__evilham_runit/manifest create mode 100644 type/__evilham_runit/singleton create mode 100644 type/__evilham_runit_service/man.rst create mode 100755 type/__evilham_runit_service/manifest create mode 100644 type/__evilham_runit_service/parameter/boolean create mode 100644 type/__evilham_runit_service/parameter/required diff --git a/type/__evilham_runit/explorer/runit-etc b/type/__evilham_runit/explorer/runit-etc new file mode 100755 index 0000000..4511621 --- /dev/null +++ b/type/__evilham_runit/explorer/runit-etc @@ -0,0 +1,7 @@ +#!/bin/sh -e + +runit_etc_dir="/etc/runit" + +if [ -d "${runit_etc_dir}" ]; then + echo "YES" +fi diff --git a/type/__evilham_runit/explorer/svdir-exists b/type/__evilham_runit/explorer/svdir-exists new file mode 100755 index 0000000..4eca3d4 --- /dev/null +++ b/type/__evilham_runit/explorer/svdir-exists @@ -0,0 +1,7 @@ +#!/bin/sh -e + +svdir="/var/service" + +if [ -d "${svdir}" ]; then + echo "YES" +fi diff --git a/type/__evilham_runit/gencode-remote b/type/__evilham_runit/gencode-remote new file mode 100755 index 0000000..fd2a3e0 --- /dev/null +++ b/type/__evilham_runit/gencode-remote @@ -0,0 +1,21 @@ +#!/bin/sh -e + +svdir="/var/service" +svdir_exists="$(cat "${__object}/explorer/svdir-exists")" +runit_etc="$(cat "${__object}/explorer/runit-etc")" + +if [ -z "${svdir_exists}" ]; then + echo "mkdir '${svdir}'" + sv_restart="YES" +fi +if [ -z "${runit_etc}" ]; then + echo "cp -R '/usr/local/etc/runit' '/etc/runit'" + sv_restart="YES" +fi +if grep -q "^__key_value/runsvdir_enable" "${__messages_in}"; then + sv_restart="YES" +fi + +if [ -n "${sv_restart}" ]; then + echo "service runsvdir restart || service runsvdir start" +fi diff --git a/type/__evilham_runit/man.rst b/type/__evilham_runit/man.rst new file mode 100644 index 0000000..bd1b80c --- /dev/null +++ b/type/__evilham_runit/man.rst @@ -0,0 +1,46 @@ +cdist-type__evilham_runit(7) +============================ + +NAME +---- +cdist-type__evilham_runit - Install and configure runit + + +DESCRIPTION +----------- +This is a singleton type. + +Install and configure runit, not as an init system, but as a service monitor. +It configures and if necessary starts runsvdir as documented for the +Operating System. + +This type currently heavily focuses on FreeBSD, support for other Operating +Systems can be achieved but no effort whatsoever has been put into it. + + +REQUIRED PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __evilham_runit + +SEE ALSO +-------- +:strong:`cdist-type__evilham_runit_service`\ (7) + +AUTHORS +------- +Evilham + +COPYING +------- +Copyright \(C) 2020 Evilham. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__evilham_runit/manifest b/type/__evilham_runit/manifest new file mode 100755 index 0000000..195a70e --- /dev/null +++ b/type/__evilham_runit/manifest @@ -0,0 +1,10 @@ +#!/bin/sh -e + +__package "runit" + +__key_value \ + --file "/etc/rc.conf" \ + --key "runsvdir_enable" \ + --delimiter "=" \ + --value "yes" \ + "runsvdir_enable" diff --git a/type/__evilham_runit/singleton b/type/__evilham_runit/singleton new file mode 100644 index 0000000..e69de29 diff --git a/type/__evilham_runit_service/man.rst b/type/__evilham_runit_service/man.rst new file mode 100644 index 0000000..98bbc35 --- /dev/null +++ b/type/__evilham_runit_service/man.rst @@ -0,0 +1,58 @@ +cdist-type__evilham_runit_service(7) +==================================== + +NAME +---- +cdist-type__evilham_runit_service - Create a runit-compatible service dir. + + +DESCRIPTION +----------- +Create a directory structure compatible with runit-like service management. + +Note that sv(8) and runsvdir(8) must be present on the target system, this can +be achieved with e.g. `__runit`. + +The `__object_id` will be used as the service name. + + +REQUIRED PARAMETERS +------------------- +source + File to save as /run. If set to '-', standard input will be used. + + +BOOLEAN PARAMETERS +------------------ +log + Setup logging with `svlogd -tt ./main`. + + +EXAMPLES +-------- + +.. code-block:: sh + + require="__evilham_runit" __evilham_runit_service tasksched \ + --source - << EOF + #!/bin/sh -e + cd "${HOME}/.local/share/tasksched" + exec ./server.js 2>&1 + EOF + + +SEE ALSO +-------- +:strong:`cdist-type__evilham_runit`\ (7) + + +AUTHORS +------- +Evilham + +COPYING +------- +Copyright \(C) 2020 Evilham. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__evilham_runit_service/manifest b/type/__evilham_runit_service/manifest new file mode 100755 index 0000000..29f3312 --- /dev/null +++ b/type/__evilham_runit_service/manifest @@ -0,0 +1,33 @@ +#!/bin/sh -e + +svdir="/var/service" +sv="${__object_id}" +state="present" +run_file="${svdir}/${sv}/run" + +source="$(cat "$__object/parameter/source")" +if [ "$source" = "-" ]; then + source="$__object/stdin" +fi + +# Create this service's directory +__directory --state "${state}" "${svdir}/${sv}" + +export require="__directory${svdir}/${sv}" + + +if [ -f "${__object}/parameter/log" ]; then + # Setup logger if requested + __directory --parents "${svdir}/${sv}/log/main" + export require="${require} __directory${svdir}/${sv}/log/main" + __file "${svdir}/${sv}/log/run" \ + --state "${state}" \ + --mode 0755 \ + --source "-" < Date: Mon, 11 Jan 2021 14:13:46 +0100 Subject: [PATCH 187/332] 'unbrand' evilham's runit types --- type/__jitsi_meet/manifest | 6 +++--- type/{__evilham_runit => __runit}/explorer/runit-etc | 0 type/{__evilham_runit => __runit}/explorer/svdir-exists | 0 type/{__evilham_runit => __runit}/gencode-remote | 0 type/{__evilham_runit => __runit}/man.rst | 8 ++++---- type/{__evilham_runit => __runit}/manifest | 0 type/{__evilham_runit => __runit}/singleton | 0 type/{__evilham_runit_service => __runit_service}/man.rst | 8 ++++---- .../{__evilham_runit_service => __runit_service}/manifest | 0 .../parameter/boolean | 0 .../parameter/required | 0 11 files changed, 11 insertions(+), 11 deletions(-) rename type/{__evilham_runit => __runit}/explorer/runit-etc (100%) rename type/{__evilham_runit => __runit}/explorer/svdir-exists (100%) rename type/{__evilham_runit => __runit}/gencode-remote (100%) rename type/{__evilham_runit => __runit}/man.rst (84%) rename type/{__evilham_runit => __runit}/manifest (100%) rename type/{__evilham_runit => __runit}/singleton (100%) rename type/{__evilham_runit_service => __runit_service}/man.rst (82%) rename type/{__evilham_runit_service => __runit_service}/manifest (100%) rename type/{__evilham_runit_service => __runit_service}/parameter/boolean (100%) rename type/{__evilham_runit_service => __runit_service}/parameter/required (100%) diff --git a/type/__jitsi_meet/manifest b/type/__jitsi_meet/manifest index 83d9dab..ec65883 100755 --- a/type/__jitsi_meet/manifest +++ b/type/__jitsi_meet/manifest @@ -145,8 +145,8 @@ PROMETHEUS_JITSI_EXPORTER_VERSION_FILE="/usr/local/bin/.prometheus-jitsi-meet-ex if [ ! -f "${__object}/parameter/disable-prometheus-exporter" ]; then case "${init}" in init|sysvinit) - __evilham_runit - require="__evilham_runit" __evilham_runit_service \ + __runit + require="__runit" __runit_service \ prometheus-jitsi-meet-exporter --log --source - <&1 EOF - export require="__evilham_runit_service/prometheus-jitsi-meet-exporter" + export require="__runit_service/prometheus-jitsi-meet-exporter" JITSI_MEET_EXPORTER_SERVICE="sv %s prometheus-jitsi-meet-exporter" ;; systemd) diff --git a/type/__evilham_runit/explorer/runit-etc b/type/__runit/explorer/runit-etc similarity index 100% rename from type/__evilham_runit/explorer/runit-etc rename to type/__runit/explorer/runit-etc diff --git a/type/__evilham_runit/explorer/svdir-exists b/type/__runit/explorer/svdir-exists similarity index 100% rename from type/__evilham_runit/explorer/svdir-exists rename to type/__runit/explorer/svdir-exists diff --git a/type/__evilham_runit/gencode-remote b/type/__runit/gencode-remote similarity index 100% rename from type/__evilham_runit/gencode-remote rename to type/__runit/gencode-remote diff --git a/type/__evilham_runit/man.rst b/type/__runit/man.rst similarity index 84% rename from type/__evilham_runit/man.rst rename to type/__runit/man.rst index bd1b80c..d58e143 100644 --- a/type/__evilham_runit/man.rst +++ b/type/__runit/man.rst @@ -1,9 +1,9 @@ -cdist-type__evilham_runit(7) +cdist-type__runit(7) ============================ NAME ---- -cdist-type__evilham_runit - Install and configure runit +cdist-type__runit - Install and configure runit DESCRIPTION @@ -28,11 +28,11 @@ EXAMPLES .. code-block:: sh - __evilham_runit + __runit SEE ALSO -------- -:strong:`cdist-type__evilham_runit_service`\ (7) +:strong:`cdist-type__runit_service`\ (7) AUTHORS ------- diff --git a/type/__evilham_runit/manifest b/type/__runit/manifest similarity index 100% rename from type/__evilham_runit/manifest rename to type/__runit/manifest diff --git a/type/__evilham_runit/singleton b/type/__runit/singleton similarity index 100% rename from type/__evilham_runit/singleton rename to type/__runit/singleton diff --git a/type/__evilham_runit_service/man.rst b/type/__runit_service/man.rst similarity index 82% rename from type/__evilham_runit_service/man.rst rename to type/__runit_service/man.rst index 98bbc35..7b1db84 100644 --- a/type/__evilham_runit_service/man.rst +++ b/type/__runit_service/man.rst @@ -1,9 +1,9 @@ -cdist-type__evilham_runit_service(7) +cdist-type__runit_service(7) ==================================== NAME ---- -cdist-type__evilham_runit_service - Create a runit-compatible service dir. +cdist-type__runit_service - Create a runit-compatible service dir. DESCRIPTION @@ -33,7 +33,7 @@ EXAMPLES .. code-block:: sh - require="__evilham_runit" __evilham_runit_service tasksched \ + require="__runit" __runit_service tasksched \ --source - << EOF #!/bin/sh -e cd "${HOME}/.local/share/tasksched" @@ -43,7 +43,7 @@ EXAMPLES SEE ALSO -------- -:strong:`cdist-type__evilham_runit`\ (7) +:strong:`cdist-type__runit`\ (7) AUTHORS diff --git a/type/__evilham_runit_service/manifest b/type/__runit_service/manifest similarity index 100% rename from type/__evilham_runit_service/manifest rename to type/__runit_service/manifest diff --git a/type/__evilham_runit_service/parameter/boolean b/type/__runit_service/parameter/boolean similarity index 100% rename from type/__evilham_runit_service/parameter/boolean rename to type/__runit_service/parameter/boolean diff --git a/type/__evilham_runit_service/parameter/required b/type/__runit_service/parameter/required similarity index 100% rename from type/__evilham_runit_service/parameter/required rename to type/__runit_service/parameter/required From 7cef989b1fb4c92e53c20768d65c5e8d4f3406eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 Jan 2021 07:14:24 +0100 Subject: [PATCH 188/332] Fix run-shellcheck.sh following 'conflict resolution' from gitlab web ui --- scripts/run-shellcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-shellcheck.sh b/scripts/run-shellcheck.sh index fcca722..d6c2db6 100755 --- a/scripts/run-shellcheck.sh +++ b/scripts/run-shellcheck.sh @@ -17,7 +17,7 @@ check() { rm -f "${SHELLCHECKTMP}" check -path '*/explorer/*' -check -path '*/files/*' ! -name '*.awk' ! -name '*.py' +check -path '*/files/*' -name '*.sh' check -name manifest check -name gencode-local check -name gencode-remote From 0e4bc443e2a1dcf703174df6c885281a0a6a64d3 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 14 Aug 2020 11:43:41 +0200 Subject: [PATCH 189/332] Add a type for wikijs. --- type/__wikijs/files/config.yml.sh | 34 ++++++++++++++++++ type/__wikijs/files/wikijs-openrc | 10 ++++++ type/__wikijs/gencode-remote | 25 +++++++++++++ type/__wikijs/man.rst | 55 ++++++++++++++++++++++++++++ type/__wikijs/manifest | 60 +++++++++++++++++++++++++++++++ type/__wikijs/parameter/boolean | 1 + type/__wikijs/parameter/optional | 3 ++ type/__wikijs/parameter/required | 1 + type/__wikijs/singleton | 0 9 files changed, 189 insertions(+) create mode 100755 type/__wikijs/files/config.yml.sh create mode 100644 type/__wikijs/files/wikijs-openrc create mode 100644 type/__wikijs/gencode-remote create mode 100644 type/__wikijs/man.rst create mode 100644 type/__wikijs/manifest create mode 100644 type/__wikijs/parameter/boolean create mode 100644 type/__wikijs/parameter/optional create mode 100644 type/__wikijs/parameter/required create mode 100644 type/__wikijs/singleton diff --git a/type/__wikijs/files/config.yml.sh b/type/__wikijs/files/config.yml.sh new file mode 100755 index 0000000..6f9943f --- /dev/null +++ b/type/__wikijs/files/config.yml.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +if [ $# -ne 1 ]; +then + echo "You have to give me the database password as an argument:" + echo "on some systems, anyone can read env(1)." + exit 1; +fi + +cat << EOF +port: 80 +db: + type: postgres + host: localhost + port: 5432 + user: ${DB_USER:?} + pass: $1 + db: ${DB_NAME:?} + ssl: false +ssl: + enabled: ${SSL} + port: 443 + provider: letsencrypt + domain: ${__target_host:?} + subscriberEmail: ${LE_EMAIL:?} +pool: + min: 2 + max: 10 +bindIP: 0.0.0.0 +logLevel: warn +offline: false +ha: false +dataPath: ./data +EOF diff --git a/type/__wikijs/files/wikijs-openrc b/type/__wikijs/files/wikijs-openrc new file mode 100644 index 0000000..e484647 --- /dev/null +++ b/type/__wikijs/files/wikijs-openrc @@ -0,0 +1,10 @@ +#!/sbin/openrc-run + +command='/usr/bin/node' +command_args='server' +command_background=true +description="Run wiki.js" +directory='/var/wiki' +error_log=/var/log/"$RC_SVCNAME".err +output_log=/var/log/"$RC_SVCNAME".log +pidfile="/run/$RC_SVCNAME.pid" diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote new file mode 100644 index 0000000..2faf559 --- /dev/null +++ b/type/__wikijs/gencode-remote @@ -0,0 +1,25 @@ +#!/bin/sh + +# Check for installation +cat << EOF +if [ -f '/var/wiki/LICENSE' ]; +then + # Assume everything is done already. + exit 0; +fi +EOF + +# Download and copy source +cat << EOF +TMPDIR=\$(mktemp -d) +cd \$TMPDIR || exit 1 +wget https://github.com/Requarks/wiki/releases/download/2.4.107/wiki-js.tar.gz +tar xf wiki-js.tar.gz -C /var/wiki +EOF + +# Install deps and launch +cat << EOF +cd /var/wiki || exit 1 +npm install +service start wikijs +EOF diff --git a/type/__wikijs/man.rst b/type/__wikijs/man.rst new file mode 100644 index 0000000..6573b60 --- /dev/null +++ b/type/__wikijs/man.rst @@ -0,0 +1,55 @@ +cdist-type__wikijs(7) +======================== + +NAME +---- +cdist-type__wikijs - Deploy the wiki.js software. + +DESCRIPTION +----------- + +See wiki.js.org for more information. This type deploys with a postgresql +database, since it is the upstream recommended for production, and they seem to +strongly suggest that in thenext releases, they will not support anything else. + +Currently, this type servers wikijs as standalone, listening on ports 80 and +443, and with a service file for OpenRC. Feel free to contribute a +generalisation if you require one. + +REQUIRED PARAMETERS +------------------- + +database-password + The password to the PSQL database. + +OPTIONAL PARAMETERS +------------------- + +database + The name of the PSQL database to connect to. If omitted, then 'wikijs' is + used. + +database-user + The name of the PSQL database user to connec as. If omitted, then 'wikijs' is + used. + +letsencrypt-mail + If the SSL parameter is passed, then we setup wikijs to automatically obtain + certificates: this is the email used to sign up to a LE account. + +BOOLEAN PARAMETERS +------------------ + +ssl + Whether or not to enable the wikijs automatic obtention of LE certificates. + +AUTHORS +------- +Joachim Desroches + +COPYING +------- +Copyright \(C) 2020 Joachim Desroches. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__wikijs/manifest b/type/__wikijs/manifest new file mode 100644 index 0000000..7ee42ac --- /dev/null +++ b/type/__wikijs/manifest @@ -0,0 +1,60 @@ +#!/bin/sh + +os="$(cat "${__global:?}"/explorer/os)" + +case "$os" in + alpine) + : + ;; + *) + echo "This type has no implementation for $os. Aborting." >&2; + exit 1; +esac + +DB_USER=wikijs +if [ -f "${__object:?}/parameter/database-user" ]; +then + DB_USER="$(cat "${__object:?}/parameter/database-user")" +fi +export DB_USER + +DB_NAME=wikijs +if [ -f "${__object:?}/parameter/database" ]; +then + DB_NAME="$(cat "${__object:?}/parameter/database")" +fi +export DB_NAME + +SSL=false +if [ -f "${__object:?}/parameter/ssl" ]; +then + SSL=true +fi +export SSL + +if ! [ "$SSL" = "false" ]; +then + if [ -f "${__object:?}/parameter/letsencrypt-mail" ]; + then + LE_EMAIL="$(cat "${__object:?}/parameter/letsencrypt-mail")" + export LE_EMAIL + else + echo "You must specify an email account if you request SSL." + echo "Hit me." + exit 1 + fi +fi + +db_pass="$(cat "${__object:?}/parameter/database-password")" + +__package nodejs +__package nghttp2-dev # Required for some reason, else a symbol is missing +__package npm +__directory /var/wiki/ +__file /etc/init.d/wikijs --source "${__files:?}/files/wikijs-openrc" +require='__file/etc/init.d/wikijs' __start_on_boot wikijs + +mkdir -p "${__object:?}/files" +"${__type:?}/files/config.yml.sh" "$db_pass" > "${__object:?}/files/config.yml" +require='__directory/var/wiki' \ + __file /var/wiki/config.yml --source "${__object:?}/files/config.yml" diff --git a/type/__wikijs/parameter/boolean b/type/__wikijs/parameter/boolean new file mode 100644 index 0000000..a2647ce --- /dev/null +++ b/type/__wikijs/parameter/boolean @@ -0,0 +1 @@ +ssl diff --git a/type/__wikijs/parameter/optional b/type/__wikijs/parameter/optional new file mode 100644 index 0000000..9c309c9 --- /dev/null +++ b/type/__wikijs/parameter/optional @@ -0,0 +1,3 @@ +database +database-user +letsencrypt-mail diff --git a/type/__wikijs/parameter/required b/type/__wikijs/parameter/required new file mode 100644 index 0000000..8a109a1 --- /dev/null +++ b/type/__wikijs/parameter/required @@ -0,0 +1 @@ +database-password diff --git a/type/__wikijs/singleton b/type/__wikijs/singleton new file mode 100644 index 0000000..e69de29 From ef748cf8e248f4f3f4eed88e433fc34ab24b0cb5 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 14 Aug 2020 13:02:17 +0200 Subject: [PATCH 190/332] Fix typo and drop use of useless tempdir. --- type/__wikijs/gencode-remote | 5 +---- type/__wikijs/man.rst | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote index 2faf559..66d7df7 100644 --- a/type/__wikijs/gencode-remote +++ b/type/__wikijs/gencode-remote @@ -11,10 +11,7 @@ EOF # Download and copy source cat << EOF -TMPDIR=\$(mktemp -d) -cd \$TMPDIR || exit 1 -wget https://github.com/Requarks/wiki/releases/download/2.4.107/wiki-js.tar.gz -tar xf wiki-js.tar.gz -C /var/wiki +wget -O - https://github.com/Requarks/wiki/releases/download/2.4.107/wiki-js.tar.gz | tar xz -C /var/wiki EOF # Install deps and launch diff --git a/type/__wikijs/man.rst b/type/__wikijs/man.rst index 6573b60..50dcd1d 100644 --- a/type/__wikijs/man.rst +++ b/type/__wikijs/man.rst @@ -10,7 +10,7 @@ DESCRIPTION See wiki.js.org for more information. This type deploys with a postgresql database, since it is the upstream recommended for production, and they seem to -strongly suggest that in thenext releases, they will not support anything else. +strongly suggest that in the next releases, they will not support anything else. Currently, this type servers wikijs as standalone, listening on ports 80 and 443, and with a service file for OpenRC. Feel free to contribute a From 647833580df813bdd105a7e0c28ca1d004ded668 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Fri, 14 Aug 2020 13:46:05 +0200 Subject: [PATCH 191/332] Split out service management from the installation type. --- type/__wikijs/gencode-remote | 1 - type/__wikijs/manifest | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote index 66d7df7..a45ac22 100644 --- a/type/__wikijs/gencode-remote +++ b/type/__wikijs/gencode-remote @@ -18,5 +18,4 @@ EOF cat << EOF cd /var/wiki || exit 1 npm install -service start wikijs EOF diff --git a/type/__wikijs/manifest b/type/__wikijs/manifest index 7ee42ac..2e3e96c 100644 --- a/type/__wikijs/manifest +++ b/type/__wikijs/manifest @@ -48,11 +48,12 @@ fi db_pass="$(cat "${__object:?}/parameter/database-password")" __package nodejs -__package nghttp2-dev # Required for some reason, else a symbol is missing __package npm __directory /var/wiki/ + +# These things are Alpine-dependant. __file /etc/init.d/wikijs --source "${__files:?}/files/wikijs-openrc" -require='__file/etc/init.d/wikijs' __start_on_boot wikijs +__package nghttp2-dev # Required for some reason, else a symbol is missing mkdir -p "${__object:?}/files" "${__type:?}/files/config.yml.sh" "$db_pass" > "${__object:?}/files/config.yml" From 7122fe1bee6977794bb5e9cc5d2eb8c4f0380067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 Jan 2021 07:57:34 +0100 Subject: [PATCH 192/332] __wikijs: add version management and HTTP(S) port configuration --- type/__wikijs/files/config.yml.sh | 26 ++++++++++++++++------ type/__wikijs/gencode-remote | 14 ++++++++++-- type/__wikijs/man.rst | 9 ++++++++ type/__wikijs/manifest | 8 +++++-- type/__wikijs/parameter/default/http-port | 1 + type/__wikijs/parameter/default/https-port | 1 + type/__wikijs/parameter/optional | 2 ++ type/__wikijs/parameter/required | 1 + 8 files changed, 51 insertions(+), 11 deletions(-) mode change 100644 => 100755 type/__wikijs/gencode-remote create mode 100644 type/__wikijs/parameter/default/http-port create mode 100644 type/__wikijs/parameter/default/https-port diff --git a/type/__wikijs/files/config.yml.sh b/type/__wikijs/files/config.yml.sh index 6f9943f..b66687a 100755 --- a/type/__wikijs/files/config.yml.sh +++ b/type/__wikijs/files/config.yml.sh @@ -7,8 +7,25 @@ then exit 1; fi +generate_ssl_section () { + + cat << EOF +ssl: + enabled: ${SSL} +EOF + +if [ "$SSL" = "true" ]; then + cat << EOF + port: $HTTPS_PORT + provider: letsencrypt + domain: ${__target_host:?} + subscriberEmail: ${LE_EMAIL:?} +EOF + fi +} + cat << EOF -port: 80 +port: $HTTP_PORT db: type: postgres host: localhost @@ -17,12 +34,7 @@ db: pass: $1 db: ${DB_NAME:?} ssl: false -ssl: - enabled: ${SSL} - port: 443 - provider: letsencrypt - domain: ${__target_host:?} - subscriberEmail: ${LE_EMAIL:?} +$(generate_ssl_section) pool: min: 2 max: 10 diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote old mode 100644 new mode 100755 index a45ac22..81055e2 --- a/type/__wikijs/gencode-remote +++ b/type/__wikijs/gencode-remote @@ -1,17 +1,22 @@ #!/bin/sh +VERSION_FILE=/var/wiki/version +version=$(cat "${__object:?}/parameter/version") + # Check for installation cat << EOF -if [ -f '/var/wiki/LICENSE' ]; +if [ -f $VERSION_FILE ] && [ "\$(cat $VERSION_FILE)" = "$version" ]; then # Assume everything is done already. exit 0; +else + echo "$version" > $VERSION_FILE fi EOF # Download and copy source cat << EOF -wget -O - https://github.com/Requarks/wiki/releases/download/2.4.107/wiki-js.tar.gz | tar xz -C /var/wiki +wget -O - https://github.com/Requarks/wiki/releases/download/$version/wiki-js.tar.gz | tar xz -C /var/wiki EOF # Install deps and launch @@ -19,3 +24,8 @@ cat << EOF cd /var/wiki || exit 1 npm install EOF + +# Restart service. +cat << EOF +service wikijs restart +EOF diff --git a/type/__wikijs/man.rst b/type/__wikijs/man.rst index 50dcd1d..b259c90 100644 --- a/type/__wikijs/man.rst +++ b/type/__wikijs/man.rst @@ -22,6 +22,9 @@ REQUIRED PARAMETERS database-password The password to the PSQL database. +version + 'wikijs' version to be deployed. + OPTIONAL PARAMETERS ------------------- @@ -37,6 +40,12 @@ letsencrypt-mail If the SSL parameter is passed, then we setup wikijs to automatically obtain certificates: this is the email used to sign up to a LE account. +http-port + Specify HTTP port, defaults to 80. + +http-port + Specify HTTPS port, defaults to 443. Only relevant if the SSL flag is enabled. + BOOLEAN PARAMETERS ------------------ diff --git a/type/__wikijs/manifest b/type/__wikijs/manifest index 2e3e96c..04a21af 100644 --- a/type/__wikijs/manifest +++ b/type/__wikijs/manifest @@ -32,7 +32,7 @@ then fi export SSL -if ! [ "$SSL" = "false" ]; +if [ "$SSL" = "true" ]; then if [ -f "${__object:?}/parameter/letsencrypt-mail" ]; then @@ -45,6 +45,10 @@ then fi fi +HTTP_PORT=$(cat "${__object:?}/parameter/http-port") +HTTPS_PORT=$(cat "${__object:?}/parameter/https-port") +export HTTP_PORT HTTPS_PORT + db_pass="$(cat "${__object:?}/parameter/database-password")" __package nodejs @@ -52,7 +56,7 @@ __package npm __directory /var/wiki/ # These things are Alpine-dependant. -__file /etc/init.d/wikijs --source "${__files:?}/files/wikijs-openrc" +__file /etc/init.d/wikijs --source "${__type:?}/files/wikijs-openrc" __package nghttp2-dev # Required for some reason, else a symbol is missing mkdir -p "${__object:?}/files" diff --git a/type/__wikijs/parameter/default/http-port b/type/__wikijs/parameter/default/http-port new file mode 100644 index 0000000..d15a2cc --- /dev/null +++ b/type/__wikijs/parameter/default/http-port @@ -0,0 +1 @@ +80 diff --git a/type/__wikijs/parameter/default/https-port b/type/__wikijs/parameter/default/https-port new file mode 100644 index 0000000..6a13cf6 --- /dev/null +++ b/type/__wikijs/parameter/default/https-port @@ -0,0 +1 @@ +443 diff --git a/type/__wikijs/parameter/optional b/type/__wikijs/parameter/optional index 9c309c9..be19c92 100644 --- a/type/__wikijs/parameter/optional +++ b/type/__wikijs/parameter/optional @@ -1,3 +1,5 @@ database database-user letsencrypt-mail +http-port +https-port diff --git a/type/__wikijs/parameter/required b/type/__wikijs/parameter/required index 8a109a1..ae542bc 100644 --- a/type/__wikijs/parameter/required +++ b/type/__wikijs/parameter/required @@ -1 +1,2 @@ database-password +version From 8929c566fcc0493ee50ca18974ea5488aa2f196b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 Jan 2021 08:33:50 +0100 Subject: [PATCH 193/332] __wikijs: remove uneeded npm install call See https://github.com/Requarks/wiki/issues/1325. --- type/__wikijs/gencode-remote | 5 ----- type/__wikijs/manifest | 1 - 2 files changed, 6 deletions(-) diff --git a/type/__wikijs/gencode-remote b/type/__wikijs/gencode-remote index 81055e2..37c7df7 100755 --- a/type/__wikijs/gencode-remote +++ b/type/__wikijs/gencode-remote @@ -22,10 +22,5 @@ EOF # Install deps and launch cat << EOF cd /var/wiki || exit 1 -npm install -EOF - -# Restart service. -cat << EOF service wikijs restart EOF diff --git a/type/__wikijs/manifest b/type/__wikijs/manifest index 04a21af..b047223 100644 --- a/type/__wikijs/manifest +++ b/type/__wikijs/manifest @@ -52,7 +52,6 @@ export HTTP_PORT HTTPS_PORT db_pass="$(cat "${__object:?}/parameter/database-password")" __package nodejs -__package npm __directory /var/wiki/ # These things are Alpine-dependant. From a1620e8221738d30eb1f15230f5e8e093b493f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 Jan 2021 16:32:29 +0100 Subject: [PATCH 194/332] __jitsi_meet: remove forgotten/invalid requirement --- type/__jitsi_meet/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type/__jitsi_meet/manifest b/type/__jitsi_meet/manifest index ec65883..f5fbe8c 100755 --- a/type/__jitsi_meet/manifest +++ b/type/__jitsi_meet/manifest @@ -33,7 +33,7 @@ require="__package/gnupg2" __apt_key_uri jitsi_meet \ --uri https://download.jitsi.org/jitsi-key.gpg.key \ --state present ## Now the repositories (they are a tad weird, so distribution is 'stable/') -require="__ufw __apt_key_uri/jitsi_meet" __apt_source jitsi_meet \ +require="__apt_key_uri/jitsi_meet" __apt_source jitsi_meet \ --uri 'https://download.jitsi.org' \ --distribution 'stable/' \ --state present From 87d4c33e26b2b8fa67da0dcbc01a9f70f5e9c705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 13 Jan 2021 08:54:28 +0100 Subject: [PATCH 195/332] Make shellcheck happy with the __jitsi_* types --- type/__jitsi_meet/manifest | 6 +++--- type/__jitsi_meet_domain/manifest | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/type/__jitsi_meet/manifest b/type/__jitsi_meet/manifest index f5fbe8c..d4d16dc 100755 --- a/type/__jitsi_meet/manifest +++ b/type/__jitsi_meet/manifest @@ -43,6 +43,7 @@ require="__apt_source/jitsi_meet" __apt_update_index export require="${require} __apt_source/jitsi_meet __apt_update_index" # Pre-feed debconf settings, so Jitsi's installation has a good config +# shellcheck source=type/__jitsi_meet/files/debconf_settings.sh . "${__type}/files/debconf_settings.sh" # This defines DEBCONF_SETTINGS __debconf_set_selections jitsi_meet --file - < Date: Sun, 24 Jan 2021 09:25:02 +0100 Subject: [PATCH 196/332] __matrix_element: fix download tarball name All releases after 1.7.14 have a diffrent tarball name than releases before this. This finally takes over the new name. The version comparement function was taken from __sensible_editor, to provide backward compatibility to older versions (if someone needs it). Maybe this logic can be removed in a while .. --- type/__matrix_element/gencode-remote | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/type/__matrix_element/gencode-remote b/type/__matrix_element/gencode-remote index e643976..ff3bbaa 100755 --- a/type/__matrix_element/gencode-remote +++ b/type/__matrix_element/gencode-remote @@ -18,11 +18,37 @@ # along with cdist. If not, see . # +# Function to compare version strings. Returns success (0) if the version +# given by stdin is higher than the version provided by the argument. +# +# Taken from the cdist core type __sensible_editor. +version_ge() { + awk -F '[^0-9.]' -v target="${1:?}" ' + function max(x, y) { return x > y ? x : y; } + BEGIN { + getline; + nx = split($1, x, "."); + ny = split(target, y, "."); + for (i = 1; i <= max(nx, ny); ++i) { + diff = int(x[i]) - int(y[i]); + if (diff < 0) exit 1; + else if (diff > 0) exit 0; + else continue; + } + }' +} + + VERSION=$(cat "$__object/parameter/version") INSTALL_DIR=$(cat "$__object/parameter/install_dir") OWNER=$(cat "$__object/parameter/owner") -src="riot-v$VERSION" +# tarball name changed due to application renaming +if echo "$VERSION" | version_ge 1.7.14; then + src="element-v$VERSION" +else + src="riot-v$VERSION" +fi archive="$src.tar.gz" url="https://github.com/vector-im/riot-web/releases/download/v$VERSION/$archive" From 0d431d086c967ee5bdf67d0bd568c06c1a504111 Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Wed, 27 Jan 2021 16:06:28 +0100 Subject: [PATCH 197/332] Split initializing the password store from generating passwords. --- type/__pass/gencode-local | 20 +++---- type/__pass/man.rst | 16 ++---- type/__pass_init/gencode-local | 43 ++++++++++++++ type/__pass_init/man.rst | 56 +++++++++++++++++++ type/__pass_init/parameter/required | 1 + .../parameter/required_multiple | 0 type/__pass_init/singleton | 0 7 files changed, 115 insertions(+), 21 deletions(-) create mode 100755 type/__pass_init/gencode-local create mode 100644 type/__pass_init/man.rst create mode 100644 type/__pass_init/parameter/required rename type/{__pass => __pass_init}/parameter/required_multiple (100%) create mode 100644 type/__pass_init/singleton diff --git a/type/__pass/gencode-local b/type/__pass/gencode-local index e9e983c..e1277fa 100755 --- a/type/__pass/gencode-local +++ b/type/__pass/gencode-local @@ -46,25 +46,25 @@ then NOSYMB="-n" fi -# Load required GPG ID parameters. -set -- -while read -r id; -do - set -- "$@" "$id" -done < "${__object:?}/parameter/gpgid" - # Load required password store location parameter. PASSWORD_STORE_DIR="$(cat "${__object:?}/parameter/storedir")" export PASSWORD_STORE_DIR -# Run every time in case GPG IDs are updated. -pass init "$@" >/dev/null +# Check if the password store is initialized. +if ! pass ls >/dev/null 2>&1; +then + cat <<- EOF >&2 + __pass: this type requires the password store to be initialized. + See cdist-type__pass_init(7) and pass(1) for more information. + EOF + exit 1; +fi # Generate a password if it does not already exist. if [ ! -f "${PASSWORD_STORE_DIR}/${__object_id:?}.gpg" ]; then # shellcheck disable=SC2086 - pass generate $NOSYMB "${__object_id:?}" $LENGTH + pass generate $NOSYMB "${__object_id:?}" $LENGTH >/dev/null fi # Send it out to the messages. diff --git a/type/__pass/man.rst b/type/__pass/man.rst index 60bc6f8..ea9b93c 100644 --- a/type/__pass/man.rst +++ b/type/__pass/man.rst @@ -14,9 +14,6 @@ types depending on this one should require it. This enables an administrator to ensure a password exists using this type and then, from another type, use it as need be. -This type also sets the GPG IDs used to encrypt the password store: beware that -the IDs passed in the last ran invocation of the type will be the ones set for -the store. REQUIRED PARAMETERS ------------------- @@ -25,11 +22,6 @@ storedir created if it does not exist). -REQUIRED MULTIPLE PARAMETERS ----------------------------- -gpgid - The GPG IDs of the public keys used to encrypt the password store. - OPTIONAL PARAMETERS ------------------- length @@ -37,6 +29,7 @@ length it exists, this has no effect (and hence will not update the password, even if the length is different from the one specified). + BOOLEAN PARAMETERS ------------------ no-symbols @@ -52,18 +45,19 @@ looks up in the cdist messages to find it: .. code-block:: sh - __pass database/services/arandomservice + require=__pass_init \ + __pass database/services/arandomservice \ --storedir password/store/location - --gpgpid 92296965EAA1DD86A93284EF7B21E5AA32FB9810 require='__pass/database/services/arandomservice' \ __othertype --password database/service/arandomservice + -- SEE ALSO -------- -`pass`\ (7) +`pass`\ (7), `cdist-type__pass_init`\ (7) AUTHORS diff --git a/type/__pass_init/gencode-local b/type/__pass_init/gencode-local new file mode 100755 index 0000000..0be44d9 --- /dev/null +++ b/type/__pass_init/gencode-local @@ -0,0 +1,43 @@ +#!/bin/sh -e +# +# 2020 Joachim Desroches (joachim.desroches@epfl.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# Check pass is installed. +command -v pass >/dev/null 2>&1 || + { + cat <<- EOF >&2 + __pass_init: this type requires pass installed. + See https://www.passwordstore.org/. + EOF + exit 1; + } + +# Load required GPG ID parameters. +set -- +while read -r id; +do + set -- "$@" "$id" +done < "${__object:?}/parameter/gpgid" + +# Load required password store location parameter. +PASSWORD_STORE_DIR="$(cat "${__object:?}/parameter/storedir")" +export PASSWORD_STORE_DIR + +# Do our work. +pass init "$@" >/dev/null diff --git a/type/__pass_init/man.rst b/type/__pass_init/man.rst new file mode 100644 index 0000000..7a8d01e --- /dev/null +++ b/type/__pass_init/man.rst @@ -0,0 +1,56 @@ +cdist-type__pass_init(7) +======================== + +NAME +---- +cdist-type__pass_init - Initialize a local password store. + + +DESCRIPTION +----------- +This type is intented to be used as a prerequisite to the +cdist-type__pass(7) type. It will set up a pass(1) password +store with the provided GPP2(1) public encryption key IDs. + + +REQUIRED PARAMETERS +------------------- +storedir + The host-local directory where the password store is to be found (or + created if it does not exist). + + +REQUIRED MULTIPLE PARAMETERS +---------------------------- +gpgid + The GPG IDs of the public keys used to encrypt the password store. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Setup a repository with a GPG ID + __pass_init + --storedir password/store/location + --gpgpid 92296965EAA1DD86A93284EF7B21E5AA32FB9810 + +-- + +SEE ALSO +-------- +`pass`\ (7), `cdist-type__pass`\ (7) + + +AUTHORS +------- +Joachim Desroches + + +COPYING +------- +Copyright \(C) 2021 Joachim Desroches. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/type/__pass_init/parameter/required b/type/__pass_init/parameter/required new file mode 100644 index 0000000..f2fc3a2 --- /dev/null +++ b/type/__pass_init/parameter/required @@ -0,0 +1 @@ +storedir diff --git a/type/__pass/parameter/required_multiple b/type/__pass_init/parameter/required_multiple similarity index 100% rename from type/__pass/parameter/required_multiple rename to type/__pass_init/parameter/required_multiple diff --git a/type/__pass_init/singleton b/type/__pass_init/singleton new file mode 100644 index 0000000..e69de29 From 7b892e1332b3fcaf61025d4f80000ff7ce890f06 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sat, 6 Feb 2021 20:36:57 +0100 Subject: [PATCH 198/332] [__jitsi_meet] Upgrade prometheus jitsi exporter to 1.1.3 This also fixes a typo in the URL that resulted in the exporter not being installed '-.-. --- type/__jitsi_meet/manifest | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/type/__jitsi_meet/manifest b/type/__jitsi_meet/manifest index d4d16dc..d5cf098 100755 --- a/type/__jitsi_meet/manifest +++ b/type/__jitsi_meet/manifest @@ -137,9 +137,9 @@ server { EOF # These two should be changed on new release -PROMETHEUS_JITSI_EXPORTER_SHOULD_VERSION="1.1.2" -PROMETHEUS_JITSI_EXPORTER_CHECKSUM="sha256:c64852b1cf63efd1632ec07ac9fcead09baccc1bd8c7b21ca830a0d82d00fb3d" -PROMETHEUS_JITSI_EXPORTER_URL="https://github.com/systemli/prometheus-jitsi-meet-exporter/releases/download/${PROMETHEUS_JITSI_EXPORTER_VERSION}/prometheus-jitsi-meet-exporter-linux-amd64" +PROMETHEUS_JITSI_EXPORTER_SHOULD_VERSION="1.1.3" +PROMETHEUS_JITSI_EXPORTER_CHECKSUM="sha256:8ba14ee3317048ba69716ad8a903d363d90d7b552c8484e81acc892e05b56aa8" +PROMETHEUS_JITSI_EXPORTER_URL="https://github.com/systemli/prometheus-jitsi-meet-exporter/releases/download/${PROMETHEUS_JITSI_EXPORTER_SHOULD_VERSION}/prometheus-jitsi-meet-exporter-linux-amd64" PROMETHEUS_JITSI_EXPORTER_VERSION_FILE="/usr/local/bin/.prometheus-jitsi-meet-exporter.cdist.version" if [ ! -f "${__object}/parameter/disable-prometheus-exporter" ]; then case "${init}" in From 8a27e5ef22c6c58adc550884f57f5a47b9a6dd12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 25 Jun 2020 11:16:20 +0200 Subject: [PATCH 199/332] Import __matrix_synapse type from ungleich's dot-cdist --- type/__matrix_synapse/files/environment.sh | 6 + .../__matrix_synapse/files/homeserver.yaml.sh | 1785 +++++++++++++++++ type/__matrix_synapse/files/log.config.sh | 60 + type/__matrix_synapse/man.rst | 180 ++ type/__matrix_synapse/manifest | 287 +++ type/__matrix_synapse/parameter/boolean | 14 + .../parameter/default/database_host | 0 .../parameter/default/database_password | 0 .../parameter/default/database_user | 0 .../parameter/default/event_cache_size | 1 + .../parameter/default/global_cache_factor | 1 + .../parameter/default/ldap_base_dn | 0 .../parameter/default/ldap_bind_dn | 0 .../parameter/default/ldap_bind_password | 0 .../parameter/default/ldap_filter | 1 + .../parameter/default/ldap_mail_attribute | 1 + .../parameter/default/ldap_name_attribute | 1 + .../parameter/default/ldap_uid_attribute | 1 + .../parameter/default/ldap_uri | 0 .../parameter/default/max_upload_size | 1 + .../parameter/default/rc_login_burst | 1 + .../parameter/default/rc_login_per_second | 1 + .../parameter/default/rc_message_burst | 1 + .../parameter/default/rc_message_per_second | 1 + .../parameter/default/smtp_host | 1 + .../parameter/default/smtp_pass | 0 .../parameter/default/smtp_port | 1 + .../parameter/default/smtp_user | 0 .../parameter/default/turn_user_lifetime | 1 + type/__matrix_synapse/parameter/optional | 25 + .../parameter/optional_multiple | 5 + type/__matrix_synapse/parameter/required | 4 + type/__matrix_synapse/singleton | 0 33 files changed, 2380 insertions(+) create mode 100644 type/__matrix_synapse/files/environment.sh create mode 100755 type/__matrix_synapse/files/homeserver.yaml.sh create mode 100755 type/__matrix_synapse/files/log.config.sh create mode 100644 type/__matrix_synapse/man.rst create mode 100755 type/__matrix_synapse/manifest create mode 100644 type/__matrix_synapse/parameter/boolean create mode 100644 type/__matrix_synapse/parameter/default/database_host create mode 100644 type/__matrix_synapse/parameter/default/database_password create mode 100644 type/__matrix_synapse/parameter/default/database_user create mode 100644 type/__matrix_synapse/parameter/default/event_cache_size create mode 100644 type/__matrix_synapse/parameter/default/global_cache_factor create mode 100644 type/__matrix_synapse/parameter/default/ldap_base_dn create mode 100644 type/__matrix_synapse/parameter/default/ldap_bind_dn create mode 100644 type/__matrix_synapse/parameter/default/ldap_bind_password create mode 100644 type/__matrix_synapse/parameter/default/ldap_filter create mode 100644 type/__matrix_synapse/parameter/default/ldap_mail_attribute create mode 100644 type/__matrix_synapse/parameter/default/ldap_name_attribute create mode 100644 type/__matrix_synapse/parameter/default/ldap_uid_attribute create mode 100644 type/__matrix_synapse/parameter/default/ldap_uri create mode 100644 type/__matrix_synapse/parameter/default/max_upload_size create mode 100644 type/__matrix_synapse/parameter/default/rc_login_burst create mode 100644 type/__matrix_synapse/parameter/default/rc_login_per_second create mode 100644 type/__matrix_synapse/parameter/default/rc_message_burst create mode 100644 type/__matrix_synapse/parameter/default/rc_message_per_second create mode 100644 type/__matrix_synapse/parameter/default/smtp_host create mode 100644 type/__matrix_synapse/parameter/default/smtp_pass create mode 100644 type/__matrix_synapse/parameter/default/smtp_port create mode 100644 type/__matrix_synapse/parameter/default/smtp_user create mode 100644 type/__matrix_synapse/parameter/default/turn_user_lifetime create mode 100644 type/__matrix_synapse/parameter/optional create mode 100644 type/__matrix_synapse/parameter/optional_multiple create mode 100644 type/__matrix_synapse/parameter/required create mode 100644 type/__matrix_synapse/singleton diff --git a/type/__matrix_synapse/files/environment.sh b/type/__matrix_synapse/files/environment.sh new file mode 100644 index 0000000..99179be --- /dev/null +++ b/type/__matrix_synapse/files/environment.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +cat << EOF +# Specify environment variables used when running Synapse +SYNAPSE_CACHE_FACTOR=$CACHE_FACTOR +EOF diff --git a/type/__matrix_synapse/files/homeserver.yaml.sh b/type/__matrix_synapse/files/homeserver.yaml.sh new file mode 100755 index 0000000..77be438 --- /dev/null +++ b/type/__matrix_synapse/files/homeserver.yaml.sh @@ -0,0 +1,1785 @@ +#!/bin/sh + +# NOTE: this template has been generated using the +# matrix-synapse-1.5.1-1.fc31.noarch Fedora package for use with CDIST. + +generate_extra_settings () { + for line in $EXTRA_SETTINGS; do + echo "$line" + done +} + +generate_database () { + if [ "$DATABASE_ENGINE" = "sqlite3" ]; then + cat << EOF +database: + # The database engine name + name: "$DATABASE_ENGINE" + # Arguments to pass to the engine + args: + # Path to the database + database: "$DATABASE_NAME" +EOF + else +cat << EOF +database: + # The database engine name + name: "$DATABASE_ENGINE" + # Arguments to pass to the engine + args: + database: "$DATABASE_NAME" + host: "$DATABASE_HOST" + user: "$DATABASE_USER" + password: "$DATABASE_PASSWORD" +EOF + fi +} + +generate_password_providers () { + if [ "$ENABLE_LDAP_AUTH" = "true" ]; then + cat <