From 8eccacec59b0badd7a6e2a40a27f02f0ff355ccf Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 4 Feb 2021 19:09:26 +0100 Subject: [PATCH 01/10] __package_pip: add optional dependencies This is a poor implementation of optional dependencies for pip packages. It ensures to install them if the package will be installed, but does not take into account if they must be added/removed after the package is already installed. Also, it will not be autoremoved, as all dependencies will not be removed. --- cdist/conf/type/__package_pip/gencode-remote | 13 +++++++++++++ cdist/conf/type/__package_pip/man.rst | 13 +++++++++++++ .../type/__package_pip/parameter/optional_multiple | 1 + 3 files changed, 27 insertions(+) create mode 100644 cdist/conf/type/__package_pip/parameter/optional_multiple diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index a1375c2d..58c89040 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -56,6 +56,19 @@ fi case "$state_should" in present) + if [ -f "$__object/parameter/extra" ] + then + while read extra + do + if [ "$extras" ]; then + extras="$extras,$extra" + else + extras="$extra" + fi + done < "$__object/parameter/extra" + name="${name}[${extras}]" + fi + if [ "$runas" ] then echo "su -c '$pip install -q $name' $runas" diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index 234ceee2..64ad0358 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -22,6 +22,11 @@ OPTIONAL PARAMETERS name If supplied, use the name and not the object id as the package name. +extra + Extra optional dependencies which should be installed along the selected + package. Can be specified multiple times. Will only be applied if the + package actually will be installed, but will not explicitly checked. + pip Instead of using pip from PATH, use the specific pip path. @@ -46,6 +51,14 @@ EXAMPLES # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo + # Install package with optional dependencies + __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails + # or do a little cheating + __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails + + # or take all extras + __package_pip mautrix-telegram --extra all + SEE ALSO -------- diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple new file mode 100644 index 00000000..0f228715 --- /dev/null +++ b/cdist/conf/type/__package_pip/parameter/optional_multiple @@ -0,0 +1 @@ +extra From 73a03d75d7d23fea292b5fd60aeb059d8c72c4ba Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 4 Feb 2021 19:18:02 +0100 Subject: [PATCH 02/10] __package_pip: fix shellcheck --- cdist/conf/type/__package_pip/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 58c89040..925eaf24 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -58,7 +58,7 @@ case "$state_should" in present) if [ -f "$__object/parameter/extra" ] then - while read extra + while read -r extra do if [ "$extras" ]; then extras="$extras,$extra" From 4717e5ceff078111748824e4fb365df7979e5fe7 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 10:31:07 +0100 Subject: [PATCH 03/10] __package_pip: add extras explorer The two new explorers detect all installed extras for this package. --- .../type/__package_pip/explorer/distinfo-dir | 45 +++++++++++++++++++ cdist/conf/type/__package_pip/explorer/extras | 26 +++++++++++ cdist/conf/type/__package_pip/explorer/state | 0 3 files changed, 71 insertions(+) create mode 100755 cdist/conf/type/__package_pip/explorer/distinfo-dir create mode 100755 cdist/conf/type/__package_pip/explorer/extras mode change 100644 => 100755 cdist/conf/type/__package_pip/explorer/state diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir new file mode 100755 index 00000000..18e169ae --- /dev/null +++ b/cdist/conf/type/__package_pip/explorer/distinfo-dir @@ -0,0 +1,45 @@ +#!/bin/sh +# +# 2021 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +nameparam="$__object/parameter/name" +if [ -f "$nameparam" ]; then + name=$(cat "$nameparam") +else + name="$__object_id" +fi + +pipparam="$__object/parameter/pip" +if [ -f "$pipparam" ]; then + pip=$(cat "$pipparam") +else + pip="$( "$__type_explorer/pip" )" +fi + + +if command -v "$pip" >/dev/null 2>&1; then + # assemble the path where pip stores all pip package info + "$pip" show "$name" \ + | awk -F': ' ' + $1 == "Name" {name=$2; gsub(/-/,"_",name); next} + $1 == "Version" {version=$2; next} + $1 == "Location" {location=$2; next} + END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}' +fi diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras new file mode 100755 index 00000000..92fc4732 --- /dev/null +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -0,0 +1,26 @@ +#!/bin/sh +# +# 2021 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +distinfo_dir="$("$__type_explorer/distinfo-dir")" +if [ "$distinfo_dir" ]; then + # output all extras that are installed + awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA" +fi diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state old mode 100644 new mode 100755 From 8dc6ab97384f2240e9218fbd4463ba12b004f44c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 13:49:53 +0100 Subject: [PATCH 04/10] __package_pip: install not found extras Compares the explorer against the parameters and install those extras that are not already installed. --- cdist/conf/type/__package_pip/gencode-remote | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 925eaf24..ad327e37 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -2,6 +2,7 @@ # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # 2016 Darko Poljak (darko.poljak at gmail.com) +# 2021 Matthias Stecher (matthiasstecher at gmx.de) # # This file is part of cdist. # @@ -58,14 +59,17 @@ case "$state_should" in present) if [ -f "$__object/parameter/extra" ] then - while read -r extra - do - if [ "$extras" ]; then - extras="$extras,$extra" - else - extras="$extra" - fi - done < "$__object/parameter/extra" + mkdir "$__object/files" + # sort and generalize all extras + sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params" + sort "$__object/explorer/extras" > "$__object/files/extras-explorer" + + # assemble extras + # all extras are passed to pip in a comma-separated list + extras="$(comm -23 \ + "$__object/files/extras-params" "$__object/files/extras-explorer" \ + | sed ':a; $!N; s/\n/,/; ta' # loops through all input lines and add commas between them + )" name="${name}[${extras}]" fi From 2db0ef7c98226661e5e4b55c89180b31ba65b8bc Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 22:53:26 +0100 Subject: [PATCH 05/10] __package_pip: updating real detection of extras As the previous detection took the wrong values, this explorer now checks if packages for an extra are installed or not. If not, the extra is not installed. Based on the information of the explorer, it will install the package again with the absent extras. --- cdist/conf/type/__package_pip/explorer/extras | 37 +++++++++++++++++-- cdist/conf/type/__package_pip/gencode-remote | 21 ++++------- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 92fc4732..2f5fcca6 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -17,10 +17,41 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# +# Checks if the given extras are really installed or not. It will be +# done by querring all dependencies for that extra and return it as +# "to be installed" if no dependency was found. +# distinfo_dir="$("$__type_explorer/distinfo-dir")" -if [ "$distinfo_dir" ]; then - # output all extras that are installed - awk -F': ' '$1 == "Provides-Extra"{print $2}' "$distinfo_dir/METADATA" + +# check if we have something to check +if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ] +then + # save cause freezing is slow + mkdir "$__object/files" + pip_freeze="$__object/files/pip-freeze.tmp" + pip3 freeze > "$pip_freeze" + + for extra in $(cat "$__object/parameter/extra" | tr ',' '\n') + do + # create a grep BRE pattern to search all packages + grep_pattern="$( + awk -F'(: | ; )' -v check="$extra" ' + $1 == "Requires-Dist" { + split($2, r, " "); + sub("extra == ", "", $3); gsub("'"'"'", "", $3); + if($3 == check) print r[1] + }' "$distinfo_dir/METADATA" \ + | sed ':a; $!N; s/\n/\\|/; ta' + )" + + # echo the extra if no packages where found for it + # if there is no pattern, we don't need to search ;-) + if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze" + then + echo "$extra" + fi + done fi diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index ad327e37..9abe28bf 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -26,7 +26,10 @@ state_is=$(cat "$__object/explorer/state") state_should="$(cat "$__object/parameter/state")" -[ "$state_is" = "$state_should" ] && exit 0 +# short circuit if state is the same and no extras to install +[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \ + && exit 0 + nameparam="$__object/parameter/name" if [ -f "$nameparam" ]; then @@ -57,19 +60,11 @@ fi case "$state_should" in present) - if [ -f "$__object/parameter/extra" ] + if [ -s "$__object/explorer/extras" ] then - mkdir "$__object/files" - # sort and generalize all extras - sed 's/,/\n/g' "$__object/parameter/extra" | sort > "$__object/files/extras-params" - sort "$__object/explorer/extras" > "$__object/files/extras-explorer" - - # assemble extras - # all extras are passed to pip in a comma-separated list - extras="$(comm -23 \ - "$__object/files/extras-params" "$__object/files/extras-explorer" \ - | sed ':a; $!N; s/\n/,/; ta' # loops through all input lines and add commas between them - )" + # all extras are passed to pip in a comma-separated list in the name + # sed loops through all input lines and add commas between them + extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")" name="${name}[${extras}]" fi From 7398382890a4368e588e700183e6c777e90fb069 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Feb 2021 23:12:10 +0100 Subject: [PATCH 06/10] __package_pip: fix shellcheck Useless `cat $file`, use `< $file` instead. --- cdist/conf/type/__package_pip/explorer/extras | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 2f5fcca6..d57c69db 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -34,7 +34,7 @@ then pip_freeze="$__object/files/pip-freeze.tmp" pip3 freeze > "$pip_freeze" - for extra in $(cat "$__object/parameter/extra" | tr ',' '\n') + for extra in $(tr ',' '\n' < "$__object/parameter/extra") do # create a grep BRE pattern to search all packages grep_pattern="$( From a9d7dfb2ed40d0a61978cc706ff469c354a6c8f5 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 12 Feb 2021 09:17:02 +0100 Subject: [PATCH 07/10] __package_pip: split extra 'all' to a list of all extras This will fix if a package will be upgraded from some extras to all extras. Previously, it will not work because some dependencies of 'all' are already installed, so the feature 'all' is already installed. Now, it will use a list of all extras to iterate over them separatly. This will result it will never install all extras via `[all]`, but rather `[foo,bar]`. --- cdist/conf/type/__package_pip/explorer/extras | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index d57c69db..be945e36 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -34,7 +34,14 @@ then pip_freeze="$__object/files/pip-freeze.tmp" pip3 freeze > "$pip_freeze" - for extra in $(tr ',' '\n' < "$__object/parameter/extra") + # If all is set, it searches all available extras to separatly check them. + # It would work with just 'all' (cause dependencies are given), but will + # not update if one extra is already present. Side effect is that it will + # not use [all] but instead name all extras seperatly. + for extra in $(if grep -qFx all "$__object/parameter/extra"; + then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA"; + else tr ',' '\n' < "$__object/parameter/extra"; + fi) do # create a grep BRE pattern to search all packages grep_pattern="$( From 951712740f95f8f1277f1295bcd97c0b9bafdc94 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 12 Feb 2021 13:42:51 +0100 Subject: [PATCH 08/10] __package_pip: update man.rst Adjusted comments for `explorer/extras` and updated the man page for the new behaviour of updating the extras. --- cdist/conf/type/__package_pip/explorer/extras | 7 +++--- cdist/conf/type/__package_pip/man.rst | 25 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index be945e36..8104ed47 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -35,15 +35,16 @@ then pip3 freeze > "$pip_freeze" # If all is set, it searches all available extras to separatly check them. - # It would work with just 'all' (cause dependencies are given), but will - # not update if one extra is already present. Side effect is that it will - # not use [all] but instead name all extras seperatly. + # It would work with just 'all' (cause dependencies are specified for + # 'all'), but will not update if one extra is already present. Side effect + # is that it will not use [all] but instead name all extras seperatly. for extra in $(if grep -qFx all "$__object/parameter/extra"; then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA"; else tr ',' '\n' < "$__object/parameter/extra"; fi) do # create a grep BRE pattern to search all packages + # maybe a file full of patterns for -F could be written grep_pattern="$( awk -F'(: | ; )' -v check="$extra" ' $1 == "Requires-Dist" { diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index 64ad0358..c013695c 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -24,8 +24,14 @@ name extra Extra optional dependencies which should be installed along the selected - package. Can be specified multiple times. Will only be applied if the - package actually will be installed, but will not explicitly checked. + package. Can be specified multiple times. Multiple extra optional + dependencies can also be specified in a comma-separated list to provide + a more pip-natvie style. + + Extra optional dependencies will be installed even when the base package + is already installed. Notice that the type will not remove installed extras + that are not explicitly named for the type because pip does not offer a + management for orphaned packages and they may be used by other packages. pip Instead of using pip from PATH, use the specific pip path. @@ -53,8 +59,8 @@ EXAMPLES # Install package with optional dependencies __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails - # or do a little cheating - __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails + # the extras can also be specified comma-separated + __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres # or take all extras __package_pip mautrix-telegram --extra all @@ -67,12 +73,13 @@ SEE ALSO AUTHORS ------- -Nico Schottelius +| Nico Schottelius +| Matthias Stecher COPYING ------- -Copyright \(C) 2012 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can +redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either +version 3 of the License, or (at your option) any later version. From 2ce1fce7677b64a8d17a86f29f676b1ba8475db2 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 15 Feb 2021 16:17:46 +0100 Subject: [PATCH 09/10] __package_pip: match package names case insensitive Pip matches them insensitive, so we need to do the same to avoid problems by saying extras are not installed but already is there in place. --- cdist/conf/type/__package_pip/explorer/extras | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras index 8104ed47..bbdc17ab 100755 --- a/cdist/conf/type/__package_pip/explorer/extras +++ b/cdist/conf/type/__package_pip/explorer/extras @@ -57,7 +57,8 @@ then # echo the extra if no packages where found for it # if there is no pattern, we don't need to search ;-) - if [ "$grep_pattern" ] && ! grep -q "$grep_pattern" "$pip_freeze" + # pip matches packages case-insensetive, we need to do that, too + if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze" then echo "$extra" fi From d1f45d3524f8a025488d0d2f020433b6efb7edb6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 19 Feb 2021 09:03:56 +0100 Subject: [PATCH 10/10] __package_pip: corrected typo in man .. by fully replacing it with a smaller sentence. --- cdist/conf/type/__package_pip/man.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index c013695c..5a2bc673 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -24,9 +24,8 @@ name extra Extra optional dependencies which should be installed along the selected - package. Can be specified multiple times. Multiple extra optional - dependencies can also be specified in a comma-separated list to provide - a more pip-natvie style. + package. Can be specified multiple times. Multiple extras can be passed + in one `--extra` as a comma-separated list. Extra optional dependencies will be installed even when the base package is already installed. Notice that the type will not remove installed extras