From 8c7a6906def0fcf0c5693c1cb5c4b30d5d880d51 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Fri, 5 Mar 2021 20:38:19 +0100 Subject: [PATCH 01/45] __ini_value: starting base parts Initial body of the __ini_value. It contains two awk-scripts: to detect the state that is important to trigger a code generation, and to change that line to the correct line. It might have problems which is not matured enoght. The most difficult points will be the comment detection, as this is a critical point as you don't know if the user want to have this touched. --- cdist/conf/type/__ini_value/explorer/state | 152 ++++++++++++++++++ cdist/conf/type/__ini_value/files/base.awk | 128 +++++++++++++++ cdist/conf/type/__ini_value/files/common.awk | 61 +++++++ cdist/conf/type/__ini_value/files/gen-awk.sh | 53 ++++++ .../__ini_value/files/parts/absent/found.awk | 3 + .../files/parts/commented/found.awk | 8 + .../files/parts/commented/insert.awk | 9 ++ .../__ini_value/files/parts/present/found.awk | 8 + .../files/parts/present/insert.awk | 9 ++ cdist/conf/type/__ini_value/gencode-remote | 26 +++ cdist/conf/type/__ini_value/parameter/boolean | 1 + .../parameter/default/comment-sign | 2 + .../__ini_value/parameter/default/indentation | 1 + .../type/__ini_value/parameter/default/state | 1 + .../conf/type/__ini_value/parameter/optional | 6 + .../__ini_value/parameter/optional_multiple | 1 + .../conf/type/__ini_value/parameter/required | 2 + 17 files changed, 471 insertions(+) create mode 100755 cdist/conf/type/__ini_value/explorer/state create mode 100644 cdist/conf/type/__ini_value/files/base.awk create mode 100644 cdist/conf/type/__ini_value/files/common.awk create mode 100755 cdist/conf/type/__ini_value/files/gen-awk.sh create mode 100644 cdist/conf/type/__ini_value/files/parts/absent/found.awk create mode 100644 cdist/conf/type/__ini_value/files/parts/commented/found.awk create mode 100644 cdist/conf/type/__ini_value/files/parts/commented/insert.awk create mode 100644 cdist/conf/type/__ini_value/files/parts/present/found.awk create mode 100644 cdist/conf/type/__ini_value/files/parts/present/insert.awk create mode 100755 cdist/conf/type/__ini_value/gencode-remote create mode 100644 cdist/conf/type/__ini_value/parameter/boolean create mode 100644 cdist/conf/type/__ini_value/parameter/default/comment-sign create mode 100644 cdist/conf/type/__ini_value/parameter/default/indentation create mode 100644 cdist/conf/type/__ini_value/parameter/default/state create mode 100644 cdist/conf/type/__ini_value/parameter/optional create mode 100644 cdist/conf/type/__ini_value/parameter/optional_multiple create mode 100644 cdist/conf/type/__ini_value/parameter/required diff --git a/cdist/conf/type/__ini_value/explorer/state b/cdist/conf/type/__ini_value/explorer/state new file mode 100755 index 00000000..1808d90e --- /dev/null +++ b/cdist/conf/type/__ini_value/explorer/state @@ -0,0 +1,152 @@ +#!/bin/sh -e +# __ini_value/explorer/state + +# Check the state of the key-value pair in the ini file +# +# There are following states: +# - present +# - wrongvalue +# - wrongformat +# - commented +# - absent +# - nosuchfile + +# Using ' \t' for matching spaces as char classes not implemented in mawk +# see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=65617#40 + +# Parameters +# (maybe multi-variable object id for this ..) +#state_should="$(cat "$__object/parameter/state")" +file="$(cat "$__object/parameter/file")" + +# abort if no file exist +if ! [ -f "$file" ]; then + echo nosuchfile + exit +fi + +# run awk +awk -f - "$file" <<'AWK' +function trim(var) { + sub(/^[ \t]*/, "", var) + sub(/[ \t]*$/, "", var) + return var +} +function check_spaces(var) { + return match(var, /^[ \t]*$/) == 1 +} +function state(val) { + print val + exit +} + +BEGIN { + _param = (ENVIRON["__object"] "/parameter/") + getline state_should < (_param "state") + getline section < (_param "section") + getline key < (_param "key") + getline delimiter < (_param "delimiter") + getline value < (_param "value") + getline indentation < (_param "indentation") + + do_normalization = (system("test -f " (_param "normalize")) == 0) + + i=0; _comm_param = (_param "comment-sign"); + while((getline tmp < _comm_param) > 0) { + comment_signs[i++] = tmp + } + + found=0 + curr_section="" + if(section == "") + found_section=1 + else + found_section=0 +} + +# catch sections +/^[ \t]*\[.*\][ \t]*$/ { + curr_section = trim($0) + if(found_section) + exit # game over, section ends + if(section == curr_section) + found_section=1 + next +} + +# only interesting if a delimiter was found +found_section { + line = $0 + + # index 1 cause of trimmed string + if((idel = index(line, delimiter)) && (ikey = index(line, key))) { + if(ikey > 1) { + # maybe comment character or only spaces + start_string = substr(line, 1, ikey - 1) + + # something inside rather than a space -> comment + if((icom = match(start_string, /[^ \t]+/)) > 0) { + # icom = RSTART + # only one free-standing char or directly before the key + if(RLENGTH == 1 || icom == ikey - 1) { + start_sign = substr(line, RSTART, 1) + is_com=0 + for(i in comment_signs) { + if(start_sign == comment_signs[i]) { + is_com = 1; break; + } + } + if(!is_com) next + else { + aftercom_length = ikey - icom - 1 + if(!check_spaces(substr(line, icom + 1, aftercom_length))) next + start_spaces = (icom - 1) + aftercom_length + } + } + else next + } + # must only contain spaces + else start_spaces = ikey - 1 + } + # start_spaces set or 0 + + idelspace_start = ikey + length(key) + idelspace_length = idel - (idelspace_start) + # check for delimiter is only preceded with spaces + if(idelspace_length == 0 || check_spaces(substr(line, idelspace_start, idelspace_length))) { + found = 1 + + # short-circuit on state absent to just delete + if(state_should == "absent") state("present"); + + + # extract the value + found_value = substr(line, idel + length(delimiter)) + is_value = trim(found_value) + + # check if value is incorrect + if(value != is_value) state("wrongvalue") + else { + # check if the format is important + if(do_normalization) { + # the format must exactly match, else it is incorrect + if(indentation != start_spaces || idelspace_length != 0 || found_value != is_value) + state("wrongformat") + } + + if(is_com) + state("commented") + else + state("present") + } + # this will never be reached + } + } +} + +# in the end, check if it is absent +END { + if(!found) + state("absent") +} +AWK diff --git a/cdist/conf/type/__ini_value/files/base.awk b/cdist/conf/type/__ini_value/files/base.awk new file mode 100644 index 00000000..16ff5808 --- /dev/null +++ b/cdist/conf/type/__ini_value/files/base.awk @@ -0,0 +1,128 @@ +BEGIN { + #bufindex = 0 + #maxbuflen = 2 + + # no section means the start to the first section + if(section == "") { + is_curr_section = 1 + found_section = 1 + } +} + +# legacy function +function flush_buffer() { + #while(bufindex > 0) { + # _pop_buffer() + #} + if(lastlinepopulated) { + print lastline + lastlinepopulated = 0 + } +} + +# excepts the first character is the sign to check (string is trimmed) +function is_comment(line) { + # get character and check + line_sign = substr(line, 1, 1) + for(i in comment_signs) + if(line_sign == comment_signs[i]) + return 1 + + # nothing found + return 0 +} +function was_comment(line, comment) { + line = trim(line) + if(is_comment(line)) { + return trim(substr(line, 2)) == comment + } +} + +# print everything if line found instead of processing it +# maybe just a function to loop through getline for lightest overhead +found {print; next} + +# main loop (til the line was found) +!found { + line = trim($0) + # short curcit on empty lines + if(line == "") { + # TODO code to function! + flush_buffer() + lastline = $0 + lastlinepopulated = 1 + next + } + + # write to linebuffer + #if(bufindex > maxbuflen) + # _pop_buffer() + #linebuffer[++bufindex] = $0 + + # check for a ini section + if(substr(line, 1, 1) == "[" && substr(line, length(line), 1) == "]") { + is_section = 1 + curr_section = line + + if(curr_section == section) { + found_section = 1 + is_curr_section = 1 + } + else { + # if nothing found, print it in the valid section before the next one + if(is_curr_section) { + if(!found) { + # set found as it is there now + found=1 + + # %codeblock_insert% + } + is_curr_section = 0 + } + } + } + else { + # only current session is interessting + if(is_curr_section) { + # check for a comment + is_com = is_comment(line) + if(is_com) { + line = trim(substr(line, 2)) + } + + # check for a delimiter and a key (must be at first position due to trimming) + if((idel = index(line, delimiter)) && (ikey = index(line, key)) == 1) { + # check there are only spaces between the delimiter + if(check_spaces(substr(line, ikey + length(key), idel - (length(key) + 1)))) { + found = 1 + + # %codeblock_found% + + next + } + } + } + } + + # TODO + # works cause no next statement from above *structual programming* + #print lastline + flush_buffer() + lastline = $0 + lastlinepopulated = 1 +} + +END { + # if not found, it's not already printed + if(!found) { + flush_buffer() + + # print with section if not found + if(!found_section) { + # TODO check via buffer if a empty line is necessary + print section + + # %codeblock_insert% + } + } +} diff --git a/cdist/conf/type/__ini_value/files/common.awk b/cdist/conf/type/__ini_value/files/common.awk new file mode 100644 index 00000000..8874f0b8 --- /dev/null +++ b/cdist/conf/type/__ini_value/files/common.awk @@ -0,0 +1,61 @@ +BEGIN { + # parameter variables + section = get_param_string("section") + key = get_param_string("key") + delimiter = get_param_string("delimiter") + value = get_param_string("value") + comment = get_param_string("comment") + indentation = get_param_string("indentation") + + get_param_array("comment-sign", comment_signs) + comment_sign = comment_signs[0] +} + +function trim(var) { + sub(/^[ \t]*/, "", var) + sub(/[ \t]*$/, "", var) + return var +} +function check_spaces(part) { + return match(part, /^[ \t]*$/) == 1 +} + +function get_param_string(name) { + _paramfile = (ENVIRON["__object"] "/parameter/" name) + if((getline tmp < _paramfile) > 0) { + close(_paramfile) + return tmp + } + else return "" +} +function get_param_array(name, arr) { + _paramfile = (ENVIRON["__object"] "/parameter/" name) + i=0 + split("", arr) # portable clear, like `delete arr` + while((getline tmp < _paramfile) > 0) { + arr[i++] = tmp + } + close(_paramfile) +} + +# print value +function v_print() { + spaces = "" + for(i = 0; i < indentation; i++) + spaces = (spaces " ") + printf "%s%s%s%s%s", spaces, key, delimiter, value, ORS +} +# print commented value +function v_print_commented() { + spaces = "" + for(i = 0; i < indentation; i++) + spaces = (spaces " ") + printf "%s%s%s%s%s%s", spaces, comment_sign, key, delimiter, value, ORS +} +# print comment +function c_print() { + spaces = "" + for(i = 0; i < indentation; i++) + spaces = (spaces " ") + printf "%s%s%s%s%s", spaces, comment_sign, " ", comment, ORS +} diff --git a/cdist/conf/type/__ini_value/files/gen-awk.sh b/cdist/conf/type/__ini_value/files/gen-awk.sh new file mode 100755 index 00000000..ba26672a --- /dev/null +++ b/cdist/conf/type/__ini_value/files/gen-awk.sh @@ -0,0 +1,53 @@ +#!/bin/sh -e +# __ini_value/files/gen-awk.sh + +# Generates the awk script that will modify the line +# +# Arguments: +# 1: the file to modify +# 2: the should_state + +strip_comments() { + grep -v '^[[:space:]]*\($\|#\)' +} + + +file="$1" +state="$2" + + +# Generate the basic awk struct +cat < "\$tmpfile" <<'AWK' +SHELL + + +# generate the awk script and strip unnecessary things +{ + # basic functions which everyone needs + cat "$__type/files/common.awk" + + # generate the script and strip + awk -v state="$state" ' + { + if(match($0, /^[ \t]*# %codeblock_([^%]+)%$/, result) > 0) { + file = (ENVIRON["__type"] "/files/parts/" state "/" result[1] ".awk") + while((getline line < file) > 0) + print line + close(file) + } + else print + }' "$__type/files/base.awk" +} | strip_comments + + +# end of here-doc +cat <&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__ini_value/parameter/boolean b/cdist/conf/type/__ini_value/parameter/boolean new file mode 100644 index 00000000..f9a8b6aa --- /dev/null +++ b/cdist/conf/type/__ini_value/parameter/boolean @@ -0,0 +1 @@ +normalize diff --git a/cdist/conf/type/__ini_value/parameter/default/comment-sign b/cdist/conf/type/__ini_value/parameter/default/comment-sign new file mode 100644 index 00000000..406ac1cc --- /dev/null +++ b/cdist/conf/type/__ini_value/parameter/default/comment-sign @@ -0,0 +1,2 @@ +; +# diff --git a/cdist/conf/type/__ini_value/parameter/default/indentation b/cdist/conf/type/__ini_value/parameter/default/indentation new file mode 100644 index 00000000..573541ac --- /dev/null +++ b/cdist/conf/type/__ini_value/parameter/default/indentation @@ -0,0 +1 @@ +0 diff --git a/cdist/conf/type/__ini_value/parameter/default/state b/cdist/conf/type/__ini_value/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__ini_value/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__ini_value/parameter/optional b/cdist/conf/type/__ini_value/parameter/optional new file mode 100644 index 00000000..031c807b --- /dev/null +++ b/cdist/conf/type/__ini_value/parameter/optional @@ -0,0 +1,6 @@ +section +key +state +value +indentation +comment diff --git a/cdist/conf/type/__ini_value/parameter/optional_multiple b/cdist/conf/type/__ini_value/parameter/optional_multiple new file mode 100644 index 00000000..bac05165 --- /dev/null +++ b/cdist/conf/type/__ini_value/parameter/optional_multiple @@ -0,0 +1 @@ +comment-sign diff --git a/cdist/conf/type/__ini_value/parameter/required b/cdist/conf/type/__ini_value/parameter/required new file mode 100644 index 00000000..3ae10da3 --- /dev/null +++ b/cdist/conf/type/__ini_value/parameter/required @@ -0,0 +1,2 @@ +file +delimiter From 10427fde84d8563762020565230cde63ff031752 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 6 Mar 2021 17:06:32 +0100 Subject: [PATCH 02/45] __ini_value: implement multi-line buffer This buffer gives the ability to look a bit longer into the past than just a single line. This is helpfull to get a bigger context when fiddling around comments. This also erases the need of `lastlinepopulated` as there is something in the pipe or nothing. --- cdist/conf/type/__ini_value/files/base.awk | 53 ++++++++++--------- .../__ini_value/files/parts/absent/found.awk | 8 +-- .../files/parts/commented/found.awk | 2 +- .../files/parts/commented/insert.awk | 2 +- .../__ini_value/files/parts/present/found.awk | 2 +- .../files/parts/present/insert.awk | 2 +- 6 files changed, 38 insertions(+), 31 deletions(-) diff --git a/cdist/conf/type/__ini_value/files/base.awk b/cdist/conf/type/__ini_value/files/base.awk index 16ff5808..51c3ba99 100644 --- a/cdist/conf/type/__ini_value/files/base.awk +++ b/cdist/conf/type/__ini_value/files/base.awk @@ -1,6 +1,7 @@ BEGIN { - #bufindex = 0 - #maxbuflen = 2 + bufindex = -1 + buflen = 0 + maxbuflen = 2 # no section means the start to the first section if(section == "") { @@ -9,15 +10,31 @@ BEGIN { } } -# legacy function +# controls the line buffer function flush_buffer() { - #while(bufindex > 0) { - # _pop_buffer() - #} - if(lastlinepopulated) { - print lastline - lastlinepopulated = 0 - } + while(buflen > 0) + _pop_line() +} +function push_line() { + linebuf[++bufindex] = $0 + buflen++; + while(buflen > maxbuflen) _pop_line() +} +function revert_line() { + # no delete, because it will be overwritten by the next line if any .. + bufindex-- + buflen-- +} +function lastline() { + if(buflen > 0) return linebuf[bufindex] +} +function pop_line() { + if(buflen > 0) _pop_line() +} +function _pop_line() { + _index = bufindex - (--buflen) + print linebuf[_index] + delete linebuf[_index] } # excepts the first character is the sign to check (string is trimmed) @@ -47,18 +64,10 @@ found {print; next} line = trim($0) # short curcit on empty lines if(line == "") { - # TODO code to function! - flush_buffer() - lastline = $0 - lastlinepopulated = 1 + push_line() next } - # write to linebuffer - #if(bufindex > maxbuflen) - # _pop_buffer() - #linebuffer[++bufindex] = $0 - # check for a ini section if(substr(line, 1, 1) == "[" && substr(line, length(line), 1) == "]") { is_section = 1 @@ -104,12 +113,8 @@ found {print; next} } } - # TODO # works cause no next statement from above *structual programming* - #print lastline - flush_buffer() - lastline = $0 - lastlinepopulated = 1 + push_line() } END { diff --git a/cdist/conf/type/__ini_value/files/parts/absent/found.awk b/cdist/conf/type/__ini_value/files/parts/absent/found.awk index 10e7675b..0f907b54 100644 --- a/cdist/conf/type/__ini_value/files/parts/absent/found.awk +++ b/cdist/conf/type/__ini_value/files/parts/absent/found.awk @@ -1,3 +1,5 @@ -# flush last line if it was not the meant comment -if(!was_comment(lastline, comment)) flush_buffer() -else lastlinepopulated = 0 +# revert line if it was a comment +if(was_comment(lastline, comment)) revert_line() + +# value line was not pushed to the buffer yet +flush_buffer() diff --git a/cdist/conf/type/__ini_value/files/parts/commented/found.awk b/cdist/conf/type/__ini_value/files/parts/commented/found.awk index 9643f9ff..934e3cc7 100644 --- a/cdist/conf/type/__ini_value/files/parts/commented/found.awk +++ b/cdist/conf/type/__ini_value/files/parts/commented/found.awk @@ -1,5 +1,5 @@ # check if last line was the comment -was_com_there = was_comment(lastline, comment) +was_com_there = was_comment(lastline(), comment) # print + comment if not there flush_buffer() diff --git a/cdist/conf/type/__ini_value/files/parts/commented/insert.awk b/cdist/conf/type/__ini_value/files/parts/commented/insert.awk index 8003942d..3064ab81 100644 --- a/cdist/conf/type/__ini_value/files/parts/commented/insert.awk +++ b/cdist/conf/type/__ini_value/files/parts/commented/insert.awk @@ -1,4 +1,4 @@ -was_com_there = was_comment(lastline, comment) +was_com_there = was_comment(lastline(), comment) # print before and comment flush_buffer() diff --git a/cdist/conf/type/__ini_value/files/parts/present/found.awk b/cdist/conf/type/__ini_value/files/parts/present/found.awk index 9673dc23..7ea4248f 100644 --- a/cdist/conf/type/__ini_value/files/parts/present/found.awk +++ b/cdist/conf/type/__ini_value/files/parts/present/found.awk @@ -1,5 +1,5 @@ # check if last line was the comment -was_com_there = was_comment(lastline, comment) +was_com_there = was_comment(lastline(), comment) # print + comment if not there flush_buffer() diff --git a/cdist/conf/type/__ini_value/files/parts/present/insert.awk b/cdist/conf/type/__ini_value/files/parts/present/insert.awk index 48dbaea8..c48186d8 100644 --- a/cdist/conf/type/__ini_value/files/parts/present/insert.awk +++ b/cdist/conf/type/__ini_value/files/parts/present/insert.awk @@ -1,4 +1,4 @@ -was_com_there = was_comment(lastline, comment) +was_com_there = was_comment(lastline(), comment) # print before and comment flush_buffer() From 72c6306ba2751f6bd94482a9a0220a85c23b0411 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 6 Mar 2021 17:46:40 +0100 Subject: [PATCH 03/45] __ini_value: add man.rst --- cdist/conf/type/__ini_value/man.rst | 127 ++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 cdist/conf/type/__ini_value/man.rst diff --git a/cdist/conf/type/__ini_value/man.rst b/cdist/conf/type/__ini_value/man.rst new file mode 100644 index 00000000..c583d5ad --- /dev/null +++ b/cdist/conf/type/__ini_value/man.rst @@ -0,0 +1,127 @@ +cdist-type__ini_value(7) +======================== + +NAME +---- +cdist-type__ini_value - Handles ini- and conf-style configuration options + + +DESCRIPTION +----------- +This cdist type allow changes to more advanced key-value based configurations. +Most commonly this would be ini- or conf-style configurations. + +The type can have following states: + + present + The line exists with the correct value. + + commented + The key-value is outcommented. + + absent + The key-value line does not exist in the given section. + + +REQUIRED PARAMETERS +------------------- +file + The file to modify. + +delimiter + The delimiter which seperates each key-value pair. + + +OPTIONAL PARAMETERS +------------------- +state + One of the states defined in the above section. Defaults to `present`. + +section + The section where the value is located at. It always need to be surrounded + by square brackets as common for ini files. If not, the section will not be + found. If no section is specified, the block before any section is meant. + +key + The key to identify the key-value pair. Must be set if the state is not + absent. + +value + The value assigned to the key. Must be set if the state is not absent. + Else, an empty value is assigned to the given key. + +comment + The comment which should be placed above the configuration line. + +indentation + The indentation the key-value pair should have. Will be applied on inserts, + but also be enforced if ``--normalize`` is set. + +comment-sign + This declares the comment signs that are valid to use in the configuration + file. Each parameter must declare a single character only; multiple + parameters are possible. It uses the first specified sign as comment + character if this type needs to insert comments. + + +BOOLEAN PARAMETERS +------------------ +normalize + This parameter enforces that the parameter is always pretty in the + configuration file. Even if a key-value pair is correct as-is, it will + correct the line to be pretty and perfect. + + +MESSAGES +-------- +The type currently fails to give a correct information of what he did cause of +the following construct. It has two `awk` scripts which do the job: + +1. The explorer script which will outputs a single state of the given + key-value. Because the current state can contain much more states than the + state that should be, one state is returned like `wrongvalue` even if + `commented` is correct, too. Therefor, it vanishes the information that the + line is commented, too, even this could be a nice information that the + messaging system could emit. + +2. The `code-remote` script also goes through the whole file and print out the + same file except the line line that should be changed. This is done because + it can not be garanteed that an other type already modifed the file, which + may moved the key-value to an other position. Then, the script replaces the + line which a pretty-printed key-value pair. + +So the detected state is not important for the remote script, as it only needs +to know that it must be run cause of differences and what the state should be. +So if there are a state like `wrongvalue`, it triggers to correction of the +line, but it do not care if it was `wrongvalue`, `wrongformat` or `commented` +which trigged the run. Because of this need, the explorer retuns only an +easy-to-use value to detect if something needs to be changed. + +Therefor, it is unable to correctly emit messages with the current base. + + +EXAMPLES +-------- + +.. code-block:: sh + + # set a value in a configuration + __ini_value fancy-id --file /etc/foo/bar.ini --section '[welcome]' \ + --key hi --value baz --delimiter ' = ' + + # outcomment a value + __ini_value foo --file /etc/bar/foo.conf --state commented \ + --key noop --value true --delimiter ' = ' --comment 'not this time!' + + +AUTHORS +------- +Matthias Stecher + + +COPYING +------- +Copyright \(C) 2021 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. From 3e67cdbf9b39f291d7a49c9ca18695ff0e7081d7 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 6 Mar 2021 22:09:59 +0100 Subject: [PATCH 04/45] __ini_value: add space detection on insert This detection tires to somehow inteligent places the new entry at the end of the section, but tires to not infer with existing comments for the section etc. This detection is not perfect but will work in some cases. Hope it helps (if someone whats to look at these files after they got edited). This currently only applies to the `present` state, but not to the `commented` state. This will be changed somehow when both scripts will be unifed cause of there great similarities. --- cdist/conf/type/__ini_value/files/base.awk | 18 ++++-- .../files/parts/present/insert.awk | 59 +++++++++++++++++-- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__ini_value/files/base.awk b/cdist/conf/type/__ini_value/files/base.awk index 51c3ba99..57535964 100644 --- a/cdist/conf/type/__ini_value/files/base.awk +++ b/cdist/conf/type/__ini_value/files/base.awk @@ -1,7 +1,7 @@ BEGIN { bufindex = -1 buflen = 0 - maxbuflen = 2 + maxbuflen = 10 # no section means the start to the first section if(section == "") { @@ -15,9 +15,13 @@ function flush_buffer() { while(buflen > 0) _pop_line() } +function flush_lines(n) { + while(buflen > 0 && n-- > 0) + _pop_line() +} function push_line() { linebuf[++bufindex] = $0 - buflen++; + buflen++ while(buflen > maxbuflen) _pop_line() } function revert_line() { @@ -41,8 +45,8 @@ function _pop_line() { function is_comment(line) { # get character and check line_sign = substr(line, 1, 1) - for(i in comment_signs) - if(line_sign == comment_signs[i]) + for(c in comment_signs) + if(line_sign == comment_signs[c]) return 1 # nothing found @@ -85,6 +89,10 @@ found {print; next} found=1 # %codeblock_insert% + + # print line as it would else only be populated below + print + next } is_curr_section = 0 } @@ -101,7 +109,7 @@ found {print; next} # check for a delimiter and a key (must be at first position due to trimming) if((idel = index(line, delimiter)) && (ikey = index(line, key)) == 1) { - # check there are only spaces between the delimiter + # check there are only spaces between the key and delimiter if(check_spaces(substr(line, ikey + length(key), idel - (length(key) + 1)))) { found = 1 diff --git a/cdist/conf/type/__ini_value/files/parts/present/insert.awk b/cdist/conf/type/__ini_value/files/parts/present/insert.awk index c48186d8..eeaa7eb8 100644 --- a/cdist/conf/type/__ini_value/files/parts/present/insert.awk +++ b/cdist/conf/type/__ini_value/files/parts/present/insert.awk @@ -1,9 +1,56 @@ -was_com_there = was_comment(lastline(), comment) +# check if there is a comment block before the section +firstline_index = bufindex - (buflen - 1) +insertpoint = -1 # the insertpoint marks the point before the insert +lastfreespace = -1 +for(i = bufindex; i >= firstline_index; i--) { + _line = trim(linebuf[i]) + if(_line == "") { + lastfreespace = i + continue + } + if(comment && was_comment(_line, comment)) { + insertpoint = i + 1 + no_insert_comment = 1 + if(lastfreespace != insertpoint) + insert_line_after = 1 + break + } + if(!is_comment(_line) || index(_line, delimiter) > 0) { + insertpoint = i + 1 + + # only insert a line before if we do not have a space around + if(lastfreespace == insertpoint) + insertpoint++ + else + insert_line_before = 1 + # check for empty line after the insert point + # use absolute boundary cause the insertpoint can be changed + if(trim(linebuf[i + 2]) != "") + insert_line_after = 1 + break + } +} + +# insert into the last free space +if(insertpoint == -1) { + if(lastfreespace != -1) { + insertpoint = lastfreespace + insert_line_before = 1 + } + else { + insertpoint = firstline_index + insert_line_after = 1 + } +} + +# print lines before +flush_lines(insertpoint - firstline_index) + # print before and comment -flush_buffer() -if(comment && !was_com_there) c_print() - -# print value + seperator line at the end +if(insert_line_before) print "" +if(comment && !no_insert_comment) c_print() v_print() -print "" +if(insert_line_after) print "" + +flush_buffer() From ea7ac72f9add4879ee34a507c479a264d8593ab4 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 6 Mar 2021 23:12:25 +0100 Subject: [PATCH 05/45] __ini_value: unify `present` + `commented` state awk scripts Because the difference between the both states is just the comment sign before it, it was now merged into one folder + symlink to avoid redundancy. To acomplish the difference, a further parser step was introduced to execute different print function based on the state (it is a bit of hard-coded but ok). --- cdist/conf/type/__ini_value/files/gen-awk.sh | 18 +++++++++++++++--- .../type/__ini_value/files/parts/commented | 1 + .../files/parts/commented/found.awk | 8 -------- .../files/parts/commented/insert.awk | 9 --------- .../__ini_value/files/parts/present/found.awk | 2 +- .../__ini_value/files/parts/present/insert.awk | 2 +- 6 files changed, 18 insertions(+), 22 deletions(-) create mode 120000 cdist/conf/type/__ini_value/files/parts/commented delete mode 100644 cdist/conf/type/__ini_value/files/parts/commented/found.awk delete mode 100644 cdist/conf/type/__ini_value/files/parts/commented/insert.awk diff --git a/cdist/conf/type/__ini_value/files/gen-awk.sh b/cdist/conf/type/__ini_value/files/gen-awk.sh index ba26672a..a73a3f7b 100755 --- a/cdist/conf/type/__ini_value/files/gen-awk.sh +++ b/cdist/conf/type/__ini_value/files/gen-awk.sh @@ -34,11 +34,23 @@ SHELL # generate the script and strip awk -v state="$state" ' + function parse(line) { + if(match(line, /^[ \t]*# %code_print%$/) > 0) { + if(state == "present") + print "v_print()" + else if(state == "commented") + print "v_print_commented()" + else + print "print \"script compile error! cdist state " state " unkown!\" > /dev/stderr" + } + else print line + } { - if(match($0, /^[ \t]*# %codeblock_([^%]+)%$/, result) > 0) { - file = (ENVIRON["__type"] "/files/parts/" state "/" result[1] ".awk") + if(match($0, /^[ \t]*# %codeblock_([^%]+)%$/) > 0) { + split($2, result, "_"); type = substr(result[2], 1, length(result[2]) - 1) + file = (ENVIRON["__type"] "/files/parts/" state "/" type ".awk") while((getline line < file) > 0) - print line + parse(line) close(file) } else print diff --git a/cdist/conf/type/__ini_value/files/parts/commented b/cdist/conf/type/__ini_value/files/parts/commented new file mode 120000 index 00000000..568612b9 --- /dev/null +++ b/cdist/conf/type/__ini_value/files/parts/commented @@ -0,0 +1 @@ +present \ No newline at end of file diff --git a/cdist/conf/type/__ini_value/files/parts/commented/found.awk b/cdist/conf/type/__ini_value/files/parts/commented/found.awk deleted file mode 100644 index 934e3cc7..00000000 --- a/cdist/conf/type/__ini_value/files/parts/commented/found.awk +++ /dev/null @@ -1,8 +0,0 @@ -# check if last line was the comment -was_com_there = was_comment(lastline(), comment) - -# print + comment if not there -flush_buffer() -if(comment && !was_com_there) c_print() - -v_print_commented() diff --git a/cdist/conf/type/__ini_value/files/parts/commented/insert.awk b/cdist/conf/type/__ini_value/files/parts/commented/insert.awk deleted file mode 100644 index 3064ab81..00000000 --- a/cdist/conf/type/__ini_value/files/parts/commented/insert.awk +++ /dev/null @@ -1,9 +0,0 @@ -was_com_there = was_comment(lastline(), comment) - -# print before and comment -flush_buffer() -if(comment && !was_com_there) c_print() - -# print value + seperator line at the end -v_print_commented() -print "" diff --git a/cdist/conf/type/__ini_value/files/parts/present/found.awk b/cdist/conf/type/__ini_value/files/parts/present/found.awk index 7ea4248f..4e456bbb 100644 --- a/cdist/conf/type/__ini_value/files/parts/present/found.awk +++ b/cdist/conf/type/__ini_value/files/parts/present/found.awk @@ -5,4 +5,4 @@ was_com_there = was_comment(lastline(), comment) flush_buffer() if(comment && !was_com_there) c_print() -v_print() +# %code_print% diff --git a/cdist/conf/type/__ini_value/files/parts/present/insert.awk b/cdist/conf/type/__ini_value/files/parts/present/insert.awk index eeaa7eb8..9cb70327 100644 --- a/cdist/conf/type/__ini_value/files/parts/present/insert.awk +++ b/cdist/conf/type/__ini_value/files/parts/present/insert.awk @@ -50,7 +50,7 @@ flush_lines(insertpoint - firstline_index) # print before and comment if(insert_line_before) print "" if(comment && !no_insert_comment) c_print() -v_print() +# %code_print% if(insert_line_after) print "" flush_buffer() From 9006994023ec45c224ef8fff223abe31f87b41f1 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 9 Mar 2021 18:42:10 +0100 Subject: [PATCH 06/45] __ini_value: fix explorer detect value as commented Because the value was not reseted on a fase positive. --- cdist/conf/type/__ini_value/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__ini_value/explorer/state b/cdist/conf/type/__ini_value/explorer/state index 1808d90e..5f98f869 100755 --- a/cdist/conf/type/__ini_value/explorer/state +++ b/cdist/conf/type/__ini_value/explorer/state @@ -80,6 +80,7 @@ found_section { # index 1 cause of trimmed string if((idel = index(line, delimiter)) && (ikey = index(line, key))) { + is_com=0 if(ikey > 1) { # maybe comment character or only spaces start_string = substr(line, 1, ikey - 1) @@ -90,7 +91,6 @@ found_section { # only one free-standing char or directly before the key if(RLENGTH == 1 || icom == ikey - 1) { start_sign = substr(line, RSTART, 1) - is_com=0 for(i in comment_signs) { if(start_sign == comment_signs[i]) { is_com = 1; break; From b76d8485404b6f023cd6acf7022d95459e3c3704 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 11 Mar 2021 21:50:55 +0100 Subject: [PATCH 07/45] __ini_value: add delimiter space detection This adds spaces around the delimiter to provide some flexibility than just do it in the delimiter string directly. It can be set via the parameter `--delimiter-space`. --- cdist/conf/type/__ini_value/explorer/state | 14 +++++++--- cdist/conf/type/__ini_value/files/common.awk | 26 ++++++++++--------- cdist/conf/type/__ini_value/man.rst | 8 ++++++ .../conf/type/__ini_value/parameter/optional | 1 + 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/cdist/conf/type/__ini_value/explorer/state b/cdist/conf/type/__ini_value/explorer/state index 5f98f869..e387d104 100755 --- a/cdist/conf/type/__ini_value/explorer/state +++ b/cdist/conf/type/__ini_value/explorer/state @@ -48,6 +48,7 @@ BEGIN { getline delimiter < (_param "delimiter") getline value < (_param "value") getline indentation < (_param "indentation") + getline delimiter_space < (_param "delimiter-space") do_normalization = (system("test -f " (_param "normalize")) == 0) @@ -108,10 +109,9 @@ found_section { # must only contain spaces else start_spaces = ikey - 1 } - # start_spaces set or 0 idelspace_start = ikey + length(key) - idelspace_length = idel - (idelspace_start) + idelspace_length = idel - idelspace_start # check for delimiter is only preceded with spaces if(idelspace_length == 0 || check_spaces(substr(line, idelspace_start, idelspace_length))) { found = 1 @@ -129,8 +129,16 @@ found_section { else { # check if the format is important if(do_normalization) { + if(match(found_value, /^[ \t]+/) == 1) { + found_value = substr(found_value, 1 + RLENGTH) + del_val_spacelen = RLENGTH + } + else + del_val_spacelen = 0 + # the format must exactly match, else it is incorrect - if(indentation != start_spaces || idelspace_length != 0 || found_value != is_value) + if(start_spaces != indentation || found_value != is_value || + idelspace_length != delimiter_space || del_val_spacelen != delimiter_space) state("wrongformat") } diff --git a/cdist/conf/type/__ini_value/files/common.awk b/cdist/conf/type/__ini_value/files/common.awk index 8874f0b8..7611638c 100644 --- a/cdist/conf/type/__ini_value/files/common.awk +++ b/cdist/conf/type/__ini_value/files/common.awk @@ -6,9 +6,14 @@ BEGIN { value = get_param_string("value") comment = get_param_string("comment") indentation = get_param_string("indentation") + delimiter_space = get_param_string("delimiter-space") get_param_array("comment-sign", comment_signs) comment_sign = comment_signs[0] + + base_spaces = spaces(indentation) + delimiter_spaces = spaces(delimiter_space + delimiter_w_spaces = (delimiter_spaces delimiter delimiter_spaces) } function trim(var) { @@ -16,6 +21,12 @@ function trim(var) { sub(/[ \t]*$/, "", var) return var } +function spaces(a) { + rspaces = "" + for(b = 0; b < a; b++) + rspaces = (rspaces " ") + return rspaces +} function check_spaces(part) { return match(part, /^[ \t]*$/) == 1 } @@ -40,22 +51,13 @@ function get_param_array(name, arr) { # print value function v_print() { - spaces = "" - for(i = 0; i < indentation; i++) - spaces = (spaces " ") - printf "%s%s%s%s%s", spaces, key, delimiter, value, ORS + printf "%s%s%s%s%s", base_spaces, key, delimiter_w_spaces, value, ORS } # print commented value function v_print_commented() { - spaces = "" - for(i = 0; i < indentation; i++) - spaces = (spaces " ") - printf "%s%s%s%s%s%s", spaces, comment_sign, key, delimiter, value, ORS + printf "%s%s%s%s%s%s", base_spaces, comment_sign, key, delimiter_w_spaces, value, ORS } # print comment function c_print() { - spaces = "" - for(i = 0; i < indentation; i++) - spaces = (spaces " ") - printf "%s%s%s%s%s", spaces, comment_sign, " ", comment, ORS + printf "%s%s%s%s%s", base_spaces, comment_sign, " ", comment, ORS } diff --git a/cdist/conf/type/__ini_value/man.rst b/cdist/conf/type/__ini_value/man.rst index c583d5ad..0812d432 100644 --- a/cdist/conf/type/__ini_value/man.rst +++ b/cdist/conf/type/__ini_value/man.rst @@ -63,6 +63,14 @@ comment-sign parameters are possible. It uses the first specified sign as comment character if this type needs to insert comments. +delimiter-space + The number of spaces before and after the delimiter which should be free. + This number applies to each site of the delimiter separately, so one space + means one space to the left and right side of the delimiter. + + The delimiter will be matched independendtly of this parameter and will + only be corrected if ``--normalize`` is set. + BOOLEAN PARAMETERS ------------------ diff --git a/cdist/conf/type/__ini_value/parameter/optional b/cdist/conf/type/__ini_value/parameter/optional index 031c807b..6acbf8e5 100644 --- a/cdist/conf/type/__ini_value/parameter/optional +++ b/cdist/conf/type/__ini_value/parameter/optional @@ -4,3 +4,4 @@ state value indentation comment +delimiter-space From 7380fcaaf96b047e6358e6a183e76138c33d42df Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 13 Mar 2021 18:46:04 +0100 Subject: [PATCH 08/45] __ini_value: add missing bracket (oops) --- cdist/conf/type/__ini_value/files/common.awk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__ini_value/files/common.awk b/cdist/conf/type/__ini_value/files/common.awk index 7611638c..c8505b40 100644 --- a/cdist/conf/type/__ini_value/files/common.awk +++ b/cdist/conf/type/__ini_value/files/common.awk @@ -12,7 +12,7 @@ BEGIN { comment_sign = comment_signs[0] base_spaces = spaces(indentation) - delimiter_spaces = spaces(delimiter_space + delimiter_spaces = spaces(delimiter_space) delimiter_w_spaces = (delimiter_spaces delimiter delimiter_spaces) } From e3e4a91abe526d681e2c0d1abc67fdd3281eae36 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 14 Mar 2021 12:28:55 +0100 Subject: [PATCH 09/45] __ini_value: moved awk script generation to gencode-remote The script generation of the awk script was moved from `files/gen-awk.sh` to `gencode-remote` because the size not really matters. If the file does not exist, it will be created by a predefined template via an here-doc to avoid a big awk-script where it is not needed. --- cdist/conf/type/__ini_value/files/gen-awk.sh | 65 ----------- cdist/conf/type/__ini_value/gencode-remote | 110 ++++++++++++++++++- 2 files changed, 108 insertions(+), 67 deletions(-) delete mode 100755 cdist/conf/type/__ini_value/files/gen-awk.sh diff --git a/cdist/conf/type/__ini_value/files/gen-awk.sh b/cdist/conf/type/__ini_value/files/gen-awk.sh deleted file mode 100755 index a73a3f7b..00000000 --- a/cdist/conf/type/__ini_value/files/gen-awk.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh -e -# __ini_value/files/gen-awk.sh - -# Generates the awk script that will modify the line -# -# Arguments: -# 1: the file to modify -# 2: the should_state - -strip_comments() { - grep -v '^[[:space:]]*\($\|#\)' -} - - -file="$1" -state="$2" - - -# Generate the basic awk struct -cat < "\$tmpfile" <<'AWK' -SHELL - - -# generate the awk script and strip unnecessary things -{ - # basic functions which everyone needs - cat "$__type/files/common.awk" - - # generate the script and strip - awk -v state="$state" ' - function parse(line) { - if(match(line, /^[ \t]*# %code_print%$/) > 0) { - if(state == "present") - print "v_print()" - else if(state == "commented") - print "v_print_commented()" - else - print "print \"script compile error! cdist state " state " unkown!\" > /dev/stderr" - } - else print line - } - { - if(match($0, /^[ \t]*# %codeblock_([^%]+)%$/) > 0) { - split($2, result, "_"); type = substr(result[2], 1, length(result[2]) - 1) - file = (ENVIRON["__type"] "/files/parts/" state "/" type ".awk") - while((getline line < file) > 0) - parse(line) - close(file) - } - else print - }' "$__type/files/base.awk" -} | strip_comments - - -# end of here-doc -cat < "\$tmpfile" <<'AWK' +SHELL + + # generate the awk script and strip unnecessary things + { + # basic functions which everyone needs + cat "$__type/files/common.awk" + + # generate the script + awk -v state="$state_should" ' + function parse(line) { + if(match(line, /^[ \t]*# %code_print%$/) > 0) { + if(state == "present") + print "v_print()" + else if(state == "commented") + print "v_print_commented()" + else + print "print \"script compile error! cdist state " state " unkown!\" > /dev/stderr" + } + else print line + } + { + if(match($0, /^[ \t]*# %codeblock_([^%]+)%$/) > 0) { + split($2, result, "_"); type = substr(result[2], 1, length(result[2]) - 1) + file = (ENVIRON["__type"] "/files/parts/" state "/" type ".awk") + while((getline line < file) > 0) + parse(line) + close(file) + } + else print + }' "$__type/files/base.awk" + } | strip_comments + + # end of here-doc + cat < '%s'\n" "$file" + printf "%s cdist-generated file by %s/%s\n" "$comment_sign" \ + "${__type##*/}" "$__object_id" + echo + + # print values if available + section="$__object/parameter/section" + if [ -f "$section" ]; then + cat "$section" + echo + fi + comment="$__object/parameter/comment" + if [ -f "$comment" ]; then + printf "%s%s " "$i_indent" "$comment_sign" + cat "$comment" + fi + + # print the value + printf "%s%s%s%s%s%s%s\n" \ + "$i_indent" \ + "$(if [ "$state_should" = "commented" ]; then + printf "%s" "$comment_sign"; + fi)" \ + "$key" "$i_del_space" "$delimiter" "$i_del_space" "$value" + echo + + # terminate HERE-DOC + echo "FILE" + fi ;; *) From 8f687e5ce2e8ede46149be754f1a809c6a850c6c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 14 Mar 2021 14:50:21 +0100 Subject: [PATCH 10/45] __ini_value: make shellcheck happy --- cdist/conf/type/__ini_value/gencode-remote | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__ini_value/gencode-remote b/cdist/conf/type/__ini_value/gencode-remote index d110d2a9..dc6a0431 100755 --- a/cdist/conf/type/__ini_value/gencode-remote +++ b/cdist/conf/type/__ini_value/gencode-remote @@ -75,8 +75,8 @@ SHELL elif [ "$state_should" != "absent" ]; then # generate the indentation string spaces() { - _i=$1 - while [ $_i -gt 0 ]; do + _i="$1" + while [ "$_i" -gt 0 ]; do printf " " _i=$((_i - 1)) done @@ -88,10 +88,10 @@ SHELL delimiter="$(cat "$__object/parameter/delimiter")" value="$(cat "$__object/parameter/value")" indentation="$(cat "$__object/parameter/indentation" || true)" - i_indent="$(spaces $indentation)" + i_indent="$(spaces "$indentation")" delimiter_space="$(cat "$__object/parameter/delimiter-space" || true)" - i_del_space="$(spaces $delimiter_space)" - comment_sign="$(cat "$__object/parameter/comment-sign" | cut -c1)" + i_del_space="$(spaces "$delimiter_space")" + comment_sign="$(cut -c1 < "$__object/parameter/comment-sign")" # Generate a simple shell script which just generates the file printf "cat <<'FILE' > '%s'\n" "$file" From ad736a66d055fc09b476a994e61dbe6fe10fff59 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 16 Mar 2021 21:38:08 +0100 Subject: [PATCH 11/45] __ini_value: remove is_state 'nosuchfile' Because the state 'nosuchfile' is exactly the same as 'absent', it was removed with the code connected to it. It's not important if the file exists or not - because in both cases, it contains no key-value. Also, the code if the explorer returned 'nosuchfile' was wrong: Completly overwrite the file with the assumption the file does not exist is not correct, as it can return for multiple objects of the same file and therefore, they overwrite themself. --- cdist/conf/type/__ini_value/explorer/state | 2 +- cdist/conf/type/__ini_value/gencode-remote | 118 ++++++--------------- 2 files changed, 35 insertions(+), 85 deletions(-) diff --git a/cdist/conf/type/__ini_value/explorer/state b/cdist/conf/type/__ini_value/explorer/state index e387d104..3827a354 100755 --- a/cdist/conf/type/__ini_value/explorer/state +++ b/cdist/conf/type/__ini_value/explorer/state @@ -21,7 +21,7 @@ file="$(cat "$__object/parameter/file")" # abort if no file exist if ! [ -f "$file" ]; then - echo nosuchfile + echo absent exit fi diff --git a/cdist/conf/type/__ini_value/gencode-remote b/cdist/conf/type/__ini_value/gencode-remote index dc6a0431..ab93077e 100755 --- a/cdist/conf/type/__ini_value/gencode-remote +++ b/cdist/conf/type/__ini_value/gencode-remote @@ -24,9 +24,8 @@ file="$(cat "$__object/parameter/file")" # validation check case "$state_should" in present|commented|absent) - if [ "$state_is" != "nosuchfile" ]; then - # Generate the basic awk struct if a file already exists - cat < "\$tmpfile" <<'AWK' SHELL - # generate the awk script and strip unnecessary things - { - # basic functions which everyone needs - cat "$__type/files/common.awk" + # generate the awk script and strip unnecessary things + { + # basic functions which everyone needs + cat "$__type/files/common.awk" - # generate the script - awk -v state="$state_should" ' - function parse(line) { - if(match(line, /^[ \t]*# %code_print%$/) > 0) { - if(state == "present") - print "v_print()" - else if(state == "commented") - print "v_print_commented()" - else - print "print \"script compile error! cdist state " state " unkown!\" > /dev/stderr" - } - else print line + # generate the script + awk -v state="$state_should" ' + function parse(line) { + if(match(line, /^[ \t]*# %code_print%$/) > 0) { + if(state == "present") + print "v_print()" + else if(state == "commented") + print "v_print_commented()" + else + print "print \"script compile error! cdist state " state " unkown!\" > /dev/stderr" } - { - if(match($0, /^[ \t]*# %codeblock_([^%]+)%$/) > 0) { - split($2, result, "_"); type = substr(result[2], 1, length(result[2]) - 1) - file = (ENVIRON["__type"] "/files/parts/" state "/" type ".awk") - while((getline line < file) > 0) - parse(line) - close(file) - } - else print - }' "$__type/files/base.awk" - } | strip_comments + else print line + } + { + if(match($0, /^[ \t]*# %codeblock_([^%]+)%$/) > 0) { + split($2, result, "_"); type = substr(result[2], 1, length(result[2]) - 1) + file = (ENVIRON["__type"] "/files/parts/" state "/" type ".awk") + while((getline line < file) > 0) + parse(line) + close(file) + } + else print + }' "$__type/files/base.awk" + } | strip_comments - # end of here-doc - cat < '%s'\n" "$file" - printf "%s cdist-generated file by %s/%s\n" "$comment_sign" \ - "${__type##*/}" "$__object_id" - echo - - # print values if available - section="$__object/parameter/section" - if [ -f "$section" ]; then - cat "$section" - echo - fi - comment="$__object/parameter/comment" - if [ -f "$comment" ]; then - printf "%s%s " "$i_indent" "$comment_sign" - cat "$comment" - fi - - # print the value - printf "%s%s%s%s%s%s%s\n" \ - "$i_indent" \ - "$(if [ "$state_should" = "commented" ]; then - printf "%s" "$comment_sign"; - fi)" \ - "$key" "$i_del_space" "$delimiter" "$i_del_space" "$value" - echo - - # terminate HERE-DOC - echo "FILE" - fi + # Do not threat it differently if the file does not exist. It's just + # absent. Because multiple explorers can say the file does not exist, + # so the file should not be completly overwritten all times. ;; *) From 46638f0839cff327b00c914fe3d626d8e16791f6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 17 Mar 2021 18:17:21 +0100 Subject: [PATCH 12/45] __ini_value: change empty value check Change the short-circuit check for an empty value to a more structual approach. This might be better for the code or not .. --- cdist/conf/type/__ini_value/files/base.awk | 79 +++++++++++----------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/cdist/conf/type/__ini_value/files/base.awk b/cdist/conf/type/__ini_value/files/base.awk index 57535964..ddc52949 100644 --- a/cdist/conf/type/__ini_value/files/base.awk +++ b/cdist/conf/type/__ini_value/files/base.awk @@ -66,56 +66,53 @@ found {print; next} # main loop (til the line was found) !found { line = trim($0) - # short curcit on empty lines - if(line == "") { - push_line() - next - } + # process if the line is not empty (or only contains spaces) + if(line != "") { + # check for a ini section + if(substr(line, 1, 1) == "[" && substr(line, length(line), 1) == "]") { + is_section = 1 + curr_section = line - # check for a ini section - if(substr(line, 1, 1) == "[" && substr(line, length(line), 1) == "]") { - is_section = 1 - curr_section = line + if(curr_section == section) { + found_section = 1 + is_curr_section = 1 + } + else { + # if nothing found, print it in the valid section before the next one + if(is_curr_section) { + if(!found) { + # set found as it is there now + found=1 - if(curr_section == section) { - found_section = 1 - is_curr_section = 1 + # %codeblock_insert% + + # print line as it would else only be populated below + print + next + } + is_curr_section = 0 + } + } } else { - # if nothing found, print it in the valid section before the next one + # only current session is interessting if(is_curr_section) { - if(!found) { - # set found as it is there now - found=1 - - # %codeblock_insert% - - # print line as it would else only be populated below - print - next + # check for a comment + is_com = is_comment(line) + if(is_com) { + line = trim(substr(line, 2)) } - is_curr_section = 0 - } - } - } - else { - # only current session is interessting - if(is_curr_section) { - # check for a comment - is_com = is_comment(line) - if(is_com) { - line = trim(substr(line, 2)) - } - # check for a delimiter and a key (must be at first position due to trimming) - if((idel = index(line, delimiter)) && (ikey = index(line, key)) == 1) { - # check there are only spaces between the key and delimiter - if(check_spaces(substr(line, ikey + length(key), idel - (length(key) + 1)))) { - found = 1 + # check for a delimiter and a key (must be at first position due to trimming) + if((idel = index(line, delimiter)) && (ikey = index(line, key)) == 1) { + # check there are only spaces between the key and delimiter + if(check_spaces(substr(line, ikey + length(key), idel - (length(key) + 1)))) { + found = 1 - # %codeblock_found% + # %codeblock_found% - next + next + } } } } From 19c701e95dcbc60938cd554532d9503eb71605ec Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 20 Mar 2021 15:10:11 +0100 Subject: [PATCH 13/45] __ini_value: make type nonparallel Because of the temporary files and because multiple types can change a file at the same time, it can't be run in parallel. --- cdist/conf/type/__ini_value/nonparallel | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cdist/conf/type/__ini_value/nonparallel diff --git a/cdist/conf/type/__ini_value/nonparallel b/cdist/conf/type/__ini_value/nonparallel new file mode 100644 index 00000000..e69de29b From 0d4868dcd52c5c172c680bcf5aa42b3da16c7611 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 20 Mar 2021 15:39:38 +0100 Subject: [PATCH 14/45] __ini_value: new parameter --quote This parameter ensures the value is surrounded by double quotes. As it directly edits the value variable, the quoting will be added/removed automaticly. --- cdist/conf/type/__ini_value/explorer/state | 5 +++++ cdist/conf/type/__ini_value/files/common.awk | 7 ++++++- cdist/conf/type/__ini_value/man.rst | 4 ++++ cdist/conf/type/__ini_value/parameter/boolean | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__ini_value/explorer/state b/cdist/conf/type/__ini_value/explorer/state index 3827a354..507cabe9 100755 --- a/cdist/conf/type/__ini_value/explorer/state +++ b/cdist/conf/type/__ini_value/explorer/state @@ -57,6 +57,11 @@ BEGIN { comment_signs[i++] = tmp } + if(system("test -f " (_param "quote")) == 0) { + # quote it now that it only wins checks against quoted values + value = ("\"" value "\"") + } + found=0 curr_section="" if(section == "") diff --git a/cdist/conf/type/__ini_value/files/common.awk b/cdist/conf/type/__ini_value/files/common.awk index c8505b40..e5f3594c 100644 --- a/cdist/conf/type/__ini_value/files/common.awk +++ b/cdist/conf/type/__ini_value/files/common.awk @@ -11,6 +11,11 @@ BEGIN { get_param_array("comment-sign", comment_signs) comment_sign = comment_signs[0] + if(system("test -f " (_param "quote")) == 0) { + # quote it now that it only wins checks against quoted values + value = ("\"" value "\"") + } + base_spaces = spaces(indentation) delimiter_spaces = spaces(delimiter_space) delimiter_w_spaces = (delimiter_spaces delimiter delimiter_spaces) @@ -59,5 +64,5 @@ function v_print_commented() { } # print comment function c_print() { - printf "%s%s%s%s%s", base_spaces, comment_sign, " ", comment, ORS + printf "%s%s %s%s", base_spaces, comment_sign, comment, ORS } diff --git a/cdist/conf/type/__ini_value/man.rst b/cdist/conf/type/__ini_value/man.rst index 0812d432..38a8d67c 100644 --- a/cdist/conf/type/__ini_value/man.rst +++ b/cdist/conf/type/__ini_value/man.rst @@ -79,6 +79,10 @@ normalize configuration file. Even if a key-value pair is correct as-is, it will correct the line to be pretty and perfect. +quote + Wrap double quotes (``"``) around the value. If the value is previously + unquoted, the file will be modified to quote the value. + MESSAGES -------- diff --git a/cdist/conf/type/__ini_value/parameter/boolean b/cdist/conf/type/__ini_value/parameter/boolean index f9a8b6aa..5b7d2b39 100644 --- a/cdist/conf/type/__ini_value/parameter/boolean +++ b/cdist/conf/type/__ini_value/parameter/boolean @@ -1 +1,2 @@ normalize +quote From 485283f2e531f5fa23a876fa52411e0030f32589 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 20:47:22 +0300 Subject: [PATCH 15/45] new type: __sed --- cdist/conf/type/__sed/explorer/file | 22 +++++++++ cdist/conf/type/__sed/gencode-remote | 46 +++++++++++++++++ cdist/conf/type/__sed/man.rst | 49 +++++++++++++++++++ cdist/conf/type/__sed/parameter/boolean | 1 + cdist/conf/type/__sed/parameter/optional | 1 + .../type/__sed/parameter/required_multiple | 1 + 6 files changed, 120 insertions(+) create mode 100755 cdist/conf/type/__sed/explorer/file create mode 100755 cdist/conf/type/__sed/gencode-remote create mode 100644 cdist/conf/type/__sed/man.rst create mode 100644 cdist/conf/type/__sed/parameter/boolean create mode 100644 cdist/conf/type/__sed/parameter/optional create mode 100644 cdist/conf/type/__sed/parameter/required_multiple diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file new file mode 100755 index 00000000..a40f71b3 --- /dev/null +++ b/cdist/conf/type/__sed/explorer/file @@ -0,0 +1,22 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/file" ] +then + file="$( cat "$__object/parameter/file" )" +else + file="/$__object_id" +fi + +if [ -f "$file" ] +then + if [ -s "$file" ] + then + cat "$file" + else + echo "$file is empty" >&2 + exit 1 + fi +else + echo "$file do not exist" >&2 + exit 1 +fi diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote new file mode 100755 index 00000000..391e3bc1 --- /dev/null +++ b/cdist/conf/type/__sed/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/file" ] +then + file="$( cat "$__object/parameter/file" )" +else + file="/$__object_id" +fi + +script="$( cat "$__object/parameter/script" )" + +if [ "$script" = '-' ] +then + script="$( cat "$__object/stdin" )" +elif + [ -f "$script" ] +then + script="$( cat "$script" )" +fi + +file_from_target="$__object/explorer/file" + +sed_cmd='sed' + +if [ -f "$__object/parameter/regexp-extended" ] +then + sed_cmd="$sed_cmd --regexp-extended" +fi + +if ! echo "$script" \ + | "$sed_cmd" -f - "$file_from_target" \ + | diff "$file_from_target" - \ + > /dev/null +then + echo 'tmp="$( mktemp )"' + + echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" + + echo "$script" + + echo 'EOF' + + echo "cp \"\$tmp\" '$file'" + + echo 'rm -f "$tmp"' +fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst new file mode 100644 index 00000000..fbac212d --- /dev/null +++ b/cdist/conf/type/__sed/man.rst @@ -0,0 +1,49 @@ +cdist-type__sed(7) +================== + +NAME +---- +cdist-type__sed - Transform text files with ``sed`` + + +DESCRIPTION +----------- +TODO + + +REQUIRED MULTIPLE PARAMETERS +---------------------------- +script + TODO + + +OPTIONAL PARAMETERS +------------------- +file + TODO + + +BOOLEAN PARAMETERS +------------------ +regexp-extended + TODO + + +EXAMPLES +-------- +.. code-block:: sh + + true + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean new file mode 100644 index 00000000..1ad75c5d --- /dev/null +++ b/cdist/conf/type/__sed/parameter/boolean @@ -0,0 +1 @@ +regexp-extended diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional new file mode 100644 index 00000000..f73f3093 --- /dev/null +++ b/cdist/conf/type/__sed/parameter/optional @@ -0,0 +1 @@ +file diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple new file mode 100644 index 00000000..84f7e31d --- /dev/null +++ b/cdist/conf/type/__sed/parameter/required_multiple @@ -0,0 +1 @@ +script From 7a5896acfad0f992d6e7fe9cbe0bd08035ce19db Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 21:23:25 +0300 Subject: [PATCH 16/45] add --onchange, fix shellcheck --- cdist/conf/type/__sed/gencode-remote | 9 ++++++++- cdist/conf/type/__sed/man.rst | 3 +++ cdist/conf/type/__sed/parameter/optional | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 391e3bc1..6b1bd216 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -32,6 +32,7 @@ if ! echo "$script" \ | diff "$file_from_target" - \ > /dev/null then + # shellcheck disable=SC2016 echo 'tmp="$( mktemp )"' echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" @@ -41,6 +42,12 @@ then echo 'EOF' echo "cp \"\$tmp\" '$file'" - + + # shellcheck disable=SC2016 echo 'rm -f "$tmp"' + + if [ -f "$__object/parameter/onchange" ] + then + cat "$__object/parameter/onchange" + fi fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst index fbac212d..4a728184 100644 --- a/cdist/conf/type/__sed/man.rst +++ b/cdist/conf/type/__sed/man.rst @@ -22,6 +22,9 @@ OPTIONAL PARAMETERS file TODO +onchange + TODO + BOOLEAN PARAMETERS ------------------ diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional index f73f3093..fa86f917 100644 --- a/cdist/conf/type/__sed/parameter/optional +++ b/cdist/conf/type/__sed/parameter/optional @@ -1 +1,2 @@ file +onchange From cf0032d667eb4bd8b7fe1bebacec341bbf71ed42 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 21:28:00 +0300 Subject: [PATCH 17/45] add messaging and exit earlier --- cdist/conf/type/__sed/gencode-remote | 42 +++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 6b1bd216..836506f0 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -27,27 +27,31 @@ then sed_cmd="$sed_cmd --regexp-extended" fi -if ! echo "$script" \ +if echo "$script" \ | "$sed_cmd" -f - "$file_from_target" \ | diff "$file_from_target" - \ > /dev/null then - # shellcheck disable=SC2016 - echo 'tmp="$( mktemp )"' - - echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" - - echo "$script" - - echo 'EOF' - - echo "cp \"\$tmp\" '$file'" - - # shellcheck disable=SC2016 - echo 'rm -f "$tmp"' - - if [ -f "$__object/parameter/onchange" ] - then - cat "$__object/parameter/onchange" - fi + exit 0 +fi + +# shellcheck disable=SC2016 +echo 'tmp="$( mktemp )"' + +echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" + +echo "$script" + +echo 'EOF' + +echo "cp \"\$tmp\" '$file'" + +# shellcheck disable=SC2016 +echo 'rm -f "$tmp"' + +echo 'change' >> "$__messages_out" + +if [ -f "$__object/parameter/onchange" ] +then + cat "$__object/parameter/onchange" fi From d7fdc8006f747be86b64a2790266b50219bab507 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 21:54:45 +0300 Subject: [PATCH 18/45] allow empty file --- cdist/conf/type/__sed/explorer/file | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index a40f71b3..5f0f4edd 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -9,13 +9,7 @@ fi if [ -f "$file" ] then - if [ -s "$file" ] - then - cat "$file" - else - echo "$file is empty" >&2 - exit 1 - fi + cat "$file" else echo "$file do not exist" >&2 exit 1 From 0f6e48dbc657a9d9145b965eacf79f021a1419f5 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:24:26 +0300 Subject: [PATCH 19/45] use $__object/tempfile in target instead of mktemp, add comments --- cdist/conf/type/__sed/gencode-remote | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 836506f0..df929b42 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -35,8 +35,11 @@ then exit 0 fi +# we can't use -i, because it's not posix, so we fly with tempfile and cp +# and we use cp because we want to preserve destination file's attributes + # shellcheck disable=SC2016 -echo 'tmp="$( mktemp )"' +echo 'tmp="$__object/tempfile"' echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" From 90488fcebcc00f3b4ea260867f8b31625df61e46 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:27:42 +0300 Subject: [PATCH 20/45] use -e --- cdist/conf/type/__sed/explorer/file | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index 5f0f4edd..9262b613 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -7,10 +7,10 @@ else file="/$__object_id" fi -if [ -f "$file" ] +if [ ! -e "$file" ] then - cat "$file" -else echo "$file do not exist" >&2 exit 1 fi + +cat "$file" From b7f392fa371ff78591abb910f7e224e7587c81be Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:38:55 +0300 Subject: [PATCH 21/45] use -E for better compat (not really sure if it is posix at all) --- cdist/conf/type/__sed/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index df929b42..625448e6 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -24,7 +24,7 @@ sed_cmd='sed' if [ -f "$__object/parameter/regexp-extended" ] then - sed_cmd="$sed_cmd --regexp-extended" + sed_cmd="$sed_cmd -E" fi if echo "$script" \ From aabef7f44a98ad144a7e1fe68bfb562079916ef8 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:40:06 +0300 Subject: [PATCH 22/45] remove reading script from file --- cdist/conf/type/__sed/gencode-remote | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 625448e6..be7690e1 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -12,10 +12,6 @@ script="$( cat "$__object/parameter/script" )" if [ "$script" = '-' ] then script="$( cat "$__object/stdin" )" -elif - [ -f "$script" ] -then - script="$( cat "$script" )" fi file_from_target="$__object/explorer/file" From 5bf0c71e7af49ff6e7c1cccec192f763a019503e Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:45:36 +0300 Subject: [PATCH 23/45] update man --- cdist/conf/type/__sed/man.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst index 4a728184..86789363 100644 --- a/cdist/conf/type/__sed/man.rst +++ b/cdist/conf/type/__sed/man.rst @@ -8,35 +8,40 @@ cdist-type__sed - Transform text files with ``sed`` DESCRIPTION ----------- -TODO +Transform text files with ``sed``. REQUIRED MULTIPLE PARAMETERS ---------------------------- script - TODO + ``sed`` script. + If ``-`` then the script is read from ``stdin``. OPTIONAL PARAMETERS ------------------- file - TODO + Path to the file. Defaults to ``$__object_id``. onchange - TODO + Execute this command if ``sed`` changes file. BOOLEAN PARAMETERS ------------------ regexp-extended - TODO + Use extended regular expressions in the script. + Might not be supported with every ``sed`` version. EXAMPLES -------- + .. code-block:: sh - true + __sed /tmp/foobar --script 's/foo/bar/' + + echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script - AUTHORS From cd4acde67ec9c3967e6e08450734cf65040e787f Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 15 Sep 2021 09:22:27 +0300 Subject: [PATCH 24/45] grammar --- cdist/conf/type/__sed/explorer/file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index 9262b613..ec3d0fe8 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -9,7 +9,7 @@ fi if [ ! -e "$file" ] then - echo "$file do not exist" >&2 + echo "$file does not exist" >&2 exit 1 fi From 7b6789ddeb74450f70991efe957e3eb0f11043f6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:04:12 +0200 Subject: [PATCH 25/45] __package_update_index: fix complain about suite change 2 of 4th fix for ticket #861 --- .../type/__package_update_index/gencode-remote | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 803468b5..a10c16d3 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -41,7 +41,19 @@ fi case "$type" in yum) ;; apt) - echo "apt-get --quiet update" + # There are special arguments to apt(8) to prevent aborts if apt woudn't been + # updated after the 19th April 2021 till the bullseye release. The additional + # arguments acknoledge the happend suite change (the apt(8) update does the + # same by itself). + # + # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter + # allows backward compatablility to pre-buster Debian versions. + # + # See more: ticket #861 + # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 + apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + + echo "apt-get --quiet $apt_opts update" echo "apt-cache updated (age was: $currage)" >> "$__messages_out" ;; pacman) From 12787ffe2c48f5c6d534d10f8faadde30db477ac Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:12:20 +0200 Subject: [PATCH 26/45] __apt_source: fix complain about suite change 3 of 4th fix for ticket #861 --- cdist/conf/type/__apt_source/gencode-remote | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote index 1e8592c6..973b0f6c 100755 --- a/cdist/conf/type/__apt_source/gencode-remote +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -22,7 +22,21 @@ name="$__object_id" destination="/etc/apt/sources.list.d/${name}.list" +# There are special arguments to apt(8) to prevent aborts if apt woudn't been +# updated after the 19th April 2021 till the bullseye release. The additional +# arguments acknoledge the happend suite change (the apt(8) update does the +# same by itself). +# +# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter +# allows backward compatablility to pre-buster Debian versions. +# +# See more: ticket #861 +# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 +apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + +# run 'apt-get update' only if something changed with our sources.list file +# it will be run a second time on error as a redundancy messure to success if grep -q "^__file${destination}" "$__messages_in"; then - printf 'apt-get update || apt-get update\n' + printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts" fi From d246e067109dc8b85ea89668f742affbb221d1a6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 14:49:23 +0200 Subject: [PATCH 27/45] __apt_update_index: fix complain about suite change 1 of 4th fix for ticket #861 --- .../conf/type/__apt_update_index/gencode-remote | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 70b59710..2d7f9030 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -18,9 +18,23 @@ # along with cdist. If not, see . # + +# There are special arguments to apt(8) to prevent aborts if apt woudn't been +# updated after the 19th April 2021 till the bullseye release. The additional +# arguments acknoledge the happend suite change (the apt(8) update does the +# same by itself). +# +# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter +# allows backward compatablility to pre-buster Debian versions. +# +# See more: ticket #861 +# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 +apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists +# it will be run a second time on error as a redundancy messure to success cat << DONE if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then - apt-get update || apt-get update + apt-get $apt_opts update || apt-get $apt_opts update fi DONE From 3d7b31cbb43067271c2c510281f90f19dc5ec5fb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:22:16 +0200 Subject: [PATCH 28/45] __package_apt: fix complain about suite change the last fix for ticket #861 :-) --- cdist/conf/type/__package_apt/gencode-remote | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index fbfca330..79c0d9d3 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::= case "$state_should" in present) + # There are special arguments to apt(8) to prevent aborts if apt woudn't been + # updated after the 19th April 2021 till the bullseye release. The additional + # arguments acknoledge the happend suite change (the apt(8) update does the + # same by itself). + # + # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter + # allows backward compatablility to pre-buster Debian versions. + # + # See more: ticket #861 + # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 + apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + # following is bit ugly, but important hack. # due to how cdist config run works, there isn't # currently better way to do it :( cat << EOF if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] -then echo apt-get update > /dev/null 2>&1 || true +then echo apt-get $apt_opts update > /dev/null 2>&1 || true fi EOF if [ -n "$version" ]; then From 72ff48154c958ebdedc8b6d919b6f372efa63be2 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 16 Sep 2021 21:36:39 +0300 Subject: [PATCH 29/45] add comments, add -u to diff --- cdist/conf/type/__sed/gencode-remote | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index be7690e1..f99c5a88 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -14,6 +14,8 @@ then script="$( cat "$__object/stdin" )" fi +# since stdin is not available in explorer, we pull file from target with explorer + file_from_target="$__object/explorer/file" sed_cmd='sed' @@ -23,10 +25,10 @@ then sed_cmd="$sed_cmd -E" fi -if echo "$script" \ - | "$sed_cmd" -f - "$file_from_target" \ - | diff "$file_from_target" - \ - > /dev/null +# do sed dry run, diff result and if no change, then there's nothing to do +# also redirect diff's output to stderr for debugging purposes + +if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2 then exit 0 fi From bf222d05435000063930ff199bd0dec94d7d8cd5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 21 Sep 2021 08:55:54 +0200 Subject: [PATCH 30/45] ++changelog --- docs/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog b/docs/changelog index 693d028f..ab637f5f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,11 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) + * New type: __sed (Ander Punnar) + * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __apt_source: Fix complaint about suite change (Matthias Stecher) + * Type __package_apt: Fix complaint about suite change (Matthias Stecher) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 122354a0dcc071b0b21eea644b37a988383aaec7 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 29 Sep 2021 20:23:01 +0200 Subject: [PATCH 31/45] __ini_value: draft of comment handling work in progress for a long time and I didn't touched it again till just to commit it ... I have to work on it. --- cdist/conf/type/__ini_value/files/comment.awk | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 cdist/conf/type/__ini_value/files/comment.awk diff --git a/cdist/conf/type/__ini_value/files/comment.awk b/cdist/conf/type/__ini_value/files/comment.awk new file mode 100644 index 00000000..1d3c04b7 --- /dev/null +++ b/cdist/conf/type/__ini_value/files/comment.awk @@ -0,0 +1,57 @@ +# We try to find a comment block -- how? +# check how much paragraphs it has +# check if +# +# this code is crap - at least not well written + +# Check the buffer if the comment was found +function check_comments() { + _lastline = bufindex - (buflen - 1) + _comm_size = length(comments) + + lastfreeline = 0 + lastfreecommline = 0 + comm_index = 0 + + # go through all lines + for(i = bufindex; i < _lastline; i++) { + _line = trim(linebuf[i]) + + # empty line? + if(_line == "") { + lastfreeline = i + continue + } + + # line start matched + if(_line == trim(comments[comm_index])) { + # end? else continue + if(comm_index < _comm_size) { + continue + } + else { + + } + } + # reset again cause not matched + else comm_index = 0 + + # empty comment line + if(is_comment(_line)) { + _comment = trim(substr(_line, 2)) + + # check if empty comment + if(_comment == "") { + lastfreecommline = i + } + } + + # check if comments fit in or is too big + if((_lastline - bufindex) < _comm_size) { + # too short + } + else { + #if() + } + } +} From b4f381ed4ca5fc8d012765ac3af50c79ac63692c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 29 Sep 2021 20:24:22 +0200 Subject: [PATCH 32/45] __ini_value: fix use of removed constant _param was removed in preference of the param functions. --- cdist/conf/type/__ini_value/files/common.awk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__ini_value/files/common.awk b/cdist/conf/type/__ini_value/files/common.awk index e5f3594c..bc51e0b8 100644 --- a/cdist/conf/type/__ini_value/files/common.awk +++ b/cdist/conf/type/__ini_value/files/common.awk @@ -11,7 +11,7 @@ BEGIN { get_param_array("comment-sign", comment_signs) comment_sign = comment_signs[0] - if(system("test -f " (_param "quote")) == 0) { + if(system("test -f " (ENVIRON["__object"] "/parameter/quote")) == 0) { # quote it now that it only wins checks against quoted values value = ("\"" value "\"") } From 15c642a9b7af42c368eb5a8e22cab5f07b39b95e Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 12:06:45 +0200 Subject: [PATCH 33/45] [__debconf_set_selections] Fix --file not being supported Even if deprecated, the parameter *must* be supported, which isn't the case right now. This was due to a misunderstanding of how deprecating parameters work, see: https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters --- .../conf/type/__debconf_set_selections/parameter/deprecated/file | 1 + .../__debconf_set_selections/parameter/{deprecated => optional} | 0 2 files changed, 1 insertion(+) create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/deprecated/file rename cdist/conf/type/__debconf_set_selections/parameter/{deprecated => optional} (100%) diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file new file mode 100644 index 00000000..09db545a --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file @@ -0,0 +1 @@ +'file' has been deprecated in favour of 'line' in order to provide idempotency. diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated b/cdist/conf/type/__debconf_set_selections/parameter/optional similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/deprecated rename to cdist/conf/type/__debconf_set_selections/parameter/optional From 5b7cca99f73e4ec3dd5cdb90e4dbfd40ec8fe349 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 12:09:42 +0200 Subject: [PATCH 34/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ab637f5f..142602c8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) + * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From fc9bd40c9a34cf54e2d6cfd2fec089bd25e1b172 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 13:14:57 +0200 Subject: [PATCH 35/45] Improve bullseye support, perticularly __letsencrypt_cert --- cdist/conf/type/__grafana_dashboard/manifest | 2 +- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index d145c4c3..0d944482 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -15,7 +15,7 @@ case $os in # Differntation not needed anymore apt_source_distribution=stable ;; - 10*) + 10*|11*) # Differntation not needed anymore apt_source_distribution=stable ;; diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 6394f629..638a99e0 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/stretch-backports" __package_apt certbot \ --target-release stretch-backports ;; - 10*) + 10*|11*) __package_apt certbot ;; From 560374a6861281695b61c8c2298123be41d92326 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 13:16:11 +0200 Subject: [PATCH 36/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 142602c8..6f717cf4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) + * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From c33d99ee120ad180daba144be80a25d77b473f56 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sun, 31 Oct 2021 17:38:10 +0100 Subject: [PATCH 37/45] [__haproxy_dualstack] New type with PROXY protocol support This is backwards compatible with what is already used internally @ungleich, but adds on top of that the ability to customise ports and, most importantly, it adds PROXY protocol support. --- .../conf/type/__haproxy_dualstack/files/http | 8 + .../conf/type/__haproxy_dualstack/files/https | 10 ++ .../conf/type/__haproxy_dualstack/files/imaps | 12 ++ .../conf/type/__haproxy_dualstack/files/smtps | 12 ++ cdist/conf/type/__haproxy_dualstack/man.rst | 121 ++++++++++++++ cdist/conf/type/__haproxy_dualstack/manifest | 155 ++++++++++++++++++ .../parameter/default/protocol | 1 + .../parameter/optional_multiple | 3 + cdist/conf/type/__haproxy_dualstack/singleton | 0 9 files changed, 322 insertions(+) create mode 100644 cdist/conf/type/__haproxy_dualstack/files/http create mode 100644 cdist/conf/type/__haproxy_dualstack/files/https create mode 100644 cdist/conf/type/__haproxy_dualstack/files/imaps create mode 100644 cdist/conf/type/__haproxy_dualstack/files/smtps create mode 100644 cdist/conf/type/__haproxy_dualstack/man.rst create mode 100644 cdist/conf/type/__haproxy_dualstack/manifest create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/default/protocol create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple create mode 100644 cdist/conf/type/__haproxy_dualstack/singleton diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http new file mode 100644 index 00000000..0508a465 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/http @@ -0,0 +1,8 @@ +frontend http + bind BIND@:80 + mode http + option httplog + default_backend http + +backend http + mode http diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https new file mode 100644 index 00000000..73deac46 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/https @@ -0,0 +1,10 @@ +frontend https + bind BIND@:443 + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend https + +backend https + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps new file mode 100644 index 00000000..b1ec3793 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/imaps @@ -0,0 +1,12 @@ +frontend imaps + bind BIND@:143 + bind BIND@:993 + + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend imaps + +backend imaps + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps new file mode 100644 index 00000000..dce6ed4a --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/smtps @@ -0,0 +1,12 @@ +frontend smtps + bind BIND@:25 + bind BIND@:465 + + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend smtps + +backend smtps + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst new file mode 100644 index 00000000..6c131cbe --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/man.rst @@ -0,0 +1,121 @@ +cdist-type__haproxy_dualstack(7) +================================ + + +NAME +---- +cdist-type__haproxy_dualstack - Proxy services from a dual-stack server + + +DESCRIPTION +----------- +This (singleton) type installs and configures haproxy to act as a dual-stack +proxy for single-stack services. + +This can be useful to add IPv4 support to IPv6-only services while only using +one IPv4 for many such services. + +By default this type uses the plain TCP proxy mode, which means that there is no +need for TLS termination on this host when SNI is supported. +This also means that proxied services will not receive the client's IP address, +but will see the proxy's IP address instead (that of `$__target_host`). + +This can be solved by using the PROXY protocol, but do take into account that, +e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same +port, so you will need to use other ports for that. + +As a recommendation in this type: use TCP ports 8080 and 591 respectively to +serve HTTP and HTTPS using the PROXY protocol. + +See the EXAMPLES for more details. + + +OPTIONAL PARAMETERS +------------------- +v4proxy + Proxy incoming IPv4 connections to the equivalent IPv6 endpoint. + In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is + the IP address actually providing the proxied services. + The full format of this argument is: + `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]` + Where starting with `proxy:` determines that the PROXY protocol must be + used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT + override for the given PROTOCOL (see `--protocol`), if not present the + PROTOCOL's default port will be used. + + +v6proxy + Proxy incoming IPv6 connections to the equivalent IPv4 endpoint. + In its simplest use, it must be a NAME with an `A` DNS entry, which is + the IP address actually providing the proxied services. + See `--v4proxy` for more options and details. + +protocol + Can be passed multiple times or as a space-separated list of protocols. + Currently supported protocols are: `http`, `https`, `imaps`, `smtps`. + This defaults to: `http https imaps smtps`. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Proxy the IPv6-only services so IPv4-only clients can access them + # This uses HAProxy's TCP mode for http, https, imaps and smtps + __haproxy_dualstack \ + --v4proxy ipv6.chat \ + --v4proxy matrix.ungleich.ch + + # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them + # Note this means that the backend IPv6-only server will only see + # the IPv6 address of the haproxy host managed by cdist, which can be + # troublesome if this information is relevant for analytics/security/... + # See the PROXY example below + __haproxy_dualstack \ + --protocol http --protocol https \ + --v4proxy ipv6.chat \ + --v4proxy matrix.ungleich.ch + + # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling + # IPv4-only clients to access them while maintaining the client's IP address + __haproxy_dualstack \ + --protocol http --protocol https \ + --v4proxy proxy:ipv6.chat:http=8080:https=591 \ + --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591 + # Note however that the PROXY protocol is not compatible with regular + # HTTP(S) protocols, so your nginx will have to listen on different ports + # with the PROXY settings. + # Note that you will need to restrict access to the 8080 port to prevent + # Client IP spoofing. + # This can be something like: + # server { + # # listen for regular HTTP connections + # listen [::]:80 default_server; + # listen 80 default_server; + # # listen for PROXY HTTP connections + # listen [::]:8080 proxy_protocol; + # # Accept the Client's IP from the PROXY protocol + # real_ip_header proxy_protocol; + # } + + +SEE ALSO +-------- +- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/ +- https://www.haproxy.com/blog/haproxy/proxy-protocol/ +- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/ + + +AUTHORS +------- +ungleich +Evilham + + +COPYING +------- +Copyright \(C) 2021 ungleich glarus ag. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest new file mode 100644 index 00000000..d110eea6 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/manifest @@ -0,0 +1,155 @@ +#!/bin/sh -eu + +__package haproxy +require="__package/haproxy" __start_on_boot haproxy + +tmpdir="$__object/files" +mkdir "$tmpdir" +configtmp="$__object/files/haproxy.cfg" + +os=$(cat "$__global/explorer/os") +case $os in + freebsd) + CONFIG_FILE="/usr/local/etc/haproxy.conf" + cat < "$configtmp" +global + maxconn 4000 + user nobody + group nogroup + daemon + +EOF + + ;; + *) + CONFIG_FILE="/etc/haproxy/haproxy.cfg" + cat < "$configtmp" +global + log [::1] local2 + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + daemon + + # turn on stats unix socket + stats socket /var/lib/haproxy/stats + +EOF + ;; +esac + +cat <> "$configtmp" +defaults + retries 3 + log global + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s +EOF + +dig_cmd="$(command -v dig || true)" +get_ip() { + # Usage: get_ip (ipv4|ipv6) NAME + # uses "dig" if available, else fallback to "host" + case $1 in + ipv4) + if [ -n "${dig_cmd}" ]; then + ${dig_cmd} +short A "$2" + else + host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:' + fi + ;; + ipv6) + if [ -n "${dig_cmd}" ]; then + ${dig_cmd} +short AAAA "$2" + else + host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN' + fi + ;; + esac +} + +PROTOCOLS="$(cat "$__object/parameter/protocol")" + +for proxy in v4proxy v6proxy; do + param=$__object/parameter/$proxy + # no backend? skip generating code + if [ ! -f "$param" ]; then + continue + fi + + # turn backend name into bind parameter: v4backend -> ipv4@ + bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//') + + case $bind in + ipv4) + backendproto=ipv6 + ;; + ipv6) + backendproto=ipv4 + ;; + esac + + for proto in ${PROTOCOLS}; do + # Add protocol "header" + printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp" + + sed -e "s/BIND/$bind/" \ + -e "s/\(frontend[[:space:]].*\)/\1$bind/" \ + -e "s/\(backend[[:space:]].*\)/\\1$bind/" \ + "$__type/files/$proto" >> "$configtmp" + + while read -r hostdefinition; do + if echo "$hostdefinition" | grep -qE '^proxy:'; then + # Proxy protocol was requested + host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')" + send_proxy=" send-proxy" + else + # Just use tcp proxy mode + host="$hostdefinition" + send_proxy="" + fi + if echo "$hostdefinition" | grep -qE ":${proto}="; then + # Use custom port definition if requested + port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")" + else + # Else use the default + port="" + fi + servername=$host + + res=$(get_ip "$bind" "$servername") + + if [ -z "$res" ]; then + echo "$servername does not resolve - aborting config" >&2 + exit 1 + fi + + # Treat protocols without TLS+SNI specially + if [ "$proto" = http ]; then + echo " use-server $servername if { hdr(host) -i $host }" >> "$configtmp" + else + echo " use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp" + fi + + # Create the "server" itself. + # Note that port and send_proxy will be empty unless + # they were requested by the type user + echo " server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp" + + done < "$param" + done +done + +# Create config file +require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644 + +require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \ + --pattern "^__file${CONFIG_FILE}" \ + --execute "service haproxy reload || service haproxy restart" diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol new file mode 100644 index 00000000..dc8bb7bf --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol @@ -0,0 +1 @@ +http https imaps smtps diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple new file mode 100644 index 00000000..8c482bd4 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple @@ -0,0 +1,3 @@ +protocol +v4proxy +v6proxy diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton new file mode 100644 index 00000000..e69de29b From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001 From: Evilham Date: Wed, 3 Nov 2021 11:03:33 +0100 Subject: [PATCH 38/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6f717cf4..99a8c08b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) + * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 16 Nov 2021 11:11:45 +0100 Subject: [PATCH 39/45] Python 3.10: collections.X -> collections.abc.X --- cdist/integration.py | 2 +- cdist/util/fsproperty.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/integration.py b/cdist/integration.py index 17b65f09..04470ea7 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose, """ if isinstance(host, str): hosts = [host, ] - elif isinstance(host, collections.Iterable): + elif isinstance(host, collections.abc.Iterable): hosts = host else: raise cdist.Error('Invalid host argument: {}'.format(host)) diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 09e9cc19..6bf935e8 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error): return 'Absolute path required, got: {}'.format(self.path) -class FileList(collections.MutableSequence): +class FileList(collections.abc.MutableSequence): """A list that stores it's state in a file. """ @@ -102,7 +102,7 @@ class FileList(collections.MutableSequence): self.__write(lines) -class DirectoryDict(collections.MutableMapping): +class DirectoryDict(collections.abc.MutableMapping): """A dict that stores it's items as files in a directory. """ From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:07:28 +0100 Subject: [PATCH 40/45] [__package_upgrade_all] Add new --apt-with-new-pkgs argument --- cdist/conf/type/__package_upgrade_all/gencode-remote | 6 +++++- cdist/conf/type/__package_upgrade_all/man.rst | 8 ++++++++ cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index 38aa001e..d332e851 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean" apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" +if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then + apt_with_new_pkgs="--with-new-pkgs" +fi + if [ -f "$type" ]; then type="$(cat "$type")" else @@ -54,7 +58,7 @@ case "$type" in apt) if [ -f "$apt_dist_upgrade" ] then echo "$aptget dist-upgrade" - else echo "$aptget upgrade" + else echo "$aptget $apt_with_new_pkgs upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index e9e2b8ce..0c116bac 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -33,6 +33,14 @@ BOOLEAN PARAMETERS apt-dist-upgrade Do dist-upgrade instead of upgrade. +apt-with-new-pkg + Allow installing new packages when used in conjunction with + upgrade. This is useful if the update of an installed package + requires new dependencies to be installed. Instead of holding the + package back upgrade will upgrade the package and install the new + dependencies. Note that upgrade with this option will never remove + packages, only allow adding new ones. + apt-clean Clean out the local repository of retrieved package files. diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean index 7a56a34b..cd22eb90 100644 --- a/cdist/conf/type/__package_upgrade_all/parameter/boolean +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -1,2 +1,3 @@ apt-clean apt-dist-upgrade +apt-with-new-pkgs From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:08:49 +0100 Subject: [PATCH 41/45] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 99a8c08b..26d89057 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Tue, 8 Mar 2022 12:04:58 +0100 Subject: [PATCH 42/45] Added rm of tmpfile. --- cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 61c77fb9..cbffde94 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -40,6 +40,7 @@ if [ -f "$file" ]; then grep -v -F -x '$line' '$file' >\$tmpfile fi cat "\$tmpfile" >"$file" +rm -f "\$tmpfile" DONE } From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 9 Mar 2022 16:16:44 +0100 Subject: [PATCH 43/45] ++changes --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 26d89057..81be51f6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,8 @@ next: * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) + * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) + * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 11:22:31 +0100 Subject: [PATCH 44/45] Fix typos; add default priority; comments in generated files --- cdist/conf/type/__apt_pin/manifest | 5 +++++ cdist/conf/type/__apt_pin/parameter/default/priority | 1 + cdist/conf/type/__apt_pin/parameter/optional | 1 + cdist/conf/type/__apt_pin/parameter/required | 1 - 4 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index e72a8fdd..983b2b42 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF +# Created by cdist ${__type##*/} +# Do not change. Changes will be overwritten. +# + +# $name Package: $package Pin: $pin Pin-Priority: $priority diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority new file mode 100644 index 00000000..1b79f38e --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/priority @@ -0,0 +1 @@ +500 diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional index 52f01fd2..847e703d 100644 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -1,2 +1,3 @@ state package +priority diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required index 4b4e9741..c8572d92 100644 --- a/cdist/conf/type/__apt_pin/parameter/required +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -1,2 +1 @@ distribution -priority From 22039284f57f575defa0bc65c46b8bbcbe016cd8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sun, 10 Apr 2022 23:52:53 +0200 Subject: [PATCH 45/45] __file: make file uploading and attribute changes more atomic Fixes https://code.ungleich.ch/ungleich-public/cdist/pulls/331 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++++-------- cdist/conf/type/__file/gencode-remote | 14 +++++++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 231b6927..bea3d79c 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -89,10 +89,26 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - tempfile_template="${destination}.cdist.XXXXXXXXXX" + upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + # Yes, we are aware that this is a race condition. + # However: + # a) cdist usually writes to directories that are not user writable + # (probably > 99.9%) + # b) if they are user owned, the user / attacker always wins + # (probably < 0.1%) + # c) the only case which we could improve are tmp directories and we + # don't think managing tmp directories with cdist is a typical case + # ("the rest %)" cat << DONE -destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" +$__remote_exec $__target_host test -e $upload_destination && { + echo "Refusing to upload file to existing destination: $upload_destination" >&2 + exit 1 +} DONE + # Tell gencode-remote to where we uploaded the file so it can move + # it to its final destination. + echo "$upload_destination" > "$__object/files/upload-destination" + if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix @@ -103,12 +119,8 @@ DONE my_target_host="${__target_host}" fi cat << DONE -$__remote_copy "$source" "${my_target_host}:\$destination_upload" +$__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE fi -# move uploaded file into place -cat << DONE -$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\"" -DONE fi fi diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index f7a528fd..136520a7 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -62,6 +62,13 @@ set_mode() { case "$state_should" in present|exists) + if [ -f "$__object/files/upload-destination" ]; then + final_destination="$destination" + # We change the 'global' $destination variable here so we can + # change attributes of the new/uploaded file before moving it + # to it's final destination. + destination="$(cat "$__object/files/upload-destination")" + fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -81,6 +88,11 @@ case "$state_should" in fi fi done + if [ -f "$__object/files/upload-destination" ]; then + # move uploaded file into place + printf 'rm -rf "%s"\n' "$final_destination" + printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local fire_onchange=1