From 4f7ae8425d6ce13653daca15b3339889bd707835 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 24 Feb 2015 13:01:48 +0100 Subject: [PATCH] New type __staged_file: Manage staged files Signed-off-by: Steven Armstrong --- cdist/conf/type/__staged_file/gencode-local | 98 +++++++++++++++++ cdist/conf/type/__staged_file/man.text | 103 ++++++++++++++++++ cdist/conf/type/__staged_file/manifest | 38 +++++++ .../parameter/default/fetch-command | 1 + .../__staged_file/parameter/default/stage-dir | 1 + .../__staged_file/parameter/default/state | 1 + .../type/__staged_file/parameter/optional | 7 ++ .../type/__staged_file/parameter/required | 2 + docs/changelog | 1 + 9 files changed, 252 insertions(+) create mode 100755 cdist/conf/type/__staged_file/gencode-local create mode 100644 cdist/conf/type/__staged_file/man.text create mode 100755 cdist/conf/type/__staged_file/manifest create mode 100644 cdist/conf/type/__staged_file/parameter/default/fetch-command create mode 100644 cdist/conf/type/__staged_file/parameter/default/stage-dir create mode 100644 cdist/conf/type/__staged_file/parameter/default/state create mode 100644 cdist/conf/type/__staged_file/parameter/optional create mode 100644 cdist/conf/type/__staged_file/parameter/required diff --git a/cdist/conf/type/__staged_file/gencode-local b/cdist/conf/type/__staged_file/gencode-local new file mode 100755 index 00000000..87d79f4c --- /dev/null +++ b/cdist/conf/type/__staged_file/gencode-local @@ -0,0 +1,98 @@ +#!/bin/sh +# +# 2015 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +#set -x + +destination="$__object_id" +source="$(cat "$__object/parameter/source")" +cksum="$(cat "$__object/parameter/cksum")" +stage_dir="$(cat "$__object/parameter/stage-dir")" +state="$(cat "$__object/parameter/state")" +fetch_command="$(cat "$__object/parameter/fetch-command")" +stage_file="${stage_dir}/${destination}" +stage_file_dir="${stage_file%/*}" +source_file_name="${source##*/}" + +if [ "$state" = "absent" ]; then + # nothing to do + exit 0 +fi + +#printf 'set -x\n' + +if [ ! -d "$stage_dir" ]; then + printf 'mkdir -p "%s"\n' "$stage_dir" + printf 'chmod 700 "%s"\n' "$stage_dir" +fi + +if [ ! -d "$stage_file_dir" ]; then + printf 'mkdir -p "%s"\n' "$stage_file_dir" +fi + + +get_file() { + if [ -f "$__object/parameter/prepare-command" ]; then + fetch_and_prepare_file + else + fetch_file + fi +} + +fetch_file() { + printf "$fetch_command" "$source" + printf ' > "%s"\n' "$stage_file" +} + +fetch_and_prepare_file() { + printf 'tmpdir="$(mktemp -d --tmpdir="/tmp" "%s")"\n' "${__type##*/}.XXXXXXXXXX" + printf 'cd "$tmpdir"\n' + printf "$fetch_command > \"%s\"\n" "$source" "$source_file_name" + prepare_command="$(cat "$__object/parameter/prepare-command")" + printf "$prepare_command > \"%s\"\n" "$source_file_name" "$stage_file" + printf 'cd - >/dev/null\n' + printf 'rm -rf "$tmpdir"\n' +} + +cat << DONE +verify_cksum() { + cksum_is="\$(cksum "$stage_file" | cut -d' ' -f1,2)" + cksum_should="$(cat "$__object/parameter/cksum" | cut -d' ' -f1,2)" + if [ "\$cksum_is" == "\$cksum_should" ]; then + return 0 + else + return 1 + fi +} +DONE + +if [ ! -f "$stage_file" ]; then + get_file +else + printf 'verify_cksum || {\n' + get_file + printf '}\n' +fi + +cat << DONE +verify_cksum || { + echo "Failed to verify checksum for $__object_name" >&2 + exit 1 +} +DONE diff --git a/cdist/conf/type/__staged_file/man.text b/cdist/conf/type/__staged_file/man.text new file mode 100644 index 00000000..e94e491d --- /dev/null +++ b/cdist/conf/type/__staged_file/man.text @@ -0,0 +1,103 @@ +cdist-type__staged_file(7) +========================== +Steven Armstrong + + +NAME +---- +cdist-type__staged_file - manage staged files + + +DESCRIPTION +----------- +Manages a staged file that is downloaded on the server (the machine running +cdist) and then deployed to the target host using the __file type. + + +REQUIRED PARAMETERS +------------------- +source:: + the URL from which to retreive the source file. + e.g. + https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip + file:///path/to/local/file +cksum:: + the output of running the command: `cksum $source-file` + e.g. + $ echo foobar > /tmp/foobar + $ cksum /tmp/foobar + 857691210 7 /tmp/foobar + If either checksum or file size has changed the file will be + (re)fetched from the --source. The file name can be omitted and is + ignored if given. + + +OPTIONAL PARAMETERS +------------------- +fetch-command:: + the command used to fetch the staged file using printf formatting. + Where a single %s will be replaced with the value of the given --source + parameter. The --fetch-command is expected to output the fetched file to + stdout. + Defaults to 'curl -s -L "%s"'. +group:: + see cdist-type__file +owner:: + see cdist-type__file +mode:: + see cdist-type__file +prepare-command:: + the optional command used to prepare or preprocess the staged file for later + use by the file type. + If given, it must be a string in printf formatting where a single %s will + be replaced with the last segment (filename) of the value of the given + --source parameter. + It is executed in the same directory into which the fetched file has been + saved. The --prepare-command is expected to output the final file to stdout. + + So for example given a --source of https://example.com/my-zip.zip, and a + --prepare-command of 'unzip -p "%s"', the code `unzip -p "my-zip.zip"` will + be executed in the folder containing the downloaded file my-zip.zip. + A more complex example might be --prepare-command 'tar -xz "%s"; cat path/from/archive' +stage-dir:: + the directory in which to store downloaded and prepared files. + Defaults to '/var/tmp/cdist/__staged_file' +state:: + see cdist-type__file + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__staged_file /usr/local/bin/consul \ + --source file:///path/to/local/copy/consul \ + --cksum '428915666 15738724' \ + --state present \ + --group root \ + --owner root \ + --mode 755 + +__staged_file /usr/local/bin/consul \ + --source https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip \ + --cksum '428915666 15738724' \ + --fetch-command 'curl -s -L "%s"' \ + --prepare-command 'unzip -p "%s"' \ + --state present \ + --group root \ + --owner root \ + --mode 755 + +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__file(7) + + +COPYING +------- +Copyright \(C) 2015 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__staged_file/manifest b/cdist/conf/type/__staged_file/manifest new file mode 100755 index 00000000..454948b4 --- /dev/null +++ b/cdist/conf/type/__staged_file/manifest @@ -0,0 +1,38 @@ +#!/bin/sh +# +# 2015 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="$__object_id" +source="$(cat "$__object/parameter/source")" +cksum="$(cat "$__object/parameter/cksum")" +stage_dir="$(cat "$__object/parameter/stage-dir")" +state="$(cat "$__object/parameter/state")" +fetch_command="$(cat "$__object/parameter/fetch-command")" +stage_file="${stage_dir}/${destination}" + +set -- "/${destination}" +for param in owner group mode state; do + if [ -f "$__object/parameter/$param" ]; then + set -- "$@" "--${param}" "$(cat "$__object/parameter/$param")" + fi +done +set -- "$@" --source "$stage_file" + +require="$__object_name" \ + __file "$@" diff --git a/cdist/conf/type/__staged_file/parameter/default/fetch-command b/cdist/conf/type/__staged_file/parameter/default/fetch-command new file mode 100644 index 00000000..b4dc1211 --- /dev/null +++ b/cdist/conf/type/__staged_file/parameter/default/fetch-command @@ -0,0 +1 @@ +curl -s -L "%s" diff --git a/cdist/conf/type/__staged_file/parameter/default/stage-dir b/cdist/conf/type/__staged_file/parameter/default/stage-dir new file mode 100644 index 00000000..9420b510 --- /dev/null +++ b/cdist/conf/type/__staged_file/parameter/default/stage-dir @@ -0,0 +1 @@ +/var/tmp/cdist/__staged_file diff --git a/cdist/conf/type/__staged_file/parameter/default/state b/cdist/conf/type/__staged_file/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__staged_file/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__staged_file/parameter/optional b/cdist/conf/type/__staged_file/parameter/optional new file mode 100644 index 00000000..18f4e87a --- /dev/null +++ b/cdist/conf/type/__staged_file/parameter/optional @@ -0,0 +1,7 @@ +fetch-command +group +owner +mode +prepare-command +stage-dir +state diff --git a/cdist/conf/type/__staged_file/parameter/required b/cdist/conf/type/__staged_file/parameter/required new file mode 100644 index 00000000..bfb1d5bf --- /dev/null +++ b/cdist/conf/type/__staged_file/parameter/required @@ -0,0 +1,2 @@ +cksum +source diff --git a/docs/changelog b/docs/changelog index 62095c27..9017f526 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ Changelog * Exception: No braces means author == Nico Schottelius next: + * New type __staged_file: Manage staged files * New type __config_file: Manage configuration files and run code on change 3.1.11: