Merge branch '__download_improvements' into 'master'

__download improvements

See merge request ungleich-public/cdist!895
This commit is contained in:
poljakowski 2020-07-08 12:33:41 +02:00
commit fde5627721
10 changed files with 173 additions and 27 deletions

View file

@ -0,0 +1,19 @@
#!/bin/sh -e
if [ -f "$__object/parameter/cmd-get" ]
then
cmd="$( cat "$__object/parameter/cmd-get" )"
elif command -v curl > /dev/null
then
cmd="curl -L -o - '%s'"
elif command -v fetch > /dev/null
then
cmd="fetch -o - '%s'"
else
cmd="wget -O - '%s'"
fi
echo "$cmd"

View file

@ -2,19 +2,71 @@
dst="/$__object_id" dst="/$__object_id"
# shellcheck disable=SC2059 if [ ! -f "$dst" ]
cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )"
sum="$( cat "$__object/parameter/sum" )"
if [ -f "$dst" ]
then then
if [ "$( eval "$cmd" )" = "$sum" ]
then
echo 'present'
else
echo 'mismatch'
fi
else
echo 'absent' echo 'absent'
exit 0
fi
sum_should="$( cat "$__object/parameter/sum" )"
if [ -f "$__object/parameter/cmd-sum" ]
then
# shellcheck disable=SC2059
sum_is="$( eval "$( printf \
"$( cat "$__object/parameter/cmd-sum" )" \
"$dst" )" )"
else
os="$( "$__explorer/os" )"
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
then
sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
then
case "$os" in
freebsd)
sum_is="md5:$( md5 -q "$dst" )"
;;
*)
sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
;;
esac
elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
then
case "$os" in
freebsd)
sum_is="sha1:$( sha1 -q "$dst" )"
;;
*)
sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
;;
esac
elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
then
case "$os" in
freebsd)
sum_is="sha256:$( sha256 -q "$dst" )"
;;
*)
sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
;;
esac
fi
fi
if [ -z "$sum_is" ]
then
echo 'no checksum from target' >&2
exit 1
fi
if [ "$sum_is" = "$sum_should" ]
then
echo 'present'
else
echo 'mismatch'
fi fi

View file

@ -1,20 +1,41 @@
#!/bin/sh -e #!/bin/sh -e
download="$( cat "$__object/parameter/download" )"
state_is="$( cat "$__object/explorer/state" )" state_is="$( cat "$__object/explorer/state" )"
if [ "$state_is" = 'present' ] if [ "$download" != 'local' ] || [ "$state_is" = 'present' ]
then then
exit 0 exit 0
fi fi
url="$( cat "$__object/parameter/url" )" url="$( cat "$__object/parameter/url" )"
cmd="$( cat "$__object/parameter/cmd-get" )"
tmp="$( mktemp )" tmp="$( mktemp )"
dst="/$__object_id" dst="/$__object_id"
if [ -f "$__object/parameter/cmd-get" ]
then
cmd="$( cat "$__object/parameter/cmd-get" )"
elif command -v wget > /dev/null
then
cmd="wget -O - '%s'"
elif command -v curl > /dev/null
then
cmd="curl -L -o - '%s'"
elif command -v fetch > /dev/null
then
cmd="fetch -o - '%s'"
else
echo 'no usable locally installed utility for downloading' >&2
exit 1
fi
printf "$cmd > %s\n" \ printf "$cmd > %s\n" \
"$url" \ "$url" \
"$tmp" "$tmp"
@ -33,3 +54,5 @@ printf '%s %s %s:%s\n' \
"$dst" "$dst"
echo "rm -f '$tmp'" echo "rm -f '$tmp'"
echo 'downloaded' > "$__messages_out"

View file

@ -0,0 +1,25 @@
#!/bin/sh -e
download="$( cat "$__object/parameter/download" )"
state_is="$( cat "$__object/explorer/state" )"
if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
then
cmd="$( cat "$__object/explorer/remote_cmd" )"
url="$( cat "$__object/parameter/url" )"
dst="/$__object_id"
printf "$cmd > %s\n" \
"$url" \
"$dst"
echo 'downloaded' > "$__messages_out"
fi
if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ]
then
cat "$__object/parameter/onchange"
fi

View file

@ -3,36 +3,56 @@ cdist-type__download(7)
NAME NAME
---- ----
cdist-type__download - Download file to local storage and copy it to target host cdist-type__download - Download a file
DESCRIPTION DESCRIPTION
----------- -----------
You must use persistent storage in target host for destination file Destination (``$__object_id``) in target host must be persistent storage
(``$__object_id``) because it will be used for checksum calculation in order to calculate checksum and decide if file must be (re-)downloaded.
in order to decide if file must be downloaded.
By default type will try to use ``wget``, ``curl`` or ``fetch``.
If download happens in target (see ``--download``) then type will
fallback to (and install) ``wget``.
If download happens in local machine, then environment variables like
``{http,https,ftp}_proxy`` etc can be used on cdist execution
(``http_proxy=foo cdist config ...``).
REQUIRED PARAMETERS REQUIRED PARAMETERS
------------------- -------------------
url url
URL from which to download the file. File's URL.
sum sum
Checksum of downloaded file. Checksum of file going to be downloaded.
By default output of ``cksum`` without filename is expected.
Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
onchange
Execute this command after download.
OPTIONAL PARAMETERS OPTIONAL PARAMETERS
------------------- -------------------
download
If ``local`` (default), then download file to local storage and copy
it to target host. If ``remote``, then download happens in target.
cmd-get cmd-get
Command used for downloading. Command used for downloading.
Default is ``wget -O- '%s'``.
Command must output to ``stdout``. Command must output to ``stdout``.
Parameter will be used for ``printf`` and must include only one
format specification ``%s`` which will become URL.
For example: ``wget -O - '%s'``.
cmd-sum cmd-sum
Command used for checksum calculation. Command used for checksum calculation.
Default is ``md5sum '%s' | awk '{print $1}'``.
Command output and ``--sum`` parameter must match. Command output and ``--sum`` parameter must match.
Parameter will be used for ``printf`` and must include only one
format specification ``%s`` which will become destination.
For example: ``md5sum '%s' | awk '{print $1}'``.
EXAMPLES EXAMPLES
@ -45,7 +65,7 @@ EXAMPLES
require='__directory/opt/cpma' \ require='__directory/opt/cpma' \
__download /opt/cpma/cnq3.zip \ __download /opt/cpma/cnq3.zip \
--url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
--sum 46da3021ca9eace277115ec9106c5b46 --sum md5:46da3021ca9eace277115ec9106c5b46
require='__download/opt/cpma/cnq3.zip' \ require='__download/opt/cpma/cnq3.zip' \
__unpack /opt/cpma/cnq3.zip \ __unpack /opt/cpma/cnq3.zip \

View file

@ -0,0 +1,6 @@
#!/bin/sh -e
if grep -Eq '^wget' "$__object/explorer/remote_cmd"
then
__package wget
fi

View file

@ -1 +0,0 @@
wget -O- '%s'

View file

@ -1 +0,0 @@
md5sum '%s' | awk '{print $1}'

View file

@ -0,0 +1 @@
local

View file

@ -1,2 +1,4 @@
cmd-get cmd-get
cmd-sum cmd-sum
download
onchange