matze/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[__download] improvements

Browse source 
1. post download checksum verification
2. detect hashes without prefix
3. add optional --destination
4. updated man
This commit is contained in:
ander 2021-06-22 16:36:30 +03:00
parent c308a28969
commit 7b3f268df2
No known key found for this signature in database
GPG key ID: 0A2971E562D618F3
9 changed files with 292 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
cdist/conf/type/__download/explorer/remote_cmd
View file

@ -1,19 +0,0 @@
#!/bin/sh -e
if [ -f "$__object/parameter/cmd-get" ]
then
cmd="$( cat "$__object/parameter/cmd-get" )"
elif command -v curl > /dev/null
then
cmd="curl -L -o - '%s'"
elif command -v fetch > /dev/null
then
cmd="fetch -o - '%s'"
else
cmd="wget -O - '%s'"
fi
echo "$cmd"

16
cdist/conf/type/__download/explorer/remote_cmd_get Executable file
View file

@ -0,0 +1,16 @@
#!/bin/sh -e
if [ -f "$__object/parameter/cmd-get" ]
then
cat "$__object/parameter/cmd-get"
elif
command -v curl > /dev/null
then
echo "curl -sSL -o - '%s'"
elif
command -v fetch > /dev/null
then
echo "fetch -o - '%s'"
else
echo "wget -O - '%s'"
fi

82
cdist/conf/type/__download/explorer/remote_cmd_sum Executable file
View file

@ -0,0 +1,82 @@
#!/bin/sh -e
if [ ! -f "$__object/parameter/sum" ]
then
exit 0
fi
if [ -f "$__object/parameter/cmd-sum" ]
then
cat "$__object/parameter/cmd-sum"
exit 0
fi
sum_should="$( cat "$__object/parameter/sum" )"
if echo "$sum_should" | grep -Fq ':'
then
sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
else
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
then
sum_hash='cksum'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
then
sum_hash='md5'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
then
sum_hash='sha1'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
then
sum_hash='sha256'
else
echo 'hash format detection failed' >&2
exit 1
fi
fi
os="$( "$__explorer/os" )"
case "$sum_hash" in
cksum)
echo "cksum %s | awk '{print \$1\" \"\$2}'"
;;
md5)
case "$os" in
freebsd)
echo "md5 -q %s"
;;
*)
echo "md5sum %s | awk '{print \$1}'"
;;
esac
;;
sha1)
case "$os" in
freebsd)
echo "sha1 -q %s"
;;
*)
echo "sha1sum %s | awk '{print \$1}'"
;;
esac
;;
sha256)
case "$os" in
freebsd)
echo "sha256 -q %s"
;;
*)
echo "sha256sum %s | awk '{print \$1}'"
;;
esac
;;
*)
# we arrive here only if --sum is given with unknown format prefix
echo "unknown hash format: $sum_hash" >&2
exit 1
;;
esac

60
cdist/conf/type/__download/explorer/state
View file

@ -1,6 +1,11 @@
#!/bin/sh -e
dst="/$__object_id"
if [ -f "$__object/parameter/destination" ]
then
dst="$( cat "$__object/parameter/destination" )"
else
dst="/$__object_id"
fi
if [ ! -f "$dst" ]
then
@ -16,57 +21,18 @@ fi
sum_should="$( cat "$__object/parameter/sum" )"
if [ -f "$__object/parameter/cmd-sum" ]
if echo "$sum_should" | grep -Fq ':'
then
# shellcheck disable=SC2059
sum_is="$( eval "$( printf \
"$( cat "$__object/parameter/cmd-sum" )" \
"$dst" )" )"
else
os="$( "$__explorer/os" )"
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
then
sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
then
case "$os" in
freebsd)
sum_is="md5:$( md5 -q "$dst" )"
;;
*)
sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
;;
esac
elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
then
case "$os" in
freebsd)
sum_is="sha1:$( sha1 -q "$dst" )"
;;
*)
sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
;;
esac
elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
then
case "$os" in
freebsd)
sum_is="sha256:$( sha256 -q "$dst" )"
;;
*)
sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
;;
esac
fi
sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
fi
sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
if [ -z "$sum_is" ]
then
echo 'no checksum from target' >&2
echo 'existing destination checksum failed' >&2
exit 1
fi

131
cdist/conf/type/__download/gencode-local
View file

@ -11,34 +11,133 @@ fi
url="$( cat "$__object/parameter/url" )"
tmp="$( mktemp )"
dst="/$__object_id"
if [ -f "$__object/parameter/destination" ]
then
dst="$( cat "$__object/parameter/destination" )"
else
dst="/$__object_id"
fi
if [ -f "$__object/parameter/cmd-get" ]
then
cmd="$( cat "$__object/parameter/cmd-get" )"
elif command -v wget > /dev/null
then
cmd="wget -O - '%s'"
elif command -v curl > /dev/null
then
cmd="curl -L -o - '%s'"
cmd="curl -sSL -o - '%s'"
elif command -v fetch > /dev/null
then
cmd="fetch -o - '%s'"
elif command -v wget > /dev/null
then
cmd="wget -O - '%s'"
else
echo 'no usable locally installed utility for downloading' >&2
echo 'local download failed, no usable utility' >&2
exit 1
fi
printf "$cmd > %s\n" \
"$url" \
"$tmp"
echo "download_tmp=\"\$( mktemp )\""
# shellcheck disable=SC2059
printf "$cmd > \"\$download_tmp\"\n" "$url"
if [ -f "$__object/parameter/sum" ]
then
sum_should="$( cat "$__object/parameter/sum" )"
if [ -f "$__object/parameter/cmd-sum" ]
then
local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
else
if echo "$sum_should" | grep -Fq ':'
then
sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
else
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
then
sum_hash='cksum'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
then
sum_hash='md5'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
then
sum_hash='sha1'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
then
sum_hash='sha256'
else
echo 'hash format detection failed' >&2
exit 1
fi
fi
case "$sum_hash" in
cksum)
local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
;;
md5)
if command -v md5 > /dev/null
then
local_cmd_sum="md5 -q %s"
elif
command -v md5sum > /dev/null
then
local_cmd_sum="md5sum %s | awk '{print \$1}'"
fi
;;
sha1)
if command -v sha1 > /dev/null
then
local_cmd_sum="sha1 -q %s"
elif
command -v sha1sum > /dev/null
then
local_cmd_sum="sha1sum %s | awk '{print \$1}'"
fi
;;
sha256)
if command -v sha256 > /dev/null
then
local_cmd_sum="sha256 -q %s"
elif
command -v sha256sum > /dev/null
then
local_cmd_sum="sha256sum %s | awk '{print \$1}'"
fi
;;
*)
# we arrive here only if --sum is given with unknown format prefix
echo "unknown hash format: $sum_hash" >&2
exit 1
;;
esac
if [ -z "$local_cmd_sum" ]
then
echo 'local checksum verification failed, no usable utility' >&2
exit 1
fi
fi
# shellcheck disable=SC2059
echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
echo "echo 'local download checksum mismatch' >&2"
echo "rm -f \"\$download_tmp\""
echo 'exit 1; fi'
fi
if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
then
@ -47,12 +146,10 @@ else
target_host="$__target_host"
fi
printf '%s %s %s:%s\n' \
# shellcheck disable=SC2016
printf '%s "$download_tmp" %s:%s\n' \
"$__remote_copy" \
"$tmp" \
"$target_host" \
"$dst"
echo "rm -f '$tmp'"
echo 'downloaded' > "$__messages_out"
echo "rm -f \"\$download_tmp\""

46
cdist/conf/type/__download/gencode-remote
View file

@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
then
cmd="$( cat "$__object/explorer/remote_cmd" )"
cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
url="$( cat "$__object/parameter/url" )"
dst="/$__object_id"
if [ -f "$__object/parameter/destination" ]
then
dst="$( cat "$__object/parameter/destination" )"
else
dst="/$__object_id"
fi
printf "$cmd > %s\n" \
"$url" \
"$dst"
echo "download_tmp=\"\$( mktemp )\""
echo 'downloaded' > "$__messages_out"
# shellcheck disable=SC2059
printf "$cmd_get > \"\$download_tmp\"\n" "$url"
if [ -f "$__object/parameter/sum" ]
then
sum_should="$( cat "$__object/parameter/sum" )"
if [ -f "$__object/parameter/cmd-sum" ]
then
remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
else
remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
if echo "$sum_should" | grep -Fq ':'
then
sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
fi
fi
# shellcheck disable=SC2059
echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
echo "echo 'remote download checksum mismatch' >&2"
echo "rm -f \"\$download_tmp\""
echo 'exit 1; fi'
fi
echo "mv \"\$download_tmp\" '$dst'"
fi
if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]

31
cdist/conf/type/__download/man.rst
View file

@ -8,7 +8,7 @@ cdist-type__download - Download a file
DESCRIPTION
-----------
By default type will try to use ``wget``, ``curl`` or ``fetch``.
By default type will try to use ``curl``, ``fetch`` or ``wget``.
If download happens in target (see ``--download``) then type will
fallback to (and install) ``wget``.
@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like
``{http,https,ftp}_proxy`` etc can be used on cdist execution
(``http_proxy=foo cdist config ...``).
To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
REQUIRED PARAMETERS
-------------------
@ -25,14 +27,29 @@ url
OPTIONAL PARAMETERS
-------------------
destination
Downloaded file's destination in target. If unset, ``$__object_id`` is used.
sum
Checksum is used to decide if existing destination file must be redownloaded.
By default output of ``cksum`` without filename is expected.
Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
Type tries to detect hash format with regexes, but prefixes
``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
Checksum have two purposes - state check and post-download verification.
In state check, if destination checksum mismatches, then content of URL
will be downloaded to temporary file. If downloaded temporary file's
checksum matches, then it will be moved to destination (overwritten).
For local downloads it is expected that usable utilities for checksum
calculation exist in the system.
download
If ``local`` (default), then download file to local storage and copy
it to target host. If ``remote``, then download happens in target.
If ``local`` (default), then file is downloaded to local storage and copied
to target host. If ``remote``, then download happens in target.
For local downloads it is expected that usable utilities for downloading
exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
cmd-get
Command used for downloading.
@ -62,7 +79,7 @@ EXAMPLES
require='__directory/opt/cpma' \
__download /opt/cpma/cnq3.zip \
--url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
--sum md5:46da3021ca9eace277115ec9106c5b46
--sum 46da3021ca9eace277115ec9106c5b46
require='__download/opt/cpma/cnq3.zip' \
__unpack /opt/cpma/cnq3.zip \

2
cdist/conf/type/__download/manifest
View file

@ -1,6 +1,6 @@
#!/bin/sh -e
if grep -Eq '^wget' "$__object/explorer/remote_cmd"
if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
then
__package wget
fi

3
cdist/conf/type/__download/parameter/optional
View file

@ -1,5 +1,6 @@
sum
cmd-get
cmd-sum
destination
download
onchange
sum