[__download] improvements

1. post download checksum verification
2. detect hashes without prefix
3. add optional --destination
4. updated man
This commit is contained in:
ander 2021-06-22 16:36:30 +03:00
parent c308a28969
commit 7b3f268df2
No known key found for this signature in database
GPG key ID: 0A2971E562D618F3
9 changed files with 292 additions and 98 deletions

View file

@ -1,19 +0,0 @@
#!/bin/sh -e
if [ -f "$__object/parameter/cmd-get" ]
then
cmd="$( cat "$__object/parameter/cmd-get" )"
elif command -v curl > /dev/null
then
cmd="curl -L -o - '%s'"
elif command -v fetch > /dev/null
then
cmd="fetch -o - '%s'"
else
cmd="wget -O - '%s'"
fi
echo "$cmd"

View file

@ -0,0 +1,16 @@
#!/bin/sh -e
if [ -f "$__object/parameter/cmd-get" ]
then
cat "$__object/parameter/cmd-get"
elif
command -v curl > /dev/null
then
echo "curl -sSL -o - '%s'"
elif
command -v fetch > /dev/null
then
echo "fetch -o - '%s'"
else
echo "wget -O - '%s'"
fi

View file

@ -0,0 +1,82 @@
#!/bin/sh -e
if [ ! -f "$__object/parameter/sum" ]
then
exit 0
fi
if [ -f "$__object/parameter/cmd-sum" ]
then
cat "$__object/parameter/cmd-sum"
exit 0
fi
sum_should="$( cat "$__object/parameter/sum" )"
if echo "$sum_should" | grep -Fq ':'
then
sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
else
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
then
sum_hash='cksum'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
then
sum_hash='md5'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
then
sum_hash='sha1'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
then
sum_hash='sha256'
else
echo 'hash format detection failed' >&2
exit 1
fi
fi
os="$( "$__explorer/os" )"
case "$sum_hash" in
cksum)
echo "cksum %s | awk '{print \$1\" \"\$2}'"
;;
md5)
case "$os" in
freebsd)
echo "md5 -q %s"
;;
*)
echo "md5sum %s | awk '{print \$1}'"
;;
esac
;;
sha1)
case "$os" in
freebsd)
echo "sha1 -q %s"
;;
*)
echo "sha1sum %s | awk '{print \$1}'"
;;
esac
;;
sha256)
case "$os" in
freebsd)
echo "sha256 -q %s"
;;
*)
echo "sha256sum %s | awk '{print \$1}'"
;;
esac
;;
*)
# we arrive here only if --sum is given with unknown format prefix
echo "unknown hash format: $sum_hash" >&2
exit 1
;;
esac

View file

@ -1,6 +1,11 @@
#!/bin/sh -e #!/bin/sh -e
dst="/$__object_id" if [ -f "$__object/parameter/destination" ]
then
dst="$( cat "$__object/parameter/destination" )"
else
dst="/$__object_id"
fi
if [ ! -f "$dst" ] if [ ! -f "$dst" ]
then then
@ -16,57 +21,18 @@ fi
sum_should="$( cat "$__object/parameter/sum" )" sum_should="$( cat "$__object/parameter/sum" )"
if [ -f "$__object/parameter/cmd-sum" ] if echo "$sum_should" | grep -Fq ':'
then then
# shellcheck disable=SC2059 sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
sum_is="$( eval "$( printf \
"$( cat "$__object/parameter/cmd-sum" )" \
"$dst" )" )"
else
os="$( "$__explorer/os" )"
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
then
sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
then
case "$os" in
freebsd)
sum_is="md5:$( md5 -q "$dst" )"
;;
*)
sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
;;
esac
elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
then
case "$os" in
freebsd)
sum_is="sha1:$( sha1 -q "$dst" )"
;;
*)
sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
;;
esac
elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
then
case "$os" in
freebsd)
sum_is="sha256:$( sha256 -q "$dst" )"
;;
*)
sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
;;
esac
fi
fi fi
sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
if [ -z "$sum_is" ] if [ -z "$sum_is" ]
then then
echo 'no checksum from target' >&2 echo 'existing destination checksum failed' >&2
exit 1 exit 1
fi fi

View file

@ -11,34 +11,133 @@ fi
url="$( cat "$__object/parameter/url" )" url="$( cat "$__object/parameter/url" )"
tmp="$( mktemp )" if [ -f "$__object/parameter/destination" ]
then
dst="/$__object_id" dst="$( cat "$__object/parameter/destination" )"
else
dst="/$__object_id"
fi
if [ -f "$__object/parameter/cmd-get" ] if [ -f "$__object/parameter/cmd-get" ]
then then
cmd="$( cat "$__object/parameter/cmd-get" )" cmd="$( cat "$__object/parameter/cmd-get" )"
elif command -v wget > /dev/null
then
cmd="wget -O - '%s'"
elif command -v curl > /dev/null elif command -v curl > /dev/null
then then
cmd="curl -L -o - '%s'" cmd="curl -sSL -o - '%s'"
elif command -v fetch > /dev/null elif command -v fetch > /dev/null
then then
cmd="fetch -o - '%s'" cmd="fetch -o - '%s'"
elif command -v wget > /dev/null
then
cmd="wget -O - '%s'"
else else
echo 'no usable locally installed utility for downloading' >&2 echo 'local download failed, no usable utility' >&2
exit 1 exit 1
fi fi
printf "$cmd > %s\n" \ echo "download_tmp=\"\$( mktemp )\""
"$url" \
"$tmp" # shellcheck disable=SC2059
printf "$cmd > \"\$download_tmp\"\n" "$url"
if [ -f "$__object/parameter/sum" ]
then
sum_should="$( cat "$__object/parameter/sum" )"
if [ -f "$__object/parameter/cmd-sum" ]
then
local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
else
if echo "$sum_should" | grep -Fq ':'
then
sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
else
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
then
sum_hash='cksum'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
then
sum_hash='md5'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
then
sum_hash='sha1'
elif
echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
then
sum_hash='sha256'
else
echo 'hash format detection failed' >&2
exit 1
fi
fi
case "$sum_hash" in
cksum)
local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
;;
md5)
if command -v md5 > /dev/null
then
local_cmd_sum="md5 -q %s"
elif
command -v md5sum > /dev/null
then
local_cmd_sum="md5sum %s | awk '{print \$1}'"
fi
;;
sha1)
if command -v sha1 > /dev/null
then
local_cmd_sum="sha1 -q %s"
elif
command -v sha1sum > /dev/null
then
local_cmd_sum="sha1sum %s | awk '{print \$1}'"
fi
;;
sha256)
if command -v sha256 > /dev/null
then
local_cmd_sum="sha256 -q %s"
elif
command -v sha256sum > /dev/null
then
local_cmd_sum="sha256sum %s | awk '{print \$1}'"
fi
;;
*)
# we arrive here only if --sum is given with unknown format prefix
echo "unknown hash format: $sum_hash" >&2
exit 1
;;
esac
if [ -z "$local_cmd_sum" ]
then
echo 'local checksum verification failed, no usable utility' >&2
exit 1
fi
fi
# shellcheck disable=SC2059
echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
echo "echo 'local download checksum mismatch' >&2"
echo "rm -f \"\$download_tmp\""
echo 'exit 1; fi'
fi
if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
then then
@ -47,12 +146,10 @@ else
target_host="$__target_host" target_host="$__target_host"
fi fi
printf '%s %s %s:%s\n' \ # shellcheck disable=SC2016
printf '%s "$download_tmp" %s:%s\n' \
"$__remote_copy" \ "$__remote_copy" \
"$tmp" \
"$target_host" \ "$target_host" \
"$dst" "$dst"
echo "rm -f '$tmp'" echo "rm -f \"\$download_tmp\""
echo 'downloaded' > "$__messages_out"

View file

@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
then then
cmd="$( cat "$__object/explorer/remote_cmd" )" cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
url="$( cat "$__object/parameter/url" )" url="$( cat "$__object/parameter/url" )"
if [ -f "$__object/parameter/destination" ]
then
dst="$( cat "$__object/parameter/destination" )"
else
dst="/$__object_id" dst="/$__object_id"
fi
printf "$cmd > %s\n" \ echo "download_tmp=\"\$( mktemp )\""
"$url" \
"$dst"
echo 'downloaded' > "$__messages_out" # shellcheck disable=SC2059
printf "$cmd_get > \"\$download_tmp\"\n" "$url"
if [ -f "$__object/parameter/sum" ]
then
sum_should="$( cat "$__object/parameter/sum" )"
if [ -f "$__object/parameter/cmd-sum" ]
then
remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
else
remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
if echo "$sum_should" | grep -Fq ':'
then
sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
fi
fi
# shellcheck disable=SC2059
echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
echo "echo 'remote download checksum mismatch' >&2"
echo "rm -f \"\$download_tmp\""
echo 'exit 1; fi'
fi
echo "mv \"\$download_tmp\" '$dst'"
fi fi
if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]

View file

@ -8,7 +8,7 @@ cdist-type__download - Download a file
DESCRIPTION DESCRIPTION
----------- -----------
By default type will try to use ``wget``, ``curl`` or ``fetch``. By default type will try to use ``curl``, ``fetch`` or ``wget``.
If download happens in target (see ``--download``) then type will If download happens in target (see ``--download``) then type will
fallback to (and install) ``wget``. fallback to (and install) ``wget``.
@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like
``{http,https,ftp}_proxy`` etc can be used on cdist execution ``{http,https,ftp}_proxy`` etc can be used on cdist execution
(``http_proxy=foo cdist config ...``). (``http_proxy=foo cdist config ...``).
To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
REQUIRED PARAMETERS REQUIRED PARAMETERS
------------------- -------------------
@ -25,14 +27,29 @@ url
OPTIONAL PARAMETERS OPTIONAL PARAMETERS
------------------- -------------------
destination
Downloaded file's destination in target. If unset, ``$__object_id`` is used.
sum sum
Checksum is used to decide if existing destination file must be redownloaded. Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
By default output of ``cksum`` without filename is expected.
Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. Type tries to detect hash format with regexes, but prefixes
``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
Checksum have two purposes - state check and post-download verification.
In state check, if destination checksum mismatches, then content of URL
will be downloaded to temporary file. If downloaded temporary file's
checksum matches, then it will be moved to destination (overwritten).
For local downloads it is expected that usable utilities for checksum
calculation exist in the system.
download download
If ``local`` (default), then download file to local storage and copy If ``local`` (default), then file is downloaded to local storage and copied
it to target host. If ``remote``, then download happens in target. to target host. If ``remote``, then download happens in target.
For local downloads it is expected that usable utilities for downloading
exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
cmd-get cmd-get
Command used for downloading. Command used for downloading.
@ -62,7 +79,7 @@ EXAMPLES
require='__directory/opt/cpma' \ require='__directory/opt/cpma' \
__download /opt/cpma/cnq3.zip \ __download /opt/cpma/cnq3.zip \
--url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
--sum md5:46da3021ca9eace277115ec9106c5b46 --sum 46da3021ca9eace277115ec9106c5b46
require='__download/opt/cpma/cnq3.zip' \ require='__download/opt/cpma/cnq3.zip' \
__unpack /opt/cpma/cnq3.zip \ __unpack /opt/cpma/cnq3.zip \

View file

@ -1,6 +1,6 @@
#!/bin/sh -e #!/bin/sh -e
if grep -Eq '^wget' "$__object/explorer/remote_cmd" if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
then then
__package wget __package wget
fi fi

View file

@ -1,5 +1,6 @@
sum
cmd-get cmd-get
cmd-sum cmd-sum
destination
download download
onchange onchange
sum