forked from ungleich-public/cdist
Merge branch '__download_improvements' into 'master'
[__download] improvements See merge request ungleich-public/cdist!1003
This commit is contained in:
commit
99188b4822
9 changed files with 293 additions and 98 deletions
|
@ -1,19 +0,0 @@
|
|||
#!/bin/sh -e
|
||||
|
||||
if [ -f "$__object/parameter/cmd-get" ]
|
||||
then
|
||||
cmd="$( cat "$__object/parameter/cmd-get" )"
|
||||
|
||||
elif command -v curl > /dev/null
|
||||
then
|
||||
cmd="curl -L -o - '%s'"
|
||||
|
||||
elif command -v fetch > /dev/null
|
||||
then
|
||||
cmd="fetch -o - '%s'"
|
||||
|
||||
else
|
||||
cmd="wget -O - '%s'"
|
||||
fi
|
||||
|
||||
echo "$cmd"
|
16
cdist/conf/type/__download/explorer/remote_cmd_get
Executable file
16
cdist/conf/type/__download/explorer/remote_cmd_get
Executable file
|
@ -0,0 +1,16 @@
|
|||
#!/bin/sh -e
|
||||
|
||||
if [ -f "$__object/parameter/cmd-get" ]
|
||||
then
|
||||
cat "$__object/parameter/cmd-get"
|
||||
elif
|
||||
command -v curl > /dev/null
|
||||
then
|
||||
echo "curl -sSL -o - '%s'"
|
||||
elif
|
||||
command -v fetch > /dev/null
|
||||
then
|
||||
echo "fetch -o - '%s'"
|
||||
else
|
||||
echo "wget -O - '%s'"
|
||||
fi
|
82
cdist/conf/type/__download/explorer/remote_cmd_sum
Executable file
82
cdist/conf/type/__download/explorer/remote_cmd_sum
Executable file
|
@ -0,0 +1,82 @@
|
|||
#!/bin/sh -e
|
||||
|
||||
if [ ! -f "$__object/parameter/sum" ]
|
||||
then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -f "$__object/parameter/cmd-sum" ]
|
||||
then
|
||||
cat "$__object/parameter/cmd-sum"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
sum_should="$( cat "$__object/parameter/sum" )"
|
||||
|
||||
if echo "$sum_should" | grep -Fq ':'
|
||||
then
|
||||
sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
|
||||
else
|
||||
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
|
||||
then
|
||||
sum_hash='cksum'
|
||||
elif
|
||||
echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
|
||||
then
|
||||
sum_hash='md5'
|
||||
elif
|
||||
echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
|
||||
then
|
||||
sum_hash='sha1'
|
||||
elif
|
||||
echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
|
||||
then
|
||||
sum_hash='sha256'
|
||||
else
|
||||
echo 'hash format detection failed' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
os="$( "$__explorer/os" )"
|
||||
|
||||
case "$sum_hash" in
|
||||
cksum)
|
||||
echo "cksum %s | awk '{print \$1\" \"\$2}'"
|
||||
;;
|
||||
md5)
|
||||
case "$os" in
|
||||
freebsd)
|
||||
echo "md5 -q %s"
|
||||
;;
|
||||
*)
|
||||
echo "md5sum %s | awk '{print \$1}'"
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
sha1)
|
||||
case "$os" in
|
||||
freebsd)
|
||||
echo "sha1 -q %s"
|
||||
;;
|
||||
*)
|
||||
echo "sha1sum %s | awk '{print \$1}'"
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
sha256)
|
||||
case "$os" in
|
||||
freebsd)
|
||||
echo "sha256 -q %s"
|
||||
;;
|
||||
*)
|
||||
echo "sha256sum %s | awk '{print \$1}'"
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
# we arrive here only if --sum is given with unknown format prefix
|
||||
echo "unknown hash format: $sum_hash" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
|
@ -1,6 +1,11 @@
|
|||
#!/bin/sh -e
|
||||
|
||||
dst="/$__object_id"
|
||||
if [ -f "$__object/parameter/destination" ]
|
||||
then
|
||||
dst="$( cat "$__object/parameter/destination" )"
|
||||
else
|
||||
dst="/$__object_id"
|
||||
fi
|
||||
|
||||
if [ ! -f "$dst" ]
|
||||
then
|
||||
|
@ -16,57 +21,19 @@ fi
|
|||
|
||||
sum_should="$( cat "$__object/parameter/sum" )"
|
||||
|
||||
if [ -f "$__object/parameter/cmd-sum" ]
|
||||
if echo "$sum_should" | grep -Fq ':'
|
||||
then
|
||||
# shellcheck disable=SC2059
|
||||
sum_is="$( eval "$( printf \
|
||||
"$( cat "$__object/parameter/cmd-sum" )" \
|
||||
"$dst" )" )"
|
||||
else
|
||||
os="$( "$__explorer/os" )"
|
||||
|
||||
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
|
||||
then
|
||||
sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
|
||||
|
||||
elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
|
||||
then
|
||||
case "$os" in
|
||||
freebsd)
|
||||
sum_is="md5:$( md5 -q "$dst" )"
|
||||
;;
|
||||
*)
|
||||
sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
|
||||
;;
|
||||
esac
|
||||
|
||||
elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
|
||||
then
|
||||
case "$os" in
|
||||
freebsd)
|
||||
sum_is="sha1:$( sha1 -q "$dst" )"
|
||||
;;
|
||||
*)
|
||||
sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
|
||||
;;
|
||||
esac
|
||||
|
||||
elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
|
||||
then
|
||||
case "$os" in
|
||||
freebsd)
|
||||
sum_is="sha256:$( sha256 -q "$dst" )"
|
||||
;;
|
||||
*)
|
||||
sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
|
||||
fi
|
||||
|
||||
sum_cmd="$( "$__type_explorer/remote_cmd_sum" )"
|
||||
|
||||
# shellcheck disable=SC2059
|
||||
sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )"
|
||||
|
||||
if [ -z "$sum_is" ]
|
||||
then
|
||||
echo 'no checksum from target' >&2
|
||||
echo 'existing destination checksum failed' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
@ -11,34 +11,133 @@ fi
|
|||
|
||||
url="$( cat "$__object/parameter/url" )"
|
||||
|
||||
tmp="$( mktemp )"
|
||||
|
||||
dst="/$__object_id"
|
||||
if [ -f "$__object/parameter/destination" ]
|
||||
then
|
||||
dst="$( cat "$__object/parameter/destination" )"
|
||||
else
|
||||
dst="/$__object_id"
|
||||
fi
|
||||
|
||||
if [ -f "$__object/parameter/cmd-get" ]
|
||||
then
|
||||
cmd="$( cat "$__object/parameter/cmd-get" )"
|
||||
|
||||
elif command -v wget > /dev/null
|
||||
then
|
||||
cmd="wget -O - '%s'"
|
||||
|
||||
elif command -v curl > /dev/null
|
||||
then
|
||||
cmd="curl -L -o - '%s'"
|
||||
cmd="curl -sSL -o - '%s'"
|
||||
|
||||
elif command -v fetch > /dev/null
|
||||
then
|
||||
cmd="fetch -o - '%s'"
|
||||
|
||||
elif command -v wget > /dev/null
|
||||
then
|
||||
cmd="wget -O - '%s'"
|
||||
|
||||
else
|
||||
echo 'no usable locally installed utility for downloading' >&2
|
||||
echo 'local download failed, no usable utility' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "$cmd > %s\n" \
|
||||
"$url" \
|
||||
"$tmp"
|
||||
echo "download_tmp=\"\$( mktemp )\""
|
||||
|
||||
# shellcheck disable=SC2059
|
||||
printf "$cmd > \"\$download_tmp\"\n" "$url"
|
||||
|
||||
if [ -f "$__object/parameter/sum" ]
|
||||
then
|
||||
sum_should="$( cat "$__object/parameter/sum" )"
|
||||
|
||||
if [ -f "$__object/parameter/cmd-sum" ]
|
||||
then
|
||||
local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
|
||||
else
|
||||
if echo "$sum_should" | grep -Fq ':'
|
||||
then
|
||||
sum_hash="$( echo "$sum_should" | cut -d : -f 1 )"
|
||||
|
||||
sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
|
||||
else
|
||||
if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
|
||||
then
|
||||
sum_hash='cksum'
|
||||
elif
|
||||
echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$'
|
||||
then
|
||||
sum_hash='md5'
|
||||
elif
|
||||
echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$'
|
||||
then
|
||||
sum_hash='sha1'
|
||||
elif
|
||||
echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$'
|
||||
then
|
||||
sum_hash='sha256'
|
||||
else
|
||||
echo 'hash format detection failed' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
case "$sum_hash" in
|
||||
cksum)
|
||||
local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'"
|
||||
;;
|
||||
md5)
|
||||
if command -v md5 > /dev/null
|
||||
then
|
||||
local_cmd_sum="md5 -q %s"
|
||||
elif
|
||||
command -v md5sum > /dev/null
|
||||
then
|
||||
local_cmd_sum="md5sum %s | awk '{print \$1}'"
|
||||
fi
|
||||
;;
|
||||
sha1)
|
||||
if command -v sha1 > /dev/null
|
||||
then
|
||||
local_cmd_sum="sha1 -q %s"
|
||||
elif
|
||||
command -v sha1sum > /dev/null
|
||||
then
|
||||
local_cmd_sum="sha1sum %s | awk '{print \$1}'"
|
||||
fi
|
||||
;;
|
||||
sha256)
|
||||
if command -v sha256 > /dev/null
|
||||
then
|
||||
local_cmd_sum="sha256 -q %s"
|
||||
elif
|
||||
command -v sha256sum > /dev/null
|
||||
then
|
||||
local_cmd_sum="sha256sum %s | awk '{print \$1}'"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
# we arrive here only if --sum is given with unknown format prefix
|
||||
echo "unknown hash format: $sum_hash" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ -z "$local_cmd_sum" ]
|
||||
then
|
||||
echo 'local checksum verification failed, no usable utility' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2059
|
||||
echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\""
|
||||
|
||||
echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
|
||||
|
||||
echo "echo 'local download checksum mismatch' >&2"
|
||||
|
||||
echo "rm -f \"\$download_tmp\""
|
||||
|
||||
echo 'exit 1; fi'
|
||||
fi
|
||||
|
||||
if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$'
|
||||
then
|
||||
|
@ -47,12 +146,10 @@ else
|
|||
target_host="$__target_host"
|
||||
fi
|
||||
|
||||
printf '%s %s %s:%s\n' \
|
||||
# shellcheck disable=SC2016
|
||||
printf '%s "$download_tmp" %s:%s\n' \
|
||||
"$__remote_copy" \
|
||||
"$tmp" \
|
||||
"$target_host" \
|
||||
"$dst"
|
||||
|
||||
echo "rm -f '$tmp'"
|
||||
|
||||
echo 'downloaded' > "$__messages_out"
|
||||
echo "rm -f \"\$download_tmp\""
|
||||
|
|
|
@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )"
|
|||
|
||||
if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
|
||||
then
|
||||
cmd="$( cat "$__object/explorer/remote_cmd" )"
|
||||
cmd_get="$( cat "$__object/explorer/remote_cmd_get" )"
|
||||
|
||||
url="$( cat "$__object/parameter/url" )"
|
||||
|
||||
dst="/$__object_id"
|
||||
if [ -f "$__object/parameter/destination" ]
|
||||
then
|
||||
dst="$( cat "$__object/parameter/destination" )"
|
||||
else
|
||||
dst="/$__object_id"
|
||||
fi
|
||||
|
||||
printf "$cmd > %s\n" \
|
||||
"$url" \
|
||||
"$dst"
|
||||
echo "download_tmp=\"\$( mktemp )\""
|
||||
|
||||
echo 'downloaded' > "$__messages_out"
|
||||
# shellcheck disable=SC2059
|
||||
printf "$cmd_get > \"\$download_tmp\"\n" "$url"
|
||||
|
||||
if [ -f "$__object/parameter/sum" ]
|
||||
then
|
||||
sum_should="$( cat "$__object/parameter/sum" )"
|
||||
|
||||
if [ -f "$__object/parameter/cmd-sum" ]
|
||||
then
|
||||
remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )"
|
||||
else
|
||||
remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )"
|
||||
|
||||
if echo "$sum_should" | grep -Fq ':'
|
||||
then
|
||||
sum_should="$( echo "$sum_should" | cut -d : -f 2 )"
|
||||
fi
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2059
|
||||
echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\""
|
||||
|
||||
echo "if [ \"\$sum_is\" != '$sum_should' ]; then"
|
||||
|
||||
echo "echo 'remote download checksum mismatch' >&2"
|
||||
|
||||
echo "rm -f \"\$download_tmp\""
|
||||
|
||||
echo 'exit 1; fi'
|
||||
fi
|
||||
|
||||
echo "mv \"\$download_tmp\" '$dst'"
|
||||
fi
|
||||
|
||||
if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ]
|
||||
|
|
|
@ -8,7 +8,7 @@ cdist-type__download - Download a file
|
|||
|
||||
DESCRIPTION
|
||||
-----------
|
||||
By default type will try to use ``wget``, ``curl`` or ``fetch``.
|
||||
By default type will try to use ``curl``, ``fetch`` or ``wget``.
|
||||
If download happens in target (see ``--download``) then type will
|
||||
fallback to (and install) ``wget``.
|
||||
|
||||
|
@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like
|
|||
``{http,https,ftp}_proxy`` etc can be used on cdist execution
|
||||
(``http_proxy=foo cdist config ...``).
|
||||
|
||||
To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``.
|
||||
|
||||
|
||||
REQUIRED PARAMETERS
|
||||
-------------------
|
||||
|
@ -25,14 +27,29 @@ url
|
|||
|
||||
OPTIONAL PARAMETERS
|
||||
-------------------
|
||||
destination
|
||||
Downloaded file's destination in target. If unset, ``$__object_id`` is used.
|
||||
|
||||
sum
|
||||
Checksum is used to decide if existing destination file must be redownloaded.
|
||||
By default output of ``cksum`` without filename is expected.
|
||||
Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
|
||||
Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256.
|
||||
|
||||
Type tries to detect hash format with regexes, but prefixes
|
||||
``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported.
|
||||
|
||||
Checksum have two purposes - state check and post-download verification.
|
||||
In state check, if destination checksum mismatches, then content of URL
|
||||
will be downloaded to temporary file. If downloaded temporary file's
|
||||
checksum matches, then it will be moved to destination (overwritten).
|
||||
|
||||
For local downloads it is expected that usable utilities for checksum
|
||||
calculation exist in the system.
|
||||
|
||||
download
|
||||
If ``local`` (default), then download file to local storage and copy
|
||||
it to target host. If ``remote``, then download happens in target.
|
||||
If ``local`` (default), then file is downloaded to local storage and copied
|
||||
to target host. If ``remote``, then download happens in target.
|
||||
|
||||
For local downloads it is expected that usable utilities for downloading
|
||||
exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``.
|
||||
|
||||
cmd-get
|
||||
Command used for downloading.
|
||||
|
@ -62,7 +79,7 @@ EXAMPLES
|
|||
require='__directory/opt/cpma' \
|
||||
__download /opt/cpma/cnq3.zip \
|
||||
--url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
|
||||
--sum md5:46da3021ca9eace277115ec9106c5b46
|
||||
--sum 46da3021ca9eace277115ec9106c5b46
|
||||
|
||||
require='__download/opt/cpma/cnq3.zip' \
|
||||
__unpack /opt/cpma/cnq3.zip \
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/sh -e
|
||||
|
||||
if grep -Eq '^wget' "$__object/explorer/remote_cmd"
|
||||
if grep -Eq '^wget' "$__object/explorer/remote_cmd_get"
|
||||
then
|
||||
__package wget
|
||||
fi
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
sum
|
||||
cmd-get
|
||||
cmd-sum
|
||||
destination
|
||||
download
|
||||
onchange
|
||||
sum
|
||||
|
|
Loading…
Reference in a new issue