From 26dfdf37c21ce3c469ae0d6a717d390f422f9fbf Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 21 Jun 2020 01:13:30 +0300
Subject: [PATCH 1/5] [__download] support multiple checksum formats and
 download utilities, add --onchange and other minor changes

---
 cdist/conf/type/__download/explorer/state     | 78 +++++++++++++++----
 cdist/conf/type/__download/gencode-local      | 25 +++++-
 cdist/conf/type/__download/gencode-remote     |  7 ++
 cdist/conf/type/__download/man.rst            | 25 ++++--
 .../type/__download/parameter/default/cmd-get |  1 -
 .../type/__download/parameter/default/cmd-sum |  1 -
 cdist/conf/type/__download/parameter/optional |  1 +
 7 files changed, 116 insertions(+), 22 deletions(-)
 create mode 100755 cdist/conf/type/__download/gencode-remote
 delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-get
 delete mode 100644 cdist/conf/type/__download/parameter/default/cmd-sum

diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state
index 6a50f5a5..00362545 100755
--- a/cdist/conf/type/__download/explorer/state
+++ b/cdist/conf/type/__download/explorer/state
@@ -2,19 +2,71 @@
 
 dst="/$__object_id"
 
-# shellcheck disable=SC2059
-cmd="$( printf "$( cat "$__object/parameter/cmd-sum" )" "$dst" )"
-
-sum="$( cat "$__object/parameter/sum" )"
-
-if [ -f "$dst" ]
+if [ ! -f "$dst" ]
 then
-    if [ "$( eval "$cmd" )" = "$sum" ]
-    then
-        echo 'present'
-    else
-        echo 'mismatch'
-    fi
-else
     echo 'absent'
+    exit 0
+fi
+
+sum_should="$( cat "$__object/parameter/sum" )"
+
+if [ -f "$__object/parameter/cmd-sum" ]
+then
+    # shellcheck disable=SC2059
+    sum_is="$( eval "$( printf \
+        "$( cat "$__object/parameter/cmd-sum" )" \
+        "$dst" )" )"
+else
+    os="$( "$__explorer/os" )"
+
+    if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$'
+    then
+        sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )"
+
+    elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="md5:$( md5 -q "$dst" )"
+            ;;
+            *)
+                sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha1:$( sha1 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+
+    elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$'
+    then
+        case "$os" in
+            freebsd)
+                sum_is="sha256:$( sha256 -q "$dst" )"
+            ;;
+            *)
+                sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )"
+            ;;
+        esac
+    fi
+fi
+
+if [ -z "$sum_is" ]
+then
+    echo 'no checksum from target' >&2
+    exit 1
+fi
+
+if [ "$sum_is" = "$sum_should" ]
+then
+    echo 'present'
+else
+    echo 'mismatch'
 fi
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 49e9c699..85ef3a60 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -9,12 +9,31 @@ fi
 
 url="$( cat "$__object/parameter/url" )"
 
-cmd="$( cat "$__object/parameter/cmd-get" )"
-
 tmp="$( mktemp )"
 
 dst="/$__object_id"
 
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v wget > /dev/null
+then
+    cmd="wget -O - '%s'"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    echo 'no usable locally installed utility for downloading' >&2
+    exit 1
+fi
+
 printf "$cmd > %s\n" \
     "$url" \
     "$tmp"
@@ -33,3 +52,5 @@ printf '%s %s %s:%s\n' \
     "$dst"
 
 echo "rm -f '$tmp'"
+
+echo 'downloaded' > "$__messages_out"
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
new file mode 100755
index 00000000..b08d0050
--- /dev/null
+++ b/cdist/conf/type/__download/gencode-remote
@@ -0,0 +1,7 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/onchange" ] \
+    && grep -Fq "$__object_id:downloaded" "$__messages_in"
+then
+    cat "$__object/parameter/onchange"
+fi
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index c973448f..63a41bc4 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -10,7 +10,13 @@ DESCRIPTION
 -----------
 You must use persistent storage in target host for destination file
 (``$__object_id``) because it will be used for checksum calculation
-in order to decide if file must be downloaded.
+in order to decide if file must be (re-)downloaded.
+
+By default type will try to use following locally installed utilities
+for downloading (in order): ``wget``, ``curl`` or ``fetch``.
+
+Environment variables like ``{http,https,ftp}_proxy`` etc can be used on
+cdist execution (``http_proxy=foo cdist config ...``).
 
 
 REQUIRED PARAMETERS
@@ -19,20 +25,29 @@ url
    URL from which to download the file.
 
 sum
-   Checksum of downloaded file.
+   Checksum of file going to be downloaded.
+   By default output of ``cksum`` without filename is expected.
+   Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``.
+
+onchange
+   Execute this command after download.
 
 
 OPTIONAL PARAMETERS
 -------------------
 cmd-get
    Command used for downloading.
-   Default is ``wget -O- '%s'``.
    Command must output to ``stdout``.
+   Parameter will be used for ``printf`` and must include only one
+   variable ``%s`` which will become URL.
+   For example: ``wget -O - '%s'``.
 
 cmd-sum
    Command used for checksum calculation.
-   Default is ``md5sum '%s' | awk '{print $1}'``.
    Command output and ``--sum`` parameter must match.
+   Parameter will be used for ``printf`` and must include only one
+   variable ``%s`` which will become destination.
+   For example: ``md5sum '%s' | awk '{print $1}'``.
 
 
 EXAMPLES
@@ -45,7 +60,7 @@ EXAMPLES
     require='__directory/opt/cpma' \
         __download /opt/cpma/cnq3.zip \
             --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \
-            --sum 46da3021ca9eace277115ec9106c5b46
+            --sum md5:46da3021ca9eace277115ec9106c5b46
 
     require='__download/opt/cpma/cnq3.zip' \
         __unpack /opt/cpma/cnq3.zip \
diff --git a/cdist/conf/type/__download/parameter/default/cmd-get b/cdist/conf/type/__download/parameter/default/cmd-get
deleted file mode 100644
index 2daa38a1..00000000
--- a/cdist/conf/type/__download/parameter/default/cmd-get
+++ /dev/null
@@ -1 +0,0 @@
-wget -O- '%s'
diff --git a/cdist/conf/type/__download/parameter/default/cmd-sum b/cdist/conf/type/__download/parameter/default/cmd-sum
deleted file mode 100644
index 3e8a9295..00000000
--- a/cdist/conf/type/__download/parameter/default/cmd-sum
+++ /dev/null
@@ -1 +0,0 @@
-md5sum '%s' | awk '{print $1}'
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 22783e02..38c0ce4d 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,2 +1,3 @@
 cmd-get
 cmd-sum
+onchange

From 85614aabd6f3abef5af6d362203db956c9d72aa9 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 28 Jun 2020 16:38:15 +0300
Subject: [PATCH 2/5] [__download] add --download (local|remote), update manual

---
 .../conf/type/__download/explorer/remote_cmd  | 19 +++++++++++++++
 cdist/conf/type/__download/gencode-local      |  4 +++-
 cdist/conf/type/__download/gencode-remote     | 22 +++++++++++++++--
 cdist/conf/type/__download/man.rst            | 24 ++++++++++++-------
 cdist/conf/type/__download/manifest           |  6 +++++
 .../__download/parameter/default/download     |  1 +
 cdist/conf/type/__download/parameter/optional |  1 +
 7 files changed, 65 insertions(+), 12 deletions(-)
 create mode 100755 cdist/conf/type/__download/explorer/remote_cmd
 create mode 100755 cdist/conf/type/__download/manifest
 create mode 100644 cdist/conf/type/__download/parameter/default/download

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
new file mode 100755
index 00000000..fbd4d84c
--- /dev/null
+++ b/cdist/conf/type/__download/explorer/remote_cmd
@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+if [ -f "$__object/parameter/cmd-get" ]
+then
+    cmd="$( cat "$__object/parameter/cmd-get" )"
+
+elif command -v curl > /dev/null
+then
+    cmd="curl -o - '%s'"
+
+elif command -v fetch > /dev/null
+then
+    cmd="fetch -o - '%s'"
+
+else
+    cmd="wget -O - '%s'"
+fi
+
+echo "$cmd"
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 85ef3a60..339827c2 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -1,8 +1,10 @@
 #!/bin/sh -e
 
+download="$( cat "$__object/parameter/download" )"
+
 state_is="$( cat "$__object/explorer/state" )"
 
-if [ "$state_is" = 'present' ]
+if [ "$download" != 'local' ] || [ "$state_is" = 'present' ]
 then
     exit 0
 fi
diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote
index b08d0050..89ba72af 100755
--- a/cdist/conf/type/__download/gencode-remote
+++ b/cdist/conf/type/__download/gencode-remote
@@ -1,7 +1,25 @@
 #!/bin/sh -e
 
-if [ -f "$__object/parameter/onchange" ] \
-    && grep -Fq "$__object_id:downloaded" "$__messages_in"
+download="$( cat "$__object/parameter/download" )"
+
+state_is="$( cat "$__object/explorer/state" )"
+
+if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ]
+then
+    cmd="$( cat "$__object/explorer/remote_cmd" )"
+
+    url="$( cat "$__object/parameter/url" )"
+
+    dst="/$__object_id"
+
+    printf "$cmd > %s\n" \
+        "$url" \
+        "$dst"
+
+    echo 'downloaded' > "$__messages_out"
+fi
+
+if [ -f "$__object/parameter/onchange" ] && [ "$state" != "present" ]
 then
     cat "$__object/parameter/onchange"
 fi
diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index 63a41bc4..c161f4e4 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -3,26 +3,28 @@ cdist-type__download(7)
 
 NAME
 ----
-cdist-type__download - Download file to local storage and copy it to target host
+cdist-type__download - Download a file
 
 
 DESCRIPTION
 -----------
-You must use persistent storage in target host for destination file
-(``$__object_id``) because it will be used for checksum calculation
-in order to decide if file must be (re-)downloaded.
+Persistent storage for destination file in target host must be used
+(``$__object_id``) because it will be used for checksum calculation in
+order to decide if file must be (re-)downloaded.
 
-By default type will try to use following locally installed utilities
-for downloading (in order): ``wget``, ``curl`` or ``fetch``.
+By default type will try to use ``wget``, ``curl`` or ``fetch`` for
+downloading.  If ``--download remote`` type will fallback to (and
+install) ``wget``.
 
-Environment variables like ``{http,https,ftp}_proxy`` etc can be used on
-cdist execution (``http_proxy=foo cdist config ...``).
+If ``--download local`` (default), then environment variables like
+``{http,https,ftp}_proxy`` etc can be used on cdist execution
+(``http_proxy=foo cdist config ...``).
 
 
 REQUIRED PARAMETERS
 -------------------
 url
-   URL from which to download the file.
+   File's URL.
 
 sum
    Checksum of file going to be downloaded.
@@ -35,6 +37,10 @@ onchange
 
 OPTIONAL PARAMETERS
 -------------------
+download
+   If ``local`` (default), then download file to local storage and copy
+   it to target host. If ``remote``, then download happens in target.
+
 cmd-get
    Command used for downloading.
    Command must output to ``stdout``.
diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest
new file mode 100755
index 00000000..7ec8d86d
--- /dev/null
+++ b/cdist/conf/type/__download/manifest
@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+if grep -Eq '^wget' "$__object/explorer/remote_cmd"
+then
+    __package wget
+fi
diff --git a/cdist/conf/type/__download/parameter/default/download b/cdist/conf/type/__download/parameter/default/download
new file mode 100644
index 00000000..40830374
--- /dev/null
+++ b/cdist/conf/type/__download/parameter/default/download
@@ -0,0 +1 @@
+local
diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional
index 38c0ce4d..838e2fbf 100644
--- a/cdist/conf/type/__download/parameter/optional
+++ b/cdist/conf/type/__download/parameter/optional
@@ -1,3 +1,4 @@
 cmd-get
 cmd-sum
+download
 onchange

From b6bf90e3f1a775ec1bd27a99015107d24a397787 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Sun, 28 Jun 2020 16:43:45 +0300
Subject: [PATCH 3/5] [__download] update manual

---
 cdist/conf/type/__download/man.rst | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index c161f4e4..eafa9dfc 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -8,15 +8,14 @@ cdist-type__download - Download a file
 
 DESCRIPTION
 -----------
-Persistent storage for destination file in target host must be used
-(``$__object_id``) because it will be used for checksum calculation in
-order to decide if file must be (re-)downloaded.
+Destination (``$__object_id``) in target host must be persistent storage
+in order to calculate checksum and decide if file must be (re-)downloaded.
 
-By default type will try to use ``wget``, ``curl`` or ``fetch`` for
-downloading.  If ``--download remote`` type will fallback to (and
-install) ``wget``.
+By default type will try to use ``wget``, ``curl`` or ``fetch``.
+If download happens in target (see ``--download``) then type will
+fallback to (and install) ``wget``.
 
-If ``--download local`` (default), then environment variables like
+If download happens in local machine, then environment variables like
 ``{http,https,ftp}_proxy`` etc can be used on cdist execution
 (``http_proxy=foo cdist config ...``).
 

From 93506d2113e1d05202527639efe883a5b44b220d Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 8 Jul 2020 00:17:12 +0300
Subject: [PATCH 4/5] [__download] curl follow redirects

---
 cdist/conf/type/__download/explorer/remote_cmd | 2 +-
 cdist/conf/type/__download/gencode-local       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd
index fbd4d84c..e3e35b45 100755
--- a/cdist/conf/type/__download/explorer/remote_cmd
+++ b/cdist/conf/type/__download/explorer/remote_cmd
@@ -6,7 +6,7 @@ then
 
 elif command -v curl > /dev/null
 then
-    cmd="curl -o - '%s'"
+    cmd="curl -L -o - '%s'"
 
 elif command -v fetch > /dev/null
 then
diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local
index 339827c2..571d2c3c 100755
--- a/cdist/conf/type/__download/gencode-local
+++ b/cdist/conf/type/__download/gencode-local
@@ -25,7 +25,7 @@ then
 
 elif command -v curl > /dev/null
 then
-    cmd="curl -o - '%s'"
+    cmd="curl -L -o - '%s'"
 
 elif command -v fetch > /dev/null
 then

From e9062662868d7677a235889ba39a6091085388f0 Mon Sep 17 00:00:00 2001
From: Ander Punnar <ander@kvlt.ee>
Date: Wed, 8 Jul 2020 00:20:55 +0300
Subject: [PATCH 5/5] [__download] s/variable/format specification/

---
 cdist/conf/type/__download/man.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst
index eafa9dfc..6ec0b19a 100644
--- a/cdist/conf/type/__download/man.rst
+++ b/cdist/conf/type/__download/man.rst
@@ -44,14 +44,14 @@ cmd-get
    Command used for downloading.
    Command must output to ``stdout``.
    Parameter will be used for ``printf`` and must include only one
-   variable ``%s`` which will become URL.
+   format specification ``%s`` which will become URL.
    For example: ``wget -O - '%s'``.
 
 cmd-sum
    Command used for checksum calculation.
    Command output and ``--sum`` parameter must match.
    Parameter will be used for ``printf`` and must include only one
-   variable ``%s`` which will become destination.
+   format specification ``%s`` which will become destination.
    For example: ``md5sum '%s' | awk '{print $1}'``.