Merge branch 'ander/__rsync' into 'master'

[__rsync] rewrite

See merge request ungleich-public/cdist!1007

cdist/conf/type/__rsync/gencode-local

@@ -1,41 +1,104 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
-source=$(cat "$__object/parameter/source")
+if ! command -v rsync > /dev/null
-remote_user=$(cat "$__object/parameter/remote-user")
+then
+    echo 'rsync is missing in local machine' >&2
+    exit 1
+fi
 
-if [ -f "$__object/parameter/destination" ]; then
+src="$( cat "$__object/parameter/source" )"
-    destination=$(cat "$__object/parameter/destination")
+
+if [ ! -e "$src" ]
+then
+    echo "$src not found" >&2
+    exit 1
+fi
+
+if [ -f "$__object/parameter/destination" ]
+then
+    dst="$( cat "$__object/parameter/destination" )"
 else
-    destination="/$__object_id"
+    dst="/$__object_id"
 fi
 
-set --
+# if source is directory, then make sure that
-if [ -f "$__object/parameter/rsync-opts" ]; then
+# source and destination are ending with slash,
-    while read -r opts; do
+# because this is what you almost always want when
-        set -- "$@" "--$opts"
+# rsyncing two directories.
-    done < "$__object/parameter/rsync-opts"
+
+if [ -d "$src" ]
+then
+    if ! echo "$src" | grep -Eq '/$'
+    then
+        src="$src/"
     fi
 
+    if ! echo "$dst" | grep -Eq '/$'
+    then
+        dst="$dst/"
+    fi
+fi
+
+remote_user="$( cat "$__object/parameter/remote-user" )"
+
+options="$( cat "$__object/parameter/options" )"
+
+if [ -f "$__object/parameter/option" ]
+then
+    while read -r l
+    do
+        # there's a limitation in argparse: value can't begin with '-'.
+        # to workaround this, let's prefix opts with '\' in manifest and remove here.
+        # read more about argparse issue: https://bugs.python.org/issue9334
+
+        options="$options $( echo "$l" | sed 's/\\//g' )"
+    done \
+        < "$__object/parameter/option"
+fi
+
+if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ]
+then
+    options="$options --chown="
+
+    if [ -f "$__object/parameter/owner" ]
+    then
+        owner="$( cat "$__object/parameter/owner" )"
+        options="$options$owner"
+    fi
+
+    if [ -f "$__object/parameter/group" ]
+    then
+        group="$( cat "$__object/parameter/group" )"
+        options="$options:$group"
+    fi
+fi
+
+if [ -f "$__object/parameter/mode" ]
+then
+    mode="$( cat "$__object/parameter/mode" )"
+    options="$options --chmod=$mode"
+fi
+
+# IMPORTANT
+#
+# 1. we first dry-run rsync with change summary to find out
+#    if there are any changes and code generation is needed.
+# 2. normally, to get current state or target host, we run
+#    such operations in type explorers, but that's not
+#    possible due to how rsync works.
+# 3. redirecting output of dry-run to stderr to ease debugging.
+# 4. to understand how that cryptic regex works, please
+#    open rsync manpage and read about --itemize-changes.
+
+export RSYNC_RSH="$__remote_exec"
+
 # shellcheck disable=SC2086
-echo rsync -a \
+if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \
-    --no-owner --no-group \
+    | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2
-    -e \"${__remote_exec}\" \
+then
-    -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}"
+    exit 0
+fi
+
+echo "export RSYNC_RSH='$__remote_exec'"
+
+echo "rsync $options $src $remote_user@$__target_host:$dst"

cdist/conf/type/__rsync/gencode-remote

@@ -1,37 +0,0 @@
-#!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-
-if [ -f "$__object/parameter/destination" ]; then
-    destination=$(cat "$__object/parameter/destination")
-else
-    destination="/$__object_id"
-fi
-
-ownergroup=""
-if [ -f "$__object/parameter/owner" ]; then
-    ownergroup=$(cat "$__object/parameter/owner")
-fi
-if [ -f "$__object/parameter/group" ]; then
-    ownergroup="${ownergroup}:$(cat "$__object/parameter/group")"
-fi
-
-if [ "$ownergroup" ]; then
-    echo chown -R "$ownergroup" "$destination"
-fi

cdist/conf/type/__rsync/man.rst

@@ -3,112 +3,73 @@ cdist-type__rsync(7)
 
 NAME
 ----
-cdist-type__rsync - Mirror directories using rsync
+cdist-type__rsync - Mirror directories using ``rsync``
 
 
 DESCRIPTION
 -----------
-WARNING: This type is of BETA quality:
+The purpose of this type is to bring power of ``rsync`` into ``cdist``.
-
-- it has not been tested widely
-- interfaces *may* change
-- if there is a better approach to solve the problem -> the type may even vanish
-
-If you are fine with these constraints, please read on.
-
-
-This cdist type allows you to mirror local directories to the
-target host using rsync. Rsync will be installed in the manifest of the type.
-If group or owner are giveng, a recursive chown will be executed on the 
-target host.
-
-A slash will be appended to the source directory so that only the contents
-of the directory are taken and not the directory name itself.
 
 
 REQUIRED PARAMETERS
 -------------------
 source
-    Where to take files from
+   Source directory in local machine.
+   If source is directory, slash (``/``) will be added to source and destination paths.
 
 
 OPTIONAL PARAMETERS
 -------------------
-group
+destination
-   Group to chgrp to.
+   Destination directory. Defaults to ``$__object_id``.
 
 owner
-   User to chown to.
+   Will be passed to ``rsync`` as ``--chown=OWNER``.
+   Read ``rsync(1)`` for more details.
 
-destination
+group
-    Use this as the base destination instead of the object id
+   Will be passed to ``rsync`` as ``--chown=:GROUP``.
+   Read ``rsync(1)`` for more details.
+
+mode
+   Will be passed to ``rsync`` as ``--chmod=MODE``.
+   Read ``rsync(1)`` for more details.
+
+options
+   Defaults to ``--recursive --links --perms --times``.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
 
 remote-user
-    Use this user instead of the default "root" for rsync operations.
+   Defaults to ``root``.
 
 
 OPTIONAL MULTIPLE PARAMETERS
 ----------------------------
-rsync-opts
+option
-    Use this option to give rsync options with.
+   Pass additional options to ``rsync``.
-    See rsync(1) for available options.
+   See ``rsync(1)`` for all possible options.
-    Only "--" options are supported.
+   Due to `bug in Python's argparse<https://bugs.python.org/issue9334>`_, value must be prefixed with ``\``.
-    Write the options without the beginning "--"
-    Can be specified multiple times.
-
-
-MESSAGES
---------
-NONE
 
 
 EXAMPLES
 --------
-
 .. code-block:: sh
 
-    # You can use any source directory
+    __rsync /var/www/example.com \
-    __rsync /tmp/testdir \
+        --owner root \
-        --source /etc
+        --group www-data \
-
+        --mode 'D750,F640' \
-    # Use source from type
+        --source "$__files/example.com/www"
-    __rsync /etc \
-        --source "$__type/files/package"
-
-    # Allow multiple __rsync objects to write to the same dir
-    __rsync mystuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package"
-
-    __rsync otherstuff \
-        --destination /usr/local/bin \
-        --source "$__type/files/package2"
-
-    # Use rsync option --exclude
-    __rsync /tmp/testdir \
-        --source /etc \
-        --rsync-opts exclude=sshd_conf
-
-    # Use rsync with multiple options --exclude --dry-run
-    __rsync /tmp/testing \
-        --source /home/tester \
-        --rsync-opts exclude=id_rsa \
-        --rsync-opts dry-run
-
-
-SEE ALSO
---------
-:strong:`rsync`\ (1)
 
 
 AUTHORS
 -------
-Nico Schottelius <nico-cdist--@--schottelius.org>
+Ander Punnar <ander-at-kvlt-dot-ee>
 
 
 COPYING
 -------
-Copyright \(C) 2015 Nico Schottelius. You can redistribute it
+Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it
-and/or modify it under the terms of the GNU General Public License as
+under the terms of the GNU General Public License as published by the Free
-published by the Free Software Foundation, either version 3 of the
+Software Foundation, either version 3 of the License, or (at your option)
-License, or (at your option) any later version.
+any later version.

cdist/conf/type/__rsync/manifest

@@ -1,21 +1,3 @@
 #!/bin/sh -e
-#
-# 2015 Dominique Roux (dominique.roux4 at gmail.com)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
 
 __package rsync

cdist/conf/type/__rsync/parameter/default/options

@@ -0,0 +1 @@
+--recursive --links --perms --times

cdist/conf/type/__rsync/parameter/optional

@@ -1,4 +1,6 @@
 destination
-owner
 group
+mode
+options
+owner
 remote-user

cdist/conf/type/__rsync/parameter/optional_multiple

@@ -1 +1 @@
-rsync-opts
+option