From c7fbdc8195f781d3affe82538f09ccb9b9759ae5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 22 May 2012 12:40:38 +0200 Subject: [PATCH 01/13] add --remote-exec and --remote-copy to command line args Signed-off-by: Nico Schottelius --- bin/cdist | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bin/cdist b/bin/cdist index c6467477..897ddbfc 100755 --- a/bin/cdist +++ b/bin/cdist @@ -68,6 +68,12 @@ def commandline(): parser['configinstall'].add_argument('-s', '--sequential', help='Operate on multiple hosts sequentially (default)', action='store_false', dest='parallel') + parser['configinstall'].add_argument('--remote-exec', + help='Command to use for remote execution (should behave like ssh)', + action='store_true', dest='remote_exec') + parser['configinstall'].add_argument('--remote-copy', + help='Command to use for remote copy (should behave like scp)', + action='store_true', dest='remote_copy') # Config parser['config'] = parser['sub'].add_parser('config', From 108283bbebb8319bcc99946eb58262e275f34cb6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 22 May 2012 13:01:31 +0200 Subject: [PATCH 02/13] add default values in argparse Signed-off-by: Nico Schottelius --- bin/cdist | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 897ddbfc..e011ec6d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -68,12 +68,15 @@ def commandline(): parser['configinstall'].add_argument('-s', '--sequential', help='Operate on multiple hosts sequentially (default)', action='store_false', dest='parallel') - parser['configinstall'].add_argument('--remote-exec', - help='Command to use for remote execution (should behave like ssh)', - action='store_true', dest='remote_exec') + parser['configinstall'].add_argument('--remote-copy', help='Command to use for remote copy (should behave like scp)', - action='store_true', dest='remote_copy') + action='store', dest='remote_copy', + default="scp -o User=root -q") + parser['configinstall'].add_argument('--remote-exec', + help='Command to use for remote execution (should behave like ssh)', + action='store', dest='remote_exec', + default="ssh -o User=root -q") # Config parser['config'] = parser['sub'].add_parser('config', @@ -165,6 +168,8 @@ def configinstall_onehost(host, args, mode, parallel): context = cdist.context.Context( target_host=host, + remote_copy=args.remote_copy, + remote_exec=args.remote_exec, initial_manifest=args.manifest, base_path=args.cdist_home, exec_path=sys.argv[0], From 9f319ae1213caf8b36ceaf3708d20981442f4f2e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 22 May 2012 14:55:11 +0200 Subject: [PATCH 03/13] support passing remote_{exec, copy} to context Signed-off-by: Nico Schottelius --- lib/cdist/context.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/cdist/context.py b/lib/cdist/context.py index ab8677a7..c38d6b94 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -35,6 +35,8 @@ class Context(object): def __init__(self, target_host, + remote_copy, + remote_exec, initial_manifest=False, base_path=False, exec_path=sys.argv[0], @@ -70,10 +72,18 @@ class Context(object): self.initial_manifest = (initial_manifest or os.path.join(self.local.manifest_path, "init")) - # Remote + _init_remote(remote_copy, remote_exec) + + # Remote stuff + def _init_remote(self, remote_copy, remote_exec): + self.remote_base_path = os.environ.get('__cdist_remote_out_dir', "/var/lib/cdist") - self.remote_exec = os.environ.setdefault('__remote_exec', "ssh -o User=root -q") - self.remote_copy = os.environ.setdefault('__remote_copy', "scp -o User=root -q") + self.remote_copy = remote_copy + self.remote_exec = remote_exec + + os.environ['__remote_copy'] = self.remote_copy + os.environ['__remote_exec'] = self.remote_exec + self.remote = remote.Remote(self.target_host, self.remote_base_path, self.remote_exec, self.remote_copy) From 26d7eab852bae2eb43de9bcd4b615fc2ca0fa31e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 22 May 2012 15:06:04 +0200 Subject: [PATCH 04/13] document change + manpage Signed-off-by: Nico Schottelius --- doc/changelog | 3 +++ doc/man/man1/cdist.text | 29 ++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/doc/changelog b/doc/changelog index 63fcb6b0..966e876e 100644 --- a/doc/changelog +++ b/doc/changelog @@ -4,6 +4,9 @@ Changelog * Changes are always commented with their author in (braces) * Exception: No braces means author == Nico Schottelius +2.0.11: + * Add support for --remote-exec and --remote-copy parameters + 2.0.10: 2012-05-18 * Cleanup __group: No getent gshadow in old Redhat, use groupmod -g (Matt Coddington) diff --git a/doc/man/man1/cdist.text b/doc/man/man1/cdist.text index e46e84a3..b92fba18 100644 --- a/doc/man/man1/cdist.text +++ b/doc/man/man1/cdist.text @@ -41,23 +41,29 @@ CONFIG Configure a system -h, --help:: - Show the help screen + Show the help screen -c CDIST_HOME, --cdist-home CDIST_HOME:: - Instead of using the parent of the bin directory as cdist home, - use the specified directory + Instead of using the parent of the bin directory as cdist home, + use the specified directory -d, --debug:: - Enable debug output + Enable debug output -i MANIFEST, --initial-manifest MANIFEST:: - Path to a cdist manifest or - to read from stdin + Path to a cdist manifest or - to read from stdin -p, --parallel:: - Operate on multiple hosts in parallel + Operate on multiple hosts in parallel -s, --sequential:: - Operate on multiple hosts sequentially + Operate on multiple hosts sequentially + +--remote-copy REMOTE_COPY: + Command to use for remote copy (should behave like scp) + +--remote-exec REMOTE_EXEC: + Command to use for remote execution (should behave like ssh) EXAMPLES @@ -69,7 +75,12 @@ cdist config -d ikq05.ethz.ch # Configure hosts in parallel and use a different home directory cdist config -c ~/p/cdist-nutzung \ - -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch + -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch + +# Use custom remote exec / copy commands +cdist config --remote-exec /path/to/my/remote/exec \ + --remote-copy /path/to/my/remote/copy \ + -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch # Display banner cdist banner @@ -98,7 +109,7 @@ The following exit values shall be returned: 0:: Successful completion 1:: - One or more host configuration failed. + One or more host configurations failed SEE ALSO From 02bd4fdf3fb75b118a767d6a99288e1a3f2ab1d9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 22 May 2012 15:10:02 +0200 Subject: [PATCH 05/13] add template for tutorial for remote-exec-copy Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-remote-exec-copy.text | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 doc/man/man7/cdist-remote-exec-copy.text diff --git a/doc/man/man7/cdist-remote-exec-copy.text b/doc/man/man7/cdist-remote-exec-copy.text new file mode 100644 index 00000000..6010907a --- /dev/null +++ b/doc/man/man7/cdist-remote-exec-copy.text @@ -0,0 +1,35 @@ +cdist-remote-exec-copy(7) +========================= +Nico Schottelius +STEVEN HERE + + +NAME +---- +cdist-remote-exec-copy - How to get use remote exec and copy + + +INTRO +------- +What it is, how it works + + +EXAMPLES +-------------- +nfsroot, sudo (?), what exists + + +HACKER INFORMATION +------------------ +Not sure if needed, but may be helpful to explain how it +works internally + +SEE ALSO +-------- +- cdist(7) + + +COPYING +------- +Copyright \(C) 2011-2012 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). From eea9a72676c1ed9cf7fc7e910938bb7231e5b5a3 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 29 May 2012 09:29:01 +0200 Subject: [PATCH 06/13] bugfix: tell schroot which chroot to use Signed-off-by: Steven Armstrong --- other/examples/remote/schroot-uri | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/other/examples/remote/schroot-uri b/other/examples/remote/schroot-uri index a23277ec..9819c5a5 100755 --- a/other/examples/remote/schroot-uri +++ b/other/examples/remote/schroot-uri @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -49,7 +49,8 @@ my_name="${0##*/}" mode="$1"; shift log() { - echo "$@" | logger -t "cdist-$my_name-$mode" + # uncomment me for debugging + #echo "$@" | logger -t "cdist-$my_name-$mode" : } @@ -109,6 +110,9 @@ else copy_prefix="cp" copy_destination_prefix="" fi +log "exec_prefix: $exec_prefix" +log "copy_prefix: $copy_prefix" +log "copy_destination_prefix: $copy_destination_prefix" case "$mode" in exec) @@ -118,8 +122,9 @@ case "$mode" in ;; copy) # get directory for given chroot_name - schroot_directory="$($exec_prefix schroot $chroot_name --config | awk -F = '/directory=/ {print $2}')" + schroot_directory="$($exec_prefix schroot -c $schroot_name --config | awk -F = '/directory=/ {print $2}')" [ -n "$schroot_directory" ] || die "Failed to retreive schroot directory for schroot: $schroot_name" + log "schroot_directory: $schroot_directory" # prefix destination with chroot code="$copy_prefix $(echo "$@" | sed "s|$uri:|${copy_destination_prefix}${schroot_directory}|g")" ;; From 21b85e410e203a86dbd3cbc592ca3c4c1cab9441 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 29 May 2012 10:12:00 +0200 Subject: [PATCH 07/13] bugfix: its a method, not a function Signed-off-by: Steven Armstrong --- lib/cdist/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/context.py b/lib/cdist/context.py index c38d6b94..8b468739 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -72,7 +72,7 @@ class Context(object): self.initial_manifest = (initial_manifest or os.path.join(self.local.manifest_path, "init")) - _init_remote(remote_copy, remote_exec) + self._init_remote(remote_copy, remote_exec) # Remote stuff def _init_remote(self, remote_copy, remote_exec): From 5edf39f1114be85fe3b8d1b772f5625e3177a14f Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 29 May 2012 11:02:23 +0200 Subject: [PATCH 08/13] no special case for rsync in core. handle implementation specific details in remote-copy script instead Signed-off-by: Steven Armstrong --- lib/cdist/exec/remote.py | 7 +------ other/examples/remote/rsync/copy | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/cdist/exec/remote.py b/lib/cdist/exec/remote.py index fb90939d..487beea3 100644 --- a/lib/cdist/exec/remote.py +++ b/lib/cdist/exec/remote.py @@ -77,12 +77,7 @@ class Remote(object): self.log.debug("Remote transfer: %s -> %s", source, destination) self.rmdir(destination) command = self._copy.split() - # support rsync by appending a "/" to the source if it's a directory - if os.path.isdir(source): - command.extend(["-r", source + "/", self.target_host + ":" + destination]) - else: - command.extend(["-r", source, self.target_host + ":" + destination]) - + command.extend(["-r", source, self.target_host + ":" + destination]) self._run_command(command) def run_script(self, script, env=None, return_output=False): diff --git a/other/examples/remote/rsync/copy b/other/examples/remote/rsync/copy index f6b93c5c..96d3f3de 100755 --- a/other/examples/remote/rsync/copy +++ b/other/examples/remote/rsync/copy @@ -24,7 +24,24 @@ # at /etc/passwd~cdist. # # Usage: -# __remote_copy="/path/to/this/script" cdist config target_host +# cdist config --remote-copy /path/to/this/script target_host # +# second last argument is the source +source_index=$(($#-1)) +index=0 +for arg in $@; do + if [ $index -eq 0 ]; then + # reset $@ + set -- + fi + index=$((index+=1)) + if [ $index -eq $source_index -a -d "$arg" ]; then + echo "directory: $arg" | logger + # if the source is a directory, it has to end with "/" for rsync to do the right thing + arg="${arg%/}/" + fi + set -- "$@" "$arg" +done + rsync --backup --suffix=~cdist -e 'ssh -o User=root' $@ From d052e2d510f3ec4506400bb35a4e81596660dcc7 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 29 May 2012 11:03:15 +0200 Subject: [PATCH 09/13] --debug Signed-off-by: Steven Armstrong --- other/examples/remote/rsync/copy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/other/examples/remote/rsync/copy b/other/examples/remote/rsync/copy index 96d3f3de..0d4bd165 100755 --- a/other/examples/remote/rsync/copy +++ b/other/examples/remote/rsync/copy @@ -33,11 +33,10 @@ index=0 for arg in $@; do if [ $index -eq 0 ]; then # reset $@ - set -- + set -- fi index=$((index+=1)) if [ $index -eq $source_index -a -d "$arg" ]; then - echo "directory: $arg" | logger # if the source is a directory, it has to end with "/" for rsync to do the right thing arg="${arg%/}/" fi From fc8ff292893e060a13e456e50454299f13e27f7e Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 29 May 2012 11:14:21 +0200 Subject: [PATCH 10/13] update comments to work with arguments vs environment variables: /__remote_{exec,copy}/--remote-{exec,copy}/ Signed-off-by: Steven Armstrong --- other/examples/remote/chroot/copy | 2 +- other/examples/remote/chroot/exec | 2 +- other/examples/remote/schroot-uri | 6 ++++-- other/examples/remote/schroot/copy | 2 +- other/examples/remote/schroot/exec | 2 +- other/examples/remote/ssh/copy | 2 +- other/examples/remote/ssh/exec | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/other/examples/remote/chroot/copy b/other/examples/remote/chroot/copy index 528a5faf..8f8cb680 100755 --- a/other/examples/remote/chroot/copy +++ b/other/examples/remote/chroot/copy @@ -22,7 +22,7 @@ # to a remote target host. # # Usage: -# __remote_copy="/path/to/this/script /path/to/your/chroot" cdist config target-id +# cdist config --remote-copy "/path/to/this/script /path/to/your/chroot" target-id # log() { diff --git a/other/examples/remote/chroot/exec b/other/examples/remote/chroot/exec index 19e76b0e..4637f918 100755 --- a/other/examples/remote/chroot/exec +++ b/other/examples/remote/chroot/exec @@ -22,7 +22,7 @@ # on a remote target host. # # Usage: -# __remote_exec="/path/to/this/script /path/to/your/chroot" cdist config target-id +# cdist config --remote-exec "/path/to/this/script /path/to/your/chroot" target-id # log() { diff --git a/other/examples/remote/schroot-uri b/other/examples/remote/schroot-uri index 9819c5a5..5b50a195 100755 --- a/other/examples/remote/schroot-uri +++ b/other/examples/remote/schroot-uri @@ -21,8 +21,10 @@ # __remote_{exec,copy} script to run cdist against a schroot target uri # # Usage: -# __remote_exec="/path/to/this/script exec" cdist config target_uri -# __remote_copy="/path/to/this/script copy" cdist config target_uri +# cdist config \ +# --remote-exec "/path/to/this/script exec" \ +# --remote-copy "/path/to/this/script copy" \ +# target_uri # # # target_uri examples: # schroot:///chroot-name diff --git a/other/examples/remote/schroot/copy b/other/examples/remote/schroot/copy index 3587a4f2..cbd45573 100755 --- a/other/examples/remote/schroot/copy +++ b/other/examples/remote/schroot/copy @@ -21,7 +21,7 @@ # __remote_copy script to run cdist against a chroot on the target host over ssh. # # Usage: -# __remote_copy="/path/to/this/script schroot-chroot-name" cdist config target_host +# cdist config --remote-copy "/path/to/this/script schroot-chroot-name" target_host # log() { diff --git a/other/examples/remote/schroot/exec b/other/examples/remote/schroot/exec index 5b561de0..2510fd22 100755 --- a/other/examples/remote/schroot/exec +++ b/other/examples/remote/schroot/exec @@ -21,7 +21,7 @@ # __remote_exec script to run cdist against a chroot on the target host over ssh. # # Usage: -# __remote_exec="/path/to/this/script schroot-chroot-name" cdist config target_host +# cdist config --remote-exec "/path/to/this/script schroot-chroot-name" target_host # log() { diff --git a/other/examples/remote/ssh/copy b/other/examples/remote/ssh/copy index 0ecd8c52..5b0ed324 100755 --- a/other/examples/remote/ssh/copy +++ b/other/examples/remote/ssh/copy @@ -21,7 +21,7 @@ # same as cdist default # # Usage: -# __remote_copy="/path/to/this/script" cdist config target_host +# cdist config --remote-copy "/path/to/this/script" target_host # #echo "$@" | logger -t "cdist-ssh-copy" diff --git a/other/examples/remote/ssh/exec b/other/examples/remote/ssh/exec index b597a47f..2875c3fc 100755 --- a/other/examples/remote/ssh/exec +++ b/other/examples/remote/ssh/exec @@ -21,7 +21,7 @@ # same as cdist default # # Usage: -# __remote_exec="/path/to/this/script" cdist config target_host +# cdist config --remote-exec "/path/to/this/script" target_host # #echo "$@" | logger -t "cdist-ssh-exec" From d34ea9a9c968f8923ef0388a235458e529fbcf8a Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 29 May 2012 12:00:03 +0200 Subject: [PATCH 11/13] add missing -c argument Signed-off-by: Steven Armstrong --- other/examples/remote/schroot/copy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/examples/remote/schroot/copy b/other/examples/remote/schroot/copy index cbd45573..d995055e 100755 --- a/other/examples/remote/schroot/copy +++ b/other/examples/remote/schroot/copy @@ -33,7 +33,7 @@ chroot_name="$1"; shift target_host="$__target_host" # get directory for given chroot_name -chroot="$(ssh -o User=root -q $target_host schroot $chroot_name --config | awk -F = '/directory=/ {print $2}')" +chroot="$(ssh -o User=root -q $target_host schroot -c $chroot_name --config | awk -F = '/directory=/ {print $2}')" # prefix destination with chroot code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")" From 9d09407cc476993cc94380469b97fd350a8ec332 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 29 May 2012 12:04:35 +0200 Subject: [PATCH 12/13] ++doc Signed-off-by: Steven Armstrong --- other/examples/remote/rsync/copy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/other/examples/remote/rsync/copy b/other/examples/remote/rsync/copy index 0d4bd165..76217caf 100755 --- a/other/examples/remote/rsync/copy +++ b/other/examples/remote/rsync/copy @@ -1,6 +1,7 @@ #!/bin/sh # # 2012 Matt Coddington (mcoddington at gmail.com) +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -27,6 +28,9 @@ # cdist config --remote-copy /path/to/this/script target_host # +# For rsync to do the right thing, the source has to end with "/" if it is +# a directory. The below preprocessor loop takes care of that. + # second last argument is the source source_index=$(($#-1)) index=0 @@ -37,7 +41,6 @@ for arg in $@; do fi index=$((index+=1)) if [ $index -eq $source_index -a -d "$arg" ]; then - # if the source is a directory, it has to end with "/" for rsync to do the right thing arg="${arg%/}/" fi set -- "$@" "$arg" From 8e9e0a862a1fcb00c1b26bacacfdc870cd78bb76 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 29 May 2012 13:33:22 +0200 Subject: [PATCH 13/13] document how remote exec/copy is used Signed-off-by: Steven Armstrong --- doc/man/man7/cdist-remote-exec-copy.text | 31 ++++++++++++++++-------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/doc/man/man7/cdist-remote-exec-copy.text b/doc/man/man7/cdist-remote-exec-copy.text index 6010907a..d789b12d 100644 --- a/doc/man/man7/cdist-remote-exec-copy.text +++ b/doc/man/man7/cdist-remote-exec-copy.text @@ -1,29 +1,40 @@ cdist-remote-exec-copy(7) ========================= Nico Schottelius -STEVEN HERE NAME ---- -cdist-remote-exec-copy - How to get use remote exec and copy +cdist-remote-exec-copy - How to use remote exec and copy INTRO -------- -What it is, how it works +----- +Cdist interacts with the target host in two ways: +- it executes code (__remote_exec) +- and it copies files (__remote_copy) + +By default this is accomplished with ssh and scp respectively. +The default implementations used by cdist are: +__remote_exec: ssh -o User=root -q +__remote_copy: scp -o User=root -q + +The user can override these defaults by providing custom implementations and +passing them to cdist with the --remote-exec and/or --remote-copy arguments. + +For __remote_exec, the custom implementation must behave as if it where ssh. +For __remote_copy, it must behave like scp. + +With this simple interface the user can take total control of how cdist +interacts with the target when required, while the default implementation +remains as simple as possible. EXAMPLES -------------- -nfsroot, sudo (?), what exists +See cdist/other/examples/remote/ for some example implementations. -HACKER INFORMATION ------------------- -Not sure if needed, but may be helpful to explain how it -works internally - SEE ALSO -------- - cdist(7)