From ca9dd7338adfd371705f27793d8bd20e99762d8d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 2 Nov 2016 13:35:48 +0100 Subject: [PATCH 1/6] Support IPv6 in python code. --- cdist/exec/remote.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 9c70bdf4..f374262f 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -32,6 +32,43 @@ import cdist import cdist.exec.util as exec_util +# check whether addr is IPv6 +try: + # python 3.3+ + import ipaddress + + def _is_ipv6(addr): + try: + return ipaddress.ip_address(addr).version == 6 + except ValueError: + return False +except ImportError: + # fallback for older python versions + import socket + + def _is_ipv6(addr): + try: + socket.inet_aton(addr) + return False + except socket.error: + pass + try: + socket.inet_pton(socket.AF_INET6, addr) + return True + except socket.error: + pass + return False + + +def _wrap_addr(addr): + """If addr is IPv6 then return addr wrapped between '[' and ']', + otherwise return it intact.""" + if _is_ipv6(addr): + return "".join(("[", addr, "]", )) + else: + return addr + + class DecodeError(cdist.Error): def __init__(self, command): self.command = command @@ -118,12 +155,12 @@ class Remote(object): command = self._copy.split() path = os.path.join(source, f) command.extend([path, '{0}:{1}'.format( - self.target_host[0], destination)]) + _wrap_addr(self.target_host[0]), destination)]) self._run_command(command) else: command = self._copy.split() command.extend([source, '{0}:{1}'.format( - self.target_host[0], destination)]) + _wrap_addr(self.target_host[0]), destination)]) self._run_command(command) def transfer_dir_parallel(self, source, destination, jobs): @@ -145,7 +182,7 @@ class Remote(object): command = self._copy.split() path = os.path.join(source, f) command.extend([path, '{0}:{1}'.format( - self.target_host[0], destination)]) + _wrap_addr(self.target_host[0]), destination)]) commands.append(command) results = [ pool.apply_async(self._run_command, (cmd,)) From 4ddf6557e3d9b269650c35ee7e11040b4e747932 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 2 Nov 2016 13:55:25 +0100 Subject: [PATCH 2/6] IPv6 fix in gencode scripts. --- cdist/conf/type/__file/gencode-local | 11 ++++++++++- cdist/conf/type/__jail_freebsd10/gencode-local | 11 ++++++++++- cdist/conf/type/__jail_freebsd9/gencode-local | 11 ++++++++++- cdist/conf/type/__pf_ruleset/gencode-local | 14 ++++++++++++-- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index ed7482cb..674650b7 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -66,8 +66,17 @@ destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template" DONE if [ "$upload_file" ]; then echo upload >> "$__messages_out" + # IPv6 fix + case "${__target_host}" in + *:*) + my_target_host="[${__target_host}]" + ;; + *) + my_target_host="${__target_host}" + ;; + esac cat << DONE -$__remote_copy "$source" "${__target_host}:\$destination_upload" +$__remote_copy "$source" "${my_target_host}:\$destination_upload" DONE fi # move uploaded file into place diff --git a/cdist/conf/type/__jail_freebsd10/gencode-local b/cdist/conf/type/__jail_freebsd10/gencode-local index d4b89730..766692a2 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-local +++ b/cdist/conf/type/__jail_freebsd10/gencode-local @@ -43,7 +43,16 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then - echo "$__remote_copy" "${jailbase}" "$__target_host:${remotebase}" + # IPv6 fix + case "${__target_host}" in + *:*) + my_target_host="[${__target_host}]" + ;; + *) + my_target_host="${__target_host}" + ;; + esac + echo "$__remote_copy" "${jailbase}" "${my_target_host}:${remotebase}" fi # basepresent=NONE fi # state=present diff --git a/cdist/conf/type/__jail_freebsd9/gencode-local b/cdist/conf/type/__jail_freebsd9/gencode-local index 08c7b7bf..35b6f789 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-local +++ b/cdist/conf/type/__jail_freebsd9/gencode-local @@ -39,7 +39,16 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then - echo "$__remote_copy" "${jailbase}" "$__target_host:${remotebase}" + # IPv6 fix + case "${__target_host}" in + *:*) + my_target_host="[${__target_host}]" + ;; + *) + my_target_host="${__target_host}" + ;; + esac + echo "$__remote_copy" "${jailbase}" "${my_target_host}:${remotebase}" fi # basepresent=NONE fi # state=present diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local index c2495509..307c89ec 100644 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -59,12 +59,22 @@ case $uname in ;; esac +# IPv6 fix +case "${__target_host}" in + *:*) + my_target_host="[${__target_host}]" + ;; + *) + my_target_host="${__target_host}" + ;; +esac + if [ -n "${cksum}" ]; then if [ ! "\${currentSum}" = "${cksum}" ]; then - $__remote_copy "${source}" "$__target_host:${rcvar}.new" + $__remote_copy "${source}" "${my_target_host}:${rcvar}.new" fi else # File just doesn't exist yet - $__remote_copy "${source}" "$__target_host:${rcvar}.new" + $__remote_copy "${source}" "${my_target_host}:${rcvar}.new" fi EOF From 6ce6c7830bcc0e360349d071b00f2aecbf099dd2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 5 Nov 2016 16:38:49 +0100 Subject: [PATCH 3/6] Better test for IPv6 address. --- cdist/conf/type/__file/gencode-local | 15 +++++++-------- cdist/conf/type/__jail_freebsd10/gencode-local | 15 +++++++-------- cdist/conf/type/__jail_freebsd9/gencode-local | 15 +++++++-------- cdist/conf/type/__pf_ruleset/gencode-local | 15 +++++++-------- 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 674650b7..c9e0fa2c 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -67,14 +67,13 @@ DONE if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix - case "${__target_host}" in - *:*) - my_target_host="[${__target_host}]" - ;; - *) - my_target_host="${__target_host}" - ;; - esac + echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + if [ $? -eq 0 ] + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi cat << DONE $__remote_copy "$source" "${my_target_host}:\$destination_upload" DONE diff --git a/cdist/conf/type/__jail_freebsd10/gencode-local b/cdist/conf/type/__jail_freebsd10/gencode-local index 766692a2..bdd0ffd7 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-local +++ b/cdist/conf/type/__jail_freebsd10/gencode-local @@ -44,14 +44,13 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - case "${__target_host}" in - *:*) - my_target_host="[${__target_host}]" - ;; - *) - my_target_host="${__target_host}" - ;; - esac + echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + if [ $? -eq 0 ] + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi echo "$__remote_copy" "${jailbase}" "${my_target_host}:${remotebase}" fi # basepresent=NONE fi # state=present diff --git a/cdist/conf/type/__jail_freebsd9/gencode-local b/cdist/conf/type/__jail_freebsd9/gencode-local index 35b6f789..ce9b215e 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-local +++ b/cdist/conf/type/__jail_freebsd9/gencode-local @@ -40,14 +40,13 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - case "${__target_host}" in - *:*) - my_target_host="[${__target_host}]" - ;; - *) - my_target_host="${__target_host}" - ;; - esac + echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + if [ $? -eq 0 ] + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi echo "$__remote_copy" "${jailbase}" "${my_target_host}:${remotebase}" fi # basepresent=NONE fi # state=present diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local index 307c89ec..d14a6045 100644 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -60,14 +60,13 @@ case $uname in esac # IPv6 fix -case "${__target_host}" in - *:*) - my_target_host="[${__target_host}]" - ;; - *) - my_target_host="${__target_host}" - ;; -esac +echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' +if [ $? -eq 0 ] +then + my_target_host="[${__target_host}]" +else + my_target_host="${__target_host}" +fi if [ -n "${cksum}" ]; then if [ ! "\${currentSum}" = "${cksum}" ]; then From 835e281c9c8229fca959bb18dfc5ca77dd4ae7d7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 27 Nov 2016 17:02:51 +0100 Subject: [PATCH 4/6] Notice IPv6 address [] notation. --- docs/src/cdist-remote-exec-copy.rst | 5 +++++ docs/src/cdist-type.rst | 6 ++++++ docs/src/man1/cdist.rst | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/docs/src/cdist-remote-exec-copy.rst b/docs/src/cdist-remote-exec-copy.rst index 882f0bc2..bb818310 100644 --- a/docs/src/cdist-remote-exec-copy.rst +++ b/docs/src/cdist-remote-exec-copy.rst @@ -17,6 +17,11 @@ passing them to cdist with the --remote-exec and/or --remote-copy arguments. For __remote_exec, the custom implementation must behave as if it where ssh. For __remote_copy, it must behave like scp. +Please notice, custom implementations should work like ssh/scp so __remote_copy +must support IPv6 addresses enclosed in square brackets. For __remote_exec you +must take into account that for some options (like -L) IPv6 addresses can be +specified by enclosed in square brackets (see :strong:`ssh`\ (1) and +:strong:`scp`\ (1)). With this simple interface the user can take total control of how cdist interacts with the target when required, while the default implementation diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 62694fd8..1ff82ce0 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -251,6 +251,12 @@ script, you can write to stderr: # Output to be saved by cdist for execution on the target echo "touch /etc/cdist-configured" +Notice: if you use __remote_copy or __remote_exec directly in your scripts +then for IPv6 address with __remote_copy execution you should enclose IPv6 +address in square brackets. The same applies to __remote_exec if it behaves +the same as ssh for some options where colon is a delimiter, as for -L ssh +option (see :strong:`ssh`\ (1) and :strong:`scp`\ (1)). + Variable access from the generated scripts ------------------------------------------ diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 45ce339e..f3cb672c 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -157,6 +157,12 @@ cdist/conf The distribution configuration directory. It contains official types and explorers. This path is relative to cdist installation directory. +NOTES +----- +cdist detects if host is specified by IPv6 address. If so then remote_copy +command is executed with host address enclosed in square brackets +(see :strong:`scp`\ (1)). + EXAMPLES -------- From 36244b8a11d56106e8a041cc7b066a5debbf826b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Nov 2016 08:02:33 +0100 Subject: [PATCH 5/6] Update changelog. --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index b5959d77..07278241 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Core, types: support IPv6 (Darko Poljak) * Type __consul: add source and cksum files for Consul 0.7.0 and 0.7.1 (Carlos Ortigoza) * Type __user: FreeBSD fix (Kamila Součková) * New type: __apt_mark (Ander Punnar) From ca424a34e6f94261d681a588e6403673b965fc28 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Nov 2016 08:13:59 +0100 Subject: [PATCH 6/6] Fix shell IPv6 testing (for -e). --- cdist/conf/type/__file/gencode-local | 3 +-- cdist/conf/type/__jail_freebsd10/gencode-local | 3 +-- cdist/conf/type/__jail_freebsd9/gencode-local | 3 +-- cdist/conf/type/__pf_ruleset/gencode-local | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index c9e0fa2c..4caa6df6 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -67,8 +67,7 @@ DONE if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix - echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - if [ $? -eq 0 ] + if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__jail_freebsd10/gencode-local b/cdist/conf/type/__jail_freebsd10/gencode-local index bdd0ffd7..8c1687a9 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-local +++ b/cdist/conf/type/__jail_freebsd10/gencode-local @@ -44,8 +44,7 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - if [ $? -eq 0 ] + if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__jail_freebsd9/gencode-local b/cdist/conf/type/__jail_freebsd9/gencode-local index ce9b215e..67420a6f 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-local +++ b/cdist/conf/type/__jail_freebsd9/gencode-local @@ -40,8 +40,7 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - if [ $? -eq 0 ] + if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local index d14a6045..2db2ae06 100644 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -60,8 +60,7 @@ case $uname in esac # IPv6 fix -echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' -if [ $? -eq 0 ] +if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') then my_target_host="[${__target_host}]" else