From 0d6171c69bbfd6af35fdc3c2affb1a721f54e2f6 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 12 Jul 2011 14:15:39 +0200 Subject: [PATCH 01/55] start work on cinst partitioning Signed-off-by: Steven Armstrong --- .../__cinst_partition_msdos/gencode-remote | 20 ++++++ conf/type/__cinst_partition_msdos/man.text | 63 +++++++++++++++++++ conf/type/__cinst_partition_msdos/manifest | 25 ++++++++ .../parameter/optional | 3 + .../parameter/required | 1 + 5 files changed, 112 insertions(+) create mode 100755 conf/type/__cinst_partition_msdos/gencode-remote create mode 100644 conf/type/__cinst_partition_msdos/man.text create mode 100755 conf/type/__cinst_partition_msdos/manifest create mode 100644 conf/type/__cinst_partition_msdos/parameter/optional create mode 100644 conf/type/__cinst_partition_msdos/parameter/required diff --git a/conf/type/__cinst_partition_msdos/gencode-remote b/conf/type/__cinst_partition_msdos/gencode-remote new file mode 100755 index 00000000..211bc1db --- /dev/null +++ b/conf/type/__cinst_partition_msdos/gencode-remote @@ -0,0 +1,20 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + diff --git a/conf/type/__cinst_partition_msdos/man.text b/conf/type/__cinst_partition_msdos/man.text new file mode 100644 index 00000000..2203c2d3 --- /dev/null +++ b/conf/type/__cinst_partition_msdos/man.text @@ -0,0 +1,63 @@ +cdist-type__cinst_partition_msdos(7) +==================================== +Steven Armstrong + + +NAME +---- +cdist-type__cinst_partition_msdos - creates msdos partitions + + +DESCRIPTION +----------- +This cdist type allows you to create msdos paritions. + + +REQUIRED PARAMETERS +------------------- +type:: + the partition type used in fdisk (such as 82 or 83) or "extended" + + +OPTIONAL PARAMETERS +------------------- +device:: + defaults to object_id +bootable:: + mark partition as bootable, true or false +size:: + the size of the partition (such as 32MB or 15GB, whole numbers + only), '+' for remaining space, or 'n%' for percentage of remaining + (these should only be used after all specific partition sizes are + specified), leave blank if type is "extended". + Defaults to +. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# 128MB linux, bootable +__cinst_partition_msdos /dev/sda1 --type 83 --size 128M --bootable true +# 512MB swap +__cinst_partition_msdos /dev/sda2 --type 82 --size 512M +# extended +__cinst_partition_msdos /dev/sda3 --type extended +# 10GB, linux +__cinst_partition_msdos /dev/sda5 --type 82 --size 10G +# rest of disk, linux +__cinst_partition_msdos /dev/sda6 --type 82 --size + +# same thing as +__cinst_partition_msdos /dev/sda6 --type 82 +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/conf/type/__cinst_partition_msdos/manifest b/conf/type/__cinst_partition_msdos/manifest new file mode 100755 index 00000000..e5b04c02 --- /dev/null +++ b/conf/type/__cinst_partition_msdos/manifest @@ -0,0 +1,25 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +type + +device +bootable +size diff --git a/conf/type/__cinst_partition_msdos/parameter/optional b/conf/type/__cinst_partition_msdos/parameter/optional new file mode 100644 index 00000000..612fe769 --- /dev/null +++ b/conf/type/__cinst_partition_msdos/parameter/optional @@ -0,0 +1,3 @@ +device +bootable +size diff --git a/conf/type/__cinst_partition_msdos/parameter/required b/conf/type/__cinst_partition_msdos/parameter/required new file mode 100644 index 00000000..aa80e646 --- /dev/null +++ b/conf/type/__cinst_partition_msdos/parameter/required @@ -0,0 +1 @@ +type From b6738aaf60054beecdca9105250c80d51ebfb338 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 16 Sep 2011 12:19:09 +0200 Subject: [PATCH 02/55] rename __cinst_partition_msdos __partition_msdos Signed-off-by: Steven Armstrong --- .../gencode-remote | 0 .../man.text | 27 ++++++++++--------- .../manifest | 0 .../parameter/optional | 0 .../parameter/required | 0 5 files changed, 14 insertions(+), 13 deletions(-) rename conf/type/{__cinst_partition_msdos => __partition_msdos}/gencode-remote (100%) rename conf/type/{__cinst_partition_msdos => __partition_msdos}/man.text (59%) rename conf/type/{__cinst_partition_msdos => __partition_msdos}/manifest (100%) rename conf/type/{__cinst_partition_msdos => __partition_msdos}/parameter/optional (100%) rename conf/type/{__cinst_partition_msdos => __partition_msdos}/parameter/required (100%) diff --git a/conf/type/__cinst_partition_msdos/gencode-remote b/conf/type/__partition_msdos/gencode-remote similarity index 100% rename from conf/type/__cinst_partition_msdos/gencode-remote rename to conf/type/__partition_msdos/gencode-remote diff --git a/conf/type/__cinst_partition_msdos/man.text b/conf/type/__partition_msdos/man.text similarity index 59% rename from conf/type/__cinst_partition_msdos/man.text rename to conf/type/__partition_msdos/man.text index 2203c2d3..611451ed 100644 --- a/conf/type/__cinst_partition_msdos/man.text +++ b/conf/type/__partition_msdos/man.text @@ -1,11 +1,11 @@ -cdist-type__cinst_partition_msdos(7) -==================================== +cdist-type__partition_msdos(7) +============================== Steven Armstrong NAME ---- -cdist-type__cinst_partition_msdos - creates msdos partitions +cdist-type__partition_msdos - creates msdos partitions DESCRIPTION @@ -24,13 +24,12 @@ OPTIONAL PARAMETERS device:: defaults to object_id bootable:: - mark partition as bootable, true or false + mark partition as bootable, true or false, defaults to false size:: - the size of the partition (such as 32MB or 15GB, whole numbers + the size of the partition (such as 32M or 15G, whole numbers only), '+' for remaining space, or 'n%' for percentage of remaining (these should only be used after all specific partition sizes are - specified), leave blank if type is "extended". - Defaults to +. + specified). Defaults to +. EXAMPLES @@ -38,17 +37,19 @@ EXAMPLES -------------------------------------------------------------------------------- # 128MB linux, bootable -__cinst_partition_msdos /dev/sda1 --type 83 --size 128M --bootable true +__partition_msdos /dev/sda1 --type 83 --size 128M --bootable true # 512MB swap -__cinst_partition_msdos /dev/sda2 --type 82 --size 512M +__partition_msdos /dev/sda2 --type 82 --size 512M # extended -__cinst_partition_msdos /dev/sda3 --type extended +__partition_msdos /dev/sda3 --type extended --size 100G # 10GB, linux -__cinst_partition_msdos /dev/sda5 --type 82 --size 10G +__partition_msdos /dev/sda5 --type 83 --size 10G +# 50% of free space, linux +__partition_msdos /dev/sda6 --type 83 --size 50% # rest of disk, linux -__cinst_partition_msdos /dev/sda6 --type 82 --size + +__partition_msdos /dev/sda7 --type 83 --size + # same thing as -__cinst_partition_msdos /dev/sda6 --type 82 +__partition_msdos /dev/sda7 --type 83 -------------------------------------------------------------------------------- diff --git a/conf/type/__cinst_partition_msdos/manifest b/conf/type/__partition_msdos/manifest similarity index 100% rename from conf/type/__cinst_partition_msdos/manifest rename to conf/type/__partition_msdos/manifest diff --git a/conf/type/__cinst_partition_msdos/parameter/optional b/conf/type/__partition_msdos/parameter/optional similarity index 100% rename from conf/type/__cinst_partition_msdos/parameter/optional rename to conf/type/__partition_msdos/parameter/optional diff --git a/conf/type/__cinst_partition_msdos/parameter/required b/conf/type/__partition_msdos/parameter/required similarity index 100% rename from conf/type/__cinst_partition_msdos/parameter/required rename to conf/type/__partition_msdos/parameter/required From 31e9937098ebbcd9dd0b8535bb68c94bbb56b0eb Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 17 Sep 2011 06:53:18 +0200 Subject: [PATCH 03/55] finish new type for partition definition Signed-off-by: Steven Armstrong --- conf/type/__partition_msdos/gencode-remote | 20 ---------------- conf/type/__partition_msdos/man.text | 2 +- conf/type/__partition_msdos/manifest | 24 +++++++++++++++---- .../type/__partition_msdos/parameter/optional | 2 +- 4 files changed, 22 insertions(+), 26 deletions(-) delete mode 100755 conf/type/__partition_msdos/gencode-remote diff --git a/conf/type/__partition_msdos/gencode-remote b/conf/type/__partition_msdos/gencode-remote deleted file mode 100755 index 211bc1db..00000000 --- a/conf/type/__partition_msdos/gencode-remote +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -# -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - diff --git a/conf/type/__partition_msdos/man.text b/conf/type/__partition_msdos/man.text index 611451ed..c9ef0cf1 100644 --- a/conf/type/__partition_msdos/man.text +++ b/conf/type/__partition_msdos/man.text @@ -21,7 +21,7 @@ type:: OPTIONAL PARAMETERS ------------------- -device:: +partition:: defaults to object_id bootable:: mark partition as bootable, true or false, defaults to false diff --git a/conf/type/__partition_msdos/manifest b/conf/type/__partition_msdos/manifest index e5b04c02..0d73c405 100755 --- a/conf/type/__partition_msdos/manifest +++ b/conf/type/__partition_msdos/manifest @@ -18,8 +18,24 @@ # along with cdist. If not, see . # -type +# set defaults +if [ -f "$__object/parameter/partition" ]; then + partition="(cat "$__object/parameter/partition")" +else + partition="/$__object_id" + echo "$partition" > "$__object/parameter/partition" +fi +device="$(echo "$partition" | sed 's/[0-9]//g')" +echo "$device" > "$__object/parameter/device" +minor="$(echo "$partition" | sed 's/[^0-9]//g')" +echo "$minor" > "$__object/parameter/minor" -device -bootable -size +if [ ! -f "$__object/parameter/bootable" ]; then + echo "false" > "$__object/parameter/bootable" +fi +if [ ! -f "$__object/parameter/size" ]; then + echo "+" > "$__object/parameter/size" +fi + +# pull in the type that actually does something with the above parameters +require="$__self" __partition_msdos_apply diff --git a/conf/type/__partition_msdos/parameter/optional b/conf/type/__partition_msdos/parameter/optional index 612fe769..b2b0a4c2 100644 --- a/conf/type/__partition_msdos/parameter/optional +++ b/conf/type/__partition_msdos/parameter/optional @@ -1,3 +1,3 @@ -device +partition bootable size From 9d3fa5d4c7468142aca958a684c1425f663e2dc6 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 17 Sep 2011 08:55:24 +0200 Subject: [PATCH 04/55] new type __partition_msdos_apply Signed-off-by: Steven Armstrong --- .../explorer/partitions | 3 + .../type/__partition_msdos_apply/files/lib.sh | 58 +++++++++++ .../__partition_msdos_apply/gencode-remote | 99 +++++++++++++++++++ conf/type/__partition_msdos_apply/man.text | 42 ++++++++ conf/type/__partition_msdos_apply/singleton | 0 5 files changed, 202 insertions(+) create mode 100755 conf/type/__partition_msdos_apply/explorer/partitions create mode 100644 conf/type/__partition_msdos_apply/files/lib.sh create mode 100755 conf/type/__partition_msdos_apply/gencode-remote create mode 100644 conf/type/__partition_msdos_apply/man.text create mode 100644 conf/type/__partition_msdos_apply/singleton diff --git a/conf/type/__partition_msdos_apply/explorer/partitions b/conf/type/__partition_msdos_apply/explorer/partitions new file mode 100755 index 00000000..6be61af4 --- /dev/null +++ b/conf/type/__partition_msdos_apply/explorer/partitions @@ -0,0 +1,3 @@ +#!/bin/sh + +cat /proc/partitions diff --git a/conf/type/__partition_msdos_apply/files/lib.sh b/conf/type/__partition_msdos_apply/files/lib.sh new file mode 100644 index 00000000..944533a6 --- /dev/null +++ b/conf/type/__partition_msdos_apply/files/lib.sh @@ -0,0 +1,58 @@ +die() { + echo "[__partition_msdos_apply] $@" >&2 + exit 1 +} +debug() { + echo "[__partition_msdos_apply] $@" >&2 +} + +fdisk_command() { + local device=$1 + local cmd=$2 + + debug fdisk_command "running fdisk command '${cmd}' on device ${device}" + #echo -en "${cmd}\nw\n" | fdisk -c -u "$device" + return $? +} + +create_disklabel() { + local device=$1 + + debug create_disklabel "creating new msdos disklabel" + fdisk_command ${device} "o" + return $? +} + +create_partition() { + local device=$1 + local minor=$2 + local size=$3 + local type=$4 + local primary_count=$5 + + if [ "$type" = "extended" -o "$type" = "5" ]; then + # Extended partition + primary_extended="e\n" + first_minor="${minor}\n" + [ "${minor}" = "4" ] && first_minor="" + type_minor="${minor}\n" + [ "${minor}" = "1" ] && type_minor="" + type="5" + elif [ "${minor}" -lt "5" ]; then + primary_extended="p\n" + first_minor="${minor}\n" + [ "${minor}" = "4" ] && first_minor="" + type_minor="${minor}\n" + [ "${minor}" = "1" ] && type_minor="" + else + # Logical partitions + first_minor="${minor}\n" + type_minor="${minor}\n" + primary_extended="l\n" + [ "$primary_count" > "3" ] && primary_extended="" + fi + [ -n "${size}" ] && size="+${size}M" + fdisk_command ${device} "n\n${primary_extended}${first_minor}\n${size}\nt\n${type_minor}${type}\n" + return $? +} + diff --git a/conf/type/__partition_msdos_apply/gencode-remote b/conf/type/__partition_msdos_apply/gencode-remote new file mode 100755 index 00000000..faa712e7 --- /dev/null +++ b/conf/type/__partition_msdos_apply/gencode-remote @@ -0,0 +1,99 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +die() { + echo "[__partition_msdos_apply] $@" >&2 + exit 1 +} +debug() { + echo "[__partition_msdos_apply] $@" >&2 +} + +# Convert a size specifier 1G 100M or 50% into the corresponding numeric MB. +size_to_mb() { + local size=$1 + local available_size="$2" + + local number_suffix="$(echo ${size} | sed -e 's:\.[0-9]\+::' -e 's:\([0-9]\+\)\([MmGg%]\)[Bb]\?:\1|\2:')" + local number="$(echo ${number_suffix} | cut -d '|' -f1)" + local suffix="$(echo ${number_suffix} | cut -d '|' -f2)" + + case "$suffix" in + M|m) + size="$number" + ;; + G|g) + size="$(( $number * 1024 ))" + ;; + %) + size="$(( $available_size * $number / 100 ))" + ;; + *) + size="-1" + esac + echo "$size" +} + +# include function library for use on target +cat "$__type/files/lib.sh" + +partitions="$__object/explorer/partitions" +objects=$(find "$__global/object/__partition_msdos" -path "*.cdist") +current_device="" +available_size= +primary_count=0 +for object in $objects; do + device="$(cat "$object/parameter/device")" + if [ "$current_device" != "$device" ]; + echo "create_disklabel $device" + current_device="$device" + device_name=$(echo ${device} | sed -e 's:^/dev/::;s:/:\\/:g') + available_size=$(( $(awk "/${device_name}\$/ { print $3; }" "$partitions") / 1024)) + # make sure we don't go past the end of the drive + available_size=$((device_size - 2)) + primary_count=0 + fi + + type="$(cat "$object/parameter/type")" + partition="$(cat "$object/parameter/partition")" + minor="$(cat "$object/parameter/minor")" + + if [ "${minor}" -lt "5" ]; then + primary_count=$(( $primary_count + 1 )) + fi + bootable="$(cat "$object/parameter/bootable")" + size="$(cat "$object/parameter/size")" + if [ "$size" = "+" ]; then + # use rest of device + partition_size="" + available_size=0 + else + partition_size=$(size_to_mb "$size" "$available_size") + available_size="$(( $available_size - $partition_size ))" + fi + + [ "$partition_size" = "-1" ] && die "could not translate size '$size' to a usable value" + debug "primary_count=$primary_count" + debug "available_size=$available_size" + debug "current_device=$current_device" + + echo "create_partition $device $minor $partition_size $type $primary_count" +done + diff --git a/conf/type/__partition_msdos_apply/man.text b/conf/type/__partition_msdos_apply/man.text new file mode 100644 index 00000000..4d4f127c --- /dev/null +++ b/conf/type/__partition_msdos_apply/man.text @@ -0,0 +1,42 @@ +cdist-type__partition_msdos_apply(7) +==================================== +Steven Armstrong + + +NAME +---- +cdist-type__partition_msdos_apply + + +DESCRIPTION +----------- +Create the partitions defined with __partition_msdos + + +REQUIRED PARAMETERS +------------------- +None + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__partition_msdos_apply +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__partition_msdos_apply(7) + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/conf/type/__partition_msdos_apply/singleton b/conf/type/__partition_msdos_apply/singleton new file mode 100644 index 00000000..e69de29b From 16d86dcf04712d2d1fe7f4e7fe123190e6c7f198 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 17 Sep 2011 09:26:09 +0200 Subject: [PATCH 05/55] +debug, -type, -bug Signed-off-by: Steven Armstrong --- .../__partition_msdos_apply/gencode-remote | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/conf/type/__partition_msdos_apply/gencode-remote b/conf/type/__partition_msdos_apply/gencode-remote index faa712e7..d352abdb 100755 --- a/conf/type/__partition_msdos_apply/gencode-remote +++ b/conf/type/__partition_msdos_apply/gencode-remote @@ -61,14 +61,17 @@ available_size= primary_count=0 for object in $objects; do device="$(cat "$object/parameter/device")" - if [ "$current_device" != "$device" ]; + if [ "$current_device" != "$device" ]; then echo "create_disklabel $device" current_device="$device" device_name=$(echo ${device} | sed -e 's:^/dev/::;s:/:\\/:g') - available_size=$(( $(awk "/${device_name}\$/ { print $3; }" "$partitions") / 1024)) + available_size=$(( $(awk "/${device_name}\$/ { print \$3; }" "$partitions") / 1024)) # make sure we don't go past the end of the drive - available_size=$((device_size - 2)) + available_size=$((available_size - 2)) primary_count=0 + debug "----- $device" + debug "current_device=$current_device" + debug "available_size=$available_size" fi type="$(cat "$object/parameter/type")" @@ -90,9 +93,17 @@ for object in $objects; do fi [ "$partition_size" = "-1" ] && die "could not translate size '$size' to a usable value" + debug "----- $partition" debug "primary_count=$primary_count" - debug "available_size=$available_size" debug "current_device=$current_device" + debug "device=$device" + debug "type=$type" + debug "partition=$partition" + debug "minor=$minor" + debug "bootable=$bootable" + debug "size=$size" + debug "partition_size=$partition_size" + debug "available_size=$available_size" echo "create_partition $device $minor $partition_size $type $primary_count" done From a3086ae79580f8633c986e133cccb8d01a3ab142 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 17 Sep 2011 09:27:19 +0200 Subject: [PATCH 06/55] make it actually write the partition Signed-off-by: Steven Armstrong --- conf/type/__partition_msdos_apply/files/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__partition_msdos_apply/files/lib.sh b/conf/type/__partition_msdos_apply/files/lib.sh index 944533a6..0e9705d9 100644 --- a/conf/type/__partition_msdos_apply/files/lib.sh +++ b/conf/type/__partition_msdos_apply/files/lib.sh @@ -11,7 +11,7 @@ fdisk_command() { local cmd=$2 debug fdisk_command "running fdisk command '${cmd}' on device ${device}" - #echo -en "${cmd}\nw\n" | fdisk -c -u "$device" + echo -en "${cmd}\nw\n" | fdisk -c -u "$device" return $? } From 20bb4d044b6162bc11cd035829f64e6562f644a5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Sep 2011 13:29:08 +0200 Subject: [PATCH 07/55] --typo: .realines( vs. readlines( Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 5ce947ef..1f44ba2d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -684,7 +684,7 @@ def emulator(): sys.exit(1) else: param_fd = open(file, "r") - param_old = param_fd.realines() + param_old = param_fd.readlines() param_fd.close() if(param_old != param): From e3849b917c29cf6a86241a1fb2b1403e28b5def5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Sep 2011 14:34:19 +0200 Subject: [PATCH 08/55] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 84745512..9e9ca0e7 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -13,6 +13,10 @@ - Fix / rewrite cdist-quickstart +- write tutorial!!!!!!!!! + - like ccollect! + - include ssh control master! + -------------------------------------------------------------------------------- - Initial install support From 143a3a62dda6c7e08c0f3a66dc359507d634744b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Sep 2011 14:40:20 +0200 Subject: [PATCH 09/55] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 9e9ca0e7..44829060 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -16,6 +16,7 @@ - write tutorial!!!!!!!!! - like ccollect! - include ssh control master! + - add local/ hint (and add to git) -------------------------------------------------------------------------------- From 12e6e9288e85fc16f84ccbd3eac73e7cd08ca3e3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Sep 2011 14:45:19 +0200 Subject: [PATCH 10/55] ++tutorial Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 44829060..91dbc72f 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -17,6 +17,9 @@ - like ccollect! - include ssh control master! - add local/ hint (and add to git) + - add hint for ssh StrictHostKeyChecking no + - and that ssh will wait for answer of prompt + - nasty if used in parallel mode (scroll up!) -------------------------------------------------------------------------------- From 462ed49a74e2d8c512b85163d2cd2fafda4f4a09 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 22 Sep 2011 18:43:36 +0200 Subject: [PATCH 11/55] BUGFIX: TypeError: Can't convert 'list' object to str implicitly (in emulator) Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 1f44ba2d..9a745251 100755 --- a/bin/cdist +++ b/bin/cdist @@ -688,8 +688,8 @@ def emulator(): param_fd.close() if(param_old != param): - print("Parameter differs: " + param_old + "vs," + param) - print("Source = " + old_object_source + "new =" + object_source) + print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + print("Sources: " + " ".join(old_object_source) + " and " + object_source) sys.exit(1) else: param_fd = open(file, "w") From 7a09266abf391a98c3a9f836af4176e6d99c62a9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 13:49:02 +0200 Subject: [PATCH 12/55] allow users to check whether an object changed Signed-off-by: Nico Schottelius --- bin/cdist | 6 +++++- doc/changelog | 4 +++- doc/man/cdist-reference.text.sh | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 9a745251..e085a226 100755 --- a/bin/cdist +++ b/bin/cdist @@ -485,7 +485,7 @@ class Cdist: ] for bin in paths: if os.path.isfile(bin): - # omit "gen" from gencode and + # omit "gen" from gencode and use it for output base outfile=os.path.join(self.object_dir(cdist_object), os.path.basename(bin)[3:]) @@ -507,6 +507,10 @@ class Cdist: # Add header and make executable - identically to 0o700 os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + # Mark object as changed + open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() + + if mode == "code": local_dir = self.object_dir(cdist_object) remote_dir = self.remote_object_dir(cdist_object) diff --git a/doc/changelog b/doc/changelog index cc6aa78b..d69011ee 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,5 +1,7 @@ 2.0.1: - * Bugfix cdist: Always print source of error in case of exec errors + * Bugfix core: Always print source of error in case of exec errors + * Bugfix core: Various smaller bugs in string concatenation + * Feature: Add marker "changed" to changed objects 2.0.0: 2011-09-16 * New Type: __package_rubygem (Chase Allen James) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index e38f157d..c205bdcc 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -154,6 +154,16 @@ done cat << eof +OBJECTS +------- +For object to object communication and tests, the following paths are +usable within a object directory: + +changed:: + This empty file exists in an object directory, if the object has + code to be excuted (either remote or local) + + ENVIRONMENT VARIABLES --------------------- __explorer:: From ca3644b73a4ceb9ac0c156ca07d1b2b49851675d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 13:50:49 +0200 Subject: [PATCH 13/55] ++version Signed-off-by: Nico Schottelius --- doc/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changelog b/doc/changelog index d69011ee..b1149eb1 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,4 +1,4 @@ -2.0.1: +2.0.1: 2011-09-23 * Bugfix core: Always print source of error in case of exec errors * Bugfix core: Various smaller bugs in string concatenation * Feature: Add marker "changed" to changed objects From 412778206c15d929f26c567b36bb85557818cef1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 14:21:33 +0200 Subject: [PATCH 14/55] and increment version Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index e085a226..30db1ec9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -57,7 +57,7 @@ REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" TYPE_PREFIX = "__" -VERSION = "2.0.0" +VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() From 199245e6ce100718c47a40839930dd55ca1c9d21 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 14:54:39 +0200 Subject: [PATCH 15/55] ignore more pycache stuff Signed-off-by: Nico Schottelius --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d606aec7..259e3be3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ doc/man/man*/docbook-xsl.css cache/ # Python -bin/__pycache__/ +*/__pycache__/ From b8ff4c9609d61033deab83b4e3be082cde7b45fe Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 14:55:00 +0200 Subject: [PATCH 16/55] begin split into smaller files Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/cdist b/bin/cdist index 30db1ec9..fe3c17fd 100755 --- a/bin/cdist +++ b/bin/cdist @@ -62,6 +62,10 @@ VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() +# Begin to split into parts +sys.path.insert(0, + os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib/python'))) def file_to_list(filename): """Return list from \n seperated file""" From 211212e079fb1582658b4448e0c1ac2c0efdf0d1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 16:01:24 +0200 Subject: [PATCH 17/55] test template Signed-off-by: Nico Schottelius --- test/cdist.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 test/cdist.py diff --git a/test/cdist.py b/test/cdist.py new file mode 100644 index 00000000..3ccc69bc --- /dev/null +++ b/test/cdist.py @@ -0,0 +1,12 @@ +import cdist +import unittest + + +class CdistGeneric(unittest.TestCase): + + def test_initial_manifest(self): + self.assertEqual(numeral, result) + + +if __name__ == '__main__': + unittest.main() From ad5c33b7467e8224997183d48994b3b0de0ac9c4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 16:38:47 +0200 Subject: [PATCH 18/55] %s/\( \)/\1 /g + fix logo Signed-off-by: Nico Schottelius --- bin/cdist | 1348 ++++++++++++++++++++++++++--------------------------- 1 file changed, 674 insertions(+), 674 deletions(-) diff --git a/bin/cdist b/bin/cdist index fe3c17fd..2100bd4d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -48,745 +48,745 @@ BANNER = """ """ # Given paths from installation -REMOTE_BASE_DIR = "/var/lib/cdist" -REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") -REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") -REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") -CODE_HEADER = "#!/bin/sh -e\n" -DOT_CDIST = ".cdist" -TYPE_PREFIX = "__" -VERSION = "2.0.1" +CODE_HEADER = "#!/bin/sh -e\n" +DOT_CDIST = ".cdist" +TYPE_PREFIX = "__" +VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() # Begin to split into parts sys.path.insert(0, - os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib/python'))) + os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib/python'))) def file_to_list(filename): - """Return list from \n seperated file""" - if os.path.isfile(filename): - file_fd = open(filename, "r") - lines = file_fd.readlines() - file_fd.close() + """Return list from \n seperated file""" + if os.path.isfile(filename): + file_fd = open(filename, "r") + lines = file_fd.readlines() + file_fd.close() - # Remove \n from all lines - lines = map(lambda s: s.strip(), lines) - else: - lines = [] + # Remove \n from all lines + lines = map(lambda s: s.strip(), lines) + else: + lines = [] - return lines + return lines def exit_error(*args): - log.error(*args) - sys.exit(1) + log.error(*args) + sys.exit(1) class Cdist: - """Cdist main class to hold arbitrary data""" - - def __init__(self, target_host, - initial_manifest=False, remote_user="root", - home=None, debug=False): - self.target_host = target_host - self.remote_prefix = ["ssh", "root@" + self.target_host] - - # Setup directory paths - self.temp_dir = tempfile.mkdtemp() - - self.debug = debug - - if home: - self.base_dir = home - else: - self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - - self.conf_dir = os.path.join(self.base_dir, "conf") - self.cache_base_dir = os.path.join(self.base_dir, "cache") - self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) - self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") - self.lib_dir = os.path.join(self.base_dir, "lib") - self.manifest_dir = os.path.join(self.conf_dir, "manifest") - self.type_base_dir = os.path.join(self.conf_dir, "type") - - self.out_dir = os.path.join(self.temp_dir, "out") - os.mkdir(self.out_dir) - - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") - os.mkdir(self.global_explorer_out_dir) - - self.object_base_dir = os.path.join(self.out_dir, "object") - - # Setup binary directory + contents - self.bin_dir = os.path.join(self.out_dir, "bin") - os.mkdir(self.bin_dir) - self.link_type_to_emulator() - - # List of type explorers transferred - self.type_explorers_transferred = {} - - # objects - self.objects_prepared = [] - - self.remote_user = remote_user - - # Mostly static, but can be overwritten on user demand - if initial_manifest: - self.initial_manifest = initial_manifest - else: - self.initial_manifest = os.path.join(self.manifest_dir, "init") - - def cleanup(self): - # Do not use in __del__: - # http://docs.python.org/reference/datamodel.html#customization - # "other globals referenced by the __del__() method may already have been deleted - # or in the process of being torn down (e.g. the import machinery shutting down)" - # - log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) - # Remove previous cache - if os.path.exists(self.cache_dir): - shutil.rmtree(self.cache_dir) - shutil.move(self.temp_dir, self.cache_dir) - - def remote_mkdir(self, directory): - """Create directory on remote side""" - self.run_or_fail(["mkdir", "-p", directory], remote=True) - - def remote_cat(filename): - """Use cat on the remote side for output""" - self.run_or_fail(["cat", filename], remote=True) - - def shell_run_or_debug_fail(self, script, *args, **kargs): - # Manually execute /bin/sh, because sh -e does what we want - # and sh -c -e does not exit if /bin/false called - args[0][:0] = [ "/bin/sh", "-e" ] - - remote = False - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - remote = true - - del kargs["remote"] - - log.debug("Shell exec cmd: %s", args) - log.debug("Shell exec env: %s", kargs['env']) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - log.error("Code that raised the error:\n") - if remote: - remote_cat(script) - else: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() - - exit_error("Command failed (shell): " + " ".join(*args)) - except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) - - def run_or_fail(self, *args, **kargs): - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - - del kargs["remote"] - - log.debug("Exec: " + " ".join(*args)) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - exit_error("Command failed: " + " ".join(*args)) - except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) - - - def remove_remote_dir(self, destination): - self.run_or_fail(["rm", "-rf", destination], remote=True) - - def transfer_dir(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) - - def transfer_file(self, source, destination): - """Transfer file""" - self.run_or_fail(["scp", "-q", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) - - def global_explorer_output_path(self, explorer): - """Returns path of the output for a global explorer""" - return os.path.join(self.global_explorer_out_dir, explorer) - - def type_explorer_output_dir(self, cdist_object): - """Returns and creates dir of the output for a type explorer""" - dir = os.path.join(self.object_dir(cdist_object), "explorer") - if not os.path.isdir(dir): - os.mkdir(dir) - - return dir - - def remote_global_explorer_path(self, explorer): - """Returns path to the remote explorer""" - return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) - - def list_global_explorers(self): - """Return list of available explorers""" - return os.listdir(self.global_explorer_dir) - - def list_type_explorers(self, type): - """Return list of available explorers for a specific type""" - dir = self.type_dir(type, "explorer") - if os.path.isdir(dir): - list = os.listdir(dir) - else: - list = [] - - log.debug("Explorers for %s in %s: %s", type, dir, list) - - return list - - def list_types(self): - return os.listdir(self.type_base_dir) - - def list_object_paths(self, starting_point): - """Return list of paths of existing objects""" - object_paths = [] - - for content in os.listdir(starting_point): - full_path = os.path.join(starting_point, content) - if os.path.isdir(full_path): - object_paths.extend(self.list_object_paths(starting_point = full_path)) - - # Directory contains .cdist -> is an object - if content == DOT_CDIST: - object_paths.append(starting_point) - - return object_paths - - def get_type_from_object(self, cdist_object): - """Returns the first part (i.e. type) of an object""" - return cdist_object.split(os.sep)[0] - - def get_object_id_from_object(self, cdist_object): - """Returns everything but the first part (i.e. object_id) of an object""" - return os.sep.join(cdist_object.split(os.sep)[1:]) - - def object_dir(self, cdist_object): - """Returns the full path to the object (including .cdist)""" - return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) - - def remote_object_dir(self, cdist_object): - """Returns the remote full path to the object (including .cdist)""" - return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) - - def object_parameter_dir(self, cdist_object): - """Returns the dir to the object parameter""" - return os.path.join(self.object_dir(cdist_object), "parameter") - - def remote_object_parameter_dir(self, cdist_object): - """Returns the remote dir to the object parameter""" - return os.path.join(self.remote_object_dir(cdist_object), "parameter") - - def object_code_paths(self, cdist_object): - """Return paths to code scripts of object""" - return [os.path.join(self.object_dir(cdist_object), "code-local"), - os.path.join(self.object_dir(cdist_object), "code-remote")] - - def list_objects(self): - """Return list of existing objects""" - - objects = [] - if os.path.isdir(self.object_base_dir): - object_paths = self.list_object_paths(self.object_base_dir) - - for path in object_paths: - objects.append(os.path.relpath(path, self.object_base_dir)) - - return objects - - def type_dir(self, type, *args): - """Return directory the type""" - return os.path.join(self.type_base_dir, type, *args) - - def remote_type_explorer_dir(self, type): - """Return remote directory that holds the explorers of a type""" - return os.path.join(REMOTE_TYPE_DIR, type, "explorer") - - def transfer_object_parameter(self, cdist_object): - """Transfer the object parameter to the remote destination""" - # Create base path before using mkdir -p - self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) - - # Synchronise parameter dir afterwards - self.transfer_dir(self.object_parameter_dir(cdist_object), - self.remote_object_parameter_dir(cdist_object)) - - def transfer_global_explorers(self): - """Transfer the global explorers""" - self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) - self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) - - def transfer_type_explorers(self, type): - """Transfer explorers of a type, but only once""" - if type in self.type_explorers_transferred: - log.debug("Skipping retransfer for explorers of %s", type) - return - else: - # Do not retransfer - self.type_explorers_transferred[type] = 1 - - src = self.type_dir(type, "explorer") - remote_base = os.path.join(REMOTE_TYPE_DIR, type) - dst = self.remote_type_explorer_dir(type) - - # Only continue, if there is at least the directory - if os.path.isdir(src): - # Ensure that the path exists - self.remote_mkdir(remote_base) - self.transfer_dir(src, dst) - - - def link_type_to_emulator(self): - """Link type names to cdist-type-emulator""" - source = os.path.abspath(sys.argv[0]) - for type in self.list_types(): - destination = os.path.join(self.bin_dir, type) - log.debug("Linking %s to %s", source, destination) - os.symlink(source, destination) - - def run_global_explores(self): - """Run global explorers""" - explorers = self.list_global_explorers() - if(len(explorers) == 0): - exit_error("No explorers found in", self.global_explorer_dir) - - self.transfer_global_explorers() - for explorer in explorers: - output = self.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.remote_global_explorer_path(explorer)) - - self.run_or_fail(cmd, stdout=output_fd, remote=True) - output_fd.close() - - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" - # Based on bin/cdist-object-explorer-run - - # Transfering explorers for this type - type = self.get_type_from_object(cdist_object) - self.transfer_type_explorers(type) - - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) - - # Need to transfer at least the parameters for objects to be useful - self.transfer_object_parameter(cdist_object) - - explorers = self.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) - output_fd.close() - - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.remove_remote_dir(REMOTE_BASE_DIR) - self.remote_mkdir(REMOTE_BASE_DIR) - - def run_initial_manifest(self): - """Run the initial manifest""" - env = { "__manifest" : self.manifest_dir } - self.run_manifest(self.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.get_type_from_object(cdist_object) - manifest = self.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.object_dir(cdist_object), - "__object_id": self.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) - - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.bin_dir + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - - # Legacy stuff to make cdist-type-emulator work - env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") - env['__cdist_local_base_dir'] = self.temp_dir - - # Submit information to new type emulator - env['__cdist_manifest'] = manifest - env['__cdist_type_base_dir'] = self.type_base_dir - - # Other environment stuff - if extra_env: - env.update(extra_env) - - self.shell_run_or_debug_fail(manifest, [manifest], env=env) - - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.object_dir(cdist_object), "require") - requirements = file_to_list(file) - type = self.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - env["__object"] = self.object_dir(cdist_object) - env["__object_id"] = self.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.type_dir(type) - - if mode == "gencode": - paths = [ - self.type_dir(type, "gencode-local"), - self.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.object_dir(cdist_object), - os.path.basename(bin)[3:]) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - # Mark object as changed - open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() - - - if mode == "code": - local_dir = self.object_dir(cdist_object) - remote_dir = self.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - self.run_or_fail([bin], remote=False) - - - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.transfer_file(local_remote_code, remote_remote_code) - self.run_or_fail([remote_remote_code], remote=True) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - old_objects = [] - objects = self.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - log.debug("Prepare stage") - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue + """Cdist main class to hold arbitrary data""" + + def __init__(self, target_host, + initial_manifest=False, remote_user="root", + home=None, debug=False): + self.target_host = target_host + self.remote_prefix = ["ssh", "root@" + self.target_host] + + # Setup directory paths + self.temp_dir = tempfile.mkdtemp() + + self.debug = debug + + if home: + self.base_dir = home + else: + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + + self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) + self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") + + self.out_dir = os.path.join(self.temp_dir, "out") + os.mkdir(self.out_dir) + + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + os.mkdir(self.global_explorer_out_dir) + + self.object_base_dir = os.path.join(self.out_dir, "object") + + # Setup binary directory + contents + self.bin_dir = os.path.join(self.out_dir, "bin") + os.mkdir(self.bin_dir) + self.link_type_to_emulator() + + # List of type explorers transferred + self.type_explorers_transferred = {} + + # objects + self.objects_prepared = [] + + self.remote_user = remote_user + + # Mostly static, but can be overwritten on user demand + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(self.manifest_dir, "init") + + def cleanup(self): + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # + log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) + # Remove previous cache + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + shutil.move(self.temp_dir, self.cache_dir) + + def remote_mkdir(self, directory): + """Create directory on remote side""" + self.nun_or_fail(["mkdir", "-p", directory], remote=True) + + def remote_cat(filename): + """Use cat on the remote side for output""" + self.run_or_fail(["cat", filename], remote=True) + + def shell_run_or_debug_fail(self, script, *args, **kargs): + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0][:0] = [ "/bin/sh", "-e" ] + + remote = False + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + remote = true + + del kargs["remote"] + + log.debug("Shell exec cmd: %s", args) + log.debug("Shell exec env: %s", kargs['env']) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + log.error("Code that raised the error:\n") + if remote: + remote_cat(script) else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) + script_fd = open(script) + print(script_fd.read()) + script_fd.close() - objects = self.list_objects() + exit_error("Command failed (shell): " + " ".join(*args)) + except OSError as error: + exit_error(" ".join(*args) + ": " + error.args[1]) - def stage_run(self): - """The final (and real) step of deployment""" - log.debug("Actual run objects") - # Now do the final steps over the existing objects - for cdist_object in self.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") + def run_or_fail(self, *args, **kargs): + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() + del kargs["remote"] - self.stage_prepare() - self.stage_run() + log.debug("Exec: " + " ".join(*args)) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + exit_error("Command failed: " + " ".join(*args)) + except OSError as error: + exit_error(" ".join(*args) + ": " + error.args[1]) - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() + def remove_remote_dir(self, destination): + self.run_or_fail(["rm", "-rf", destination], remote=True) + + def transfer_dir(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_dir(destination) + self.run_or_fail(["scp", "-qr", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def transfer_file(self, source, destination): + """Transfer file""" + self.run_or_fail(["scp", "-q", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def global_explorer_output_path(self, explorer): + """Returns path of the output for a global explorer""" + return os.path.join(self.global_explorer_out_dir, explorer) + + def type_explorer_output_dir(self, cdist_object): + """Returns and creates dir of the output for a type explorer""" + dir = os.path.join(self.object_dir(cdist_object), "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + + return dir + + def remote_global_explorer_path(self, explorer): + """Returns path to the remote explorer""" + return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + + def list_global_explorers(self): + """Return list of available explorers""" + return os.listdir(self.global_explorer_dir) + + def list_type_explorers(self, type): + """Return list of available explorers for a specific type""" + dir = self.type_dir(type, "explorer") + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + log.debug("Explorers for %s in %s: %s", type, dir, list) + + return list + + def list_types(self): + return os.listdir(self.type_base_dir) + + def list_object_paths(self, starting_point): + """Return list of paths of existing objects""" + object_paths = [] + + for content in os.listdir(starting_point): + full_path = os.path.join(starting_point, content) + if os.path.isdir(full_path): + object_paths.extend(self.list_object_paths(starting_point = full_path)) + + # Directory contains .cdist -> is an object + if content == DOT_CDIST: + object_paths.append(starting_point) + + return object_paths + + def get_type_from_object(self, cdist_object): + """Returns the first part (i.e. type) of an object""" + return cdist_object.split(os.sep)[0] + + def get_object_id_from_object(self, cdist_object): + """Returns everything but the first part (i.e. object_id) of an object""" + return os.sep.join(cdist_object.split(os.sep)[1:]) + + def object_dir(self, cdist_object): + """Returns the full path to the object (including .cdist)""" + return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + + def remote_object_dir(self, cdist_object): + """Returns the remote full path to the object (including .cdist)""" + return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + + def object_parameter_dir(self, cdist_object): + """Returns the dir to the object parameter""" + return os.path.join(self.object_dir(cdist_object), "parameter") + + def remote_object_parameter_dir(self, cdist_object): + """Returns the remote dir to the object parameter""" + return os.path.join(self.remote_object_dir(cdist_object), "parameter") + + def object_code_paths(self, cdist_object): + """Return paths to code scripts of object""" + return [os.path.join(self.object_dir(cdist_object), "code-local"), + os.path.join(self.object_dir(cdist_object), "code-remote")] + + def list_objects(self): + """Return list of existing objects""" + + objects = [] + if os.path.isdir(self.object_base_dir): + object_paths = self.list_object_paths(self.object_base_dir) + + for path in object_paths: + objects.append(os.path.relpath(path, self.object_base_dir)) + + return objects + + def type_dir(self, type, *args): + """Return directory the type""" + return os.path.join(self.type_base_dir, type, *args) + + def remote_type_explorer_dir(self, type): + """Return remote directory that holds the explorers of a type""" + return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + + def transfer_object_parameter(self, cdist_object): + """Transfer the object parameter to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) + + def transfer_global_explorers(self): + """Transfer the global explorers""" + self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) + self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + + def transfer_type_explorers(self, type): + """Transfer explorers of a type, but only once""" + if type in self.type_explorers_transferred: + log.debug("Skipping retransfer for explorers of %s", type) + return + else: + # Do not retransfer + self.type_explorers_transferred[type] = 1 + + src = self.type_dir(type, "explorer") + remote_base = os.path.join(REMOTE_TYPE_DIR, type) + dst = self.remote_type_explorer_dir(type) + + # Only continue, if there is at least the directory + if os.path.isdir(src): + # Ensure that the path exists + self.remote_mkdir(remote_base) + self.transfer_dir(src, dst) + + + def link_type_to_emulator(self): + """Link type names to cdist-type-emulator""" + source = os.path.abspath(sys.argv[0]) + for type in self.list_types(): + destination = os.path.join(self.bin_dir, type) + log.debug("Linking %s to %s", source, destination) + os.symlink(source, destination) + + def run_global_explores(self): + """Run global explorers""" + explorers = self.list_global_explorers() + if(len(explorers) == 0): + exit_error("No explorers found in", self.global_explorer_dir) + + self.transfer_global_explorers() + for explorer in explorers: + output = self.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.remote_global_explorer_path(explorer)) + + self.run_or_fail(cmd, stdout=output_fd, remote=True) + output_fd.close() + + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + # Based on bin/cdist-object-explorer-run + + # Transfering explorers for this type + type = self.get_type_from_object(cdist_object) + self.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.transfer_object_parameter(cdist_object) + + explorers = self.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) + output_fd.close() + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.remove_remote_dir(REMOTE_BASE_DIR) + self.remote_mkdir(REMOTE_BASE_DIR) + + def run_initial_manifest(self): + """Run the initial manifest""" + env = { "__manifest" : self.manifest_dir } + self.run_manifest(self.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.get_type_from_object(cdist_object) + manifest = self.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.object_dir(cdist_object), + "__object_id": self.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + + # Legacy stuff to make cdist-type-emulator work + env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") + env['__cdist_local_base_dir'] = self.temp_dir + + # Submit information to new type emulator + env['__cdist_manifest'] = manifest + env['__cdist_type_base_dir'] = self.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.object_dir(cdist_object), "require") + requirements = file_to_list(file) + type = self.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + env["__object"] = self.object_dir(cdist_object) + env["__object_id"] = self.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.type_dir(type) + + if mode == "gencode": + paths = [ + self.type_dir(type, "gencode-local"), + self.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.object_dir(cdist_object) + remote_dir = self.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + self.run_or_fail([bin], remote=False) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.transfer_file(local_remote_code, remote_remote_code) + self.run_or_fail([remote_remote_code], remote=True) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + old_objects = [] + objects = self.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + log.debug("Prepare stage") + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.debug("Actual run objects") + # Now do the final steps over the existing objects + for cdist_object in self.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() def banner(args): - """Guess what :-)""" - print(BANNER) - sys.exit(0) + """Guess what :-)""" + print(BANNER) + sys.exit(0) def config(args): - """Configure remote system""" - process = {} + """Configure remote system""" + process = {} - time_start = datetime.datetime.now() + time_start = datetime.datetime.now() - for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) - if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) - process[host].start() - else: - c.deploy_and_cleanup() + for host in args.host: + c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) + if args.parallel: + log.debug("Creating child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + else: + c.deploy_and_cleanup() - if args.parallel: - for p in process.keys(): - log.debug("Joining %s", p) - process[p].join() + if args.parallel: + for p in process.keys(): + log.debug("Joining %s", p) + process[p].join() - time_end = datetime.datetime.now() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start).total_seconds()) + time_end = datetime.datetime.now() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) def install(args): - """Install remote system""" - process = {} + """Install remote system""" + process = {} def emulator(): - """Emulate type commands (i.e. __file and co)""" - type = os.path.basename(sys.argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) - param_dir = os.path.join(type_dir, "parameter") - global_dir = os.environ['__global'] - object_source = os.environ['__cdist_manifest'] + """Emulate type commands (i.e. __file and co)""" + type = os.path.basename(sys.argv[0]) + type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) + param_dir = os.path.join(type_dir, "parameter") + global_dir = os.environ['__global'] + object_source = os.environ['__cdist_manifest'] - parser = argparse.ArgumentParser(add_help=False) + parser = argparse.ArgumentParser(add_help=False) - # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=False) + # Setup optional parameters + for parameter in file_to_list(os.path.join(param_dir, "optional")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=False) - # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=True) + # Setup required parameters + for parameter in file_to_list(os.path.join(param_dir, "required")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=True) - # Setup positional parameter, if not singleton + # Setup positional parameter, if not singleton - if not os.path.isfile(os.path.join(type_dir, "singleton")): - parser.add_argument("object_id", nargs=1) + if not os.path.isfile(os.path.join(type_dir, "singleton")): + parser.add_argument("object_id", nargs=1) - # And finally verify parameter - args = parser.parse_args(sys.argv[1:]) + # And finally verify parameter + args = parser.parse_args(sys.argv[1:]) - # Setup object_id - if os.path.isfile(os.path.join(type_dir, "singleton")): - object_id = "singleton" - else: - object_id = args.object_id[0] - del args.object_id + # Setup object_id + if os.path.isfile(os.path.join(type_dir, "singleton")): + object_id = "singleton" + else: + object_id = args.object_id[0] + del args.object_id - # FIXME: / hardcoded - better portable solution available? - if object_id[0] == '/': - object_id = object_id[1:] + # FIXME: / hardcoded - better portable solution available? + if object_id[0] == '/': + object_id = object_id[1:] - # FIXME: verify object id - log.debug(args) + # FIXME: verify object id + log.debug(args) - object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) - param_out_dir = os.path.join(object_dir, "parameter") + object_dir = os.path.join(global_dir, "object", type, + object_id, DOT_CDIST) + param_out_dir = os.path.join(object_dir, "parameter") - object_source_file = os.path.join(object_dir, "source") + object_source_file = os.path.join(object_dir, "source") - if os.path.exists(param_out_dir): - object_exists = True - old_object_source_fd = open(object_source_file, "r") - old_object_source = old_object_source_fd.readlines() - old_object_source_fd.close() + if os.path.exists(param_out_dir): + object_exists = True + old_object_source_fd = open(object_source_file, "r") + old_object_source = old_object_source_fd.readlines() + old_object_source_fd.close() - else: - object_exists = False - try: - os.makedirs(param_out_dir, exist_ok=True) - except OSError as error: - exit_error(param_out_dir + ": " + error.args[1]) + else: + object_exists = False + try: + os.makedirs(param_out_dir, exist_ok=True) + except OSError as error: + exit_error(param_out_dir + ": " + error.args[1]) - # Record parameter - params = vars(args) - for param in params: - value = getattr(args, param) - if value: - file = os.path.join(param_out_dir, param) - log.debug(file + "<-" + param + " = " + value) + # Record parameter + params = vars(args) + for param in params: + value = getattr(args, param) + if value: + file = os.path.join(param_out_dir, param) + log.debug(file + "<-" + param + " = " + value) - # Already exists, verify all parameter are the same - if object_exists: - if not os.path.isfile(file): - print("New parameter + " + param + "specified, aborting") - print("Source = " + old_object_source + "new =" + object_source) - sys.exit(1) + # Already exists, verify all parameter are the same + if object_exists: + if not os.path.isfile(file): + print("New parameter + " + param + "specified, aborting") + print("Source = " + old_object_source + "new =" + object_source) + sys.exit(1) + else: + param_fd = open(file, "r") + param_old = param_fd.readlines() + param_fd.close() + + if(param_old != param): + print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + print("Sources: " + " ".join(old_object_source) + " and " + object_source) + sys.exit(1) else: - param_fd = open(file, "r") - param_old = param_fd.readlines() - param_fd.close() - - if(param_old != param): - print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) - print("Sources: " + " ".join(old_object_source) + " and " + object_source) - sys.exit(1) - else: - param_fd = open(file, "w") - param_fd.writelines(value) - param_fd.close() + param_fd = open(file, "w") + param_fd.writelines(value) + param_fd.close() - # Record requirements - if "__require" in os.environ: - requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) - require_fd = open(os.path.join(object_dir, "require"), "a") - require_fd.writelines(requirements.split(" ")) - require_fd.close() + # Record requirements + if "__require" in os.environ: + requirements = os.environ['__require'] + print(object_id + ":Writing requirements: " + requirements) + require_fd = open(os.path.join(object_dir, "require"), "a") + require_fd.writelines(requirements.split(" ")) + require_fd.close() - # Record / Append source - source_fd = open(os.path.join(object_dir, "source"), "a") - source_fd.writelines(object_source) - source_fd.close() + # Record / Append source + source_fd = open(os.path.join(object_dir, "source"), "a") + source_fd.writelines(object_source) + source_fd.close() - # sys.exit(1) - print("Finished " + type + "/" + object_id + repr(params)) + # sys.exit(1) + print("Finished " + type + "/" + object_id + repr(params)) def commandline(): - """Parse command line""" - # Construct parser others can reuse - parser = {} - # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') + """Parse command line""" + # Construct parser others can reuse + parser = {} + # Options _all_ parsers have in common + parser['most'] = argparse.ArgumentParser(add_help=False) + parser['most'].add_argument('-d', '--debug', + help='Set log level to debug', action='store_true') - # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) - parser['main'].add_argument('-V', '--version', - help='Show version', action='version', - version='%(prog)s ' + VERSION) - parser['sub'] = parser['main'].add_subparsers(title="Commands") + # Main subcommand parser + parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) + parser['main'].add_argument('-V', '--version', + help='Show version', action='version', + version='%(prog)s ' + VERSION) + parser['sub'] = parser['main'].add_subparsers(title="Commands") - # Banner - parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) - parser['banner'].set_defaults(func=banner) + # Banner + parser['banner'] = parser['sub'].add_parser('banner', + add_help=False) + parser['banner'].set_defaults(func=banner) - # Config and install (common stuff) - parser['configinstall'] = argparse.ArgumentParser(add_help=False) - parser['configinstall'].add_argument('host', nargs='+', - help='one or more hosts to operate on') - parser['configinstall'].add_argument('-c', '--cdist-home', - help='Change cdist home (default: .. from bin directory)', - action='store') - parser['configinstall'].add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest', - dest='manifest', required=False) - parser['configinstall'].add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['configinstall'].add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') + # Config and install (common stuff) + parser['configinstall'] = argparse.ArgumentParser(add_help=False) + parser['configinstall'].add_argument('host', nargs='+', + help='one or more hosts to operate on') + parser['configinstall'].add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store') + parser['configinstall'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest', + dest='manifest', required=False) + parser['configinstall'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['configinstall'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') - # Config - parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) - parser['config'].set_defaults(func=config) + # Config + parser['config'] = parser['sub'].add_parser('config', + parents=[parser['most'], parser['configinstall']]) + parser['config'].set_defaults(func=config) - # Install - parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) - parser['install'].set_defaults(func=install) + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['most'], parser['configinstall']]) + parser['install'].set_defaults(func=install) - for p in parser: - parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" + for p in parser: + parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" - args = parser['main'].parse_args(sys.argv[1:]) + args = parser['main'].parse_args(sys.argv[1:]) - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) + # Most subcommands have --debug, so handle it here + if 'debug' in args: + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) - args.func(args) + args.func(args) if __name__ == "__main__": - try: - if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - emulator() - else: - commandline() - except KeyboardInterrupt: - sys.exit(0) + try: + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): + emulator() + else: + commandline() + except KeyboardInterrupt: + sys.exit(0) From f4da3b96d8a6fc97b241b43eb0c2520798507926 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 16:41:55 +0200 Subject: [PATCH 19/55] fix indentions Signed-off-by: Nico Schottelius --- bin/cdist | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/cdist b/bin/cdist index 2100bd4d..f3b88fdd 100755 --- a/bin/cdist +++ b/bin/cdist @@ -48,16 +48,16 @@ BANNER = """ """ # Given paths from installation -REMOTE_BASE_DIR = "/var/lib/cdist" -REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") -REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") -REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") -REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") +REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") CODE_HEADER = "#!/bin/sh -e\n" -DOT_CDIST = ".cdist" +DOT_CDIST = ".cdist" TYPE_PREFIX = "__" -VERSION = "2.0.1" +VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() @@ -104,13 +104,13 @@ class Cdist: else: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - self.conf_dir = os.path.join(self.base_dir, "conf") - self.cache_base_dir = os.path.join(self.base_dir, "cache") - self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) + self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") - self.lib_dir = os.path.join(self.base_dir, "lib") - self.manifest_dir = os.path.join(self.conf_dir, "manifest") - self.type_base_dir = os.path.join(self.conf_dir, "type") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") self.out_dir = os.path.join(self.temp_dir, "out") os.mkdir(self.out_dir) From ea9dc8d60c648448c42f765fcab265c33cd55c0a Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 23 Sep 2011 16:43:25 +0200 Subject: [PATCH 20/55] handle errors with exceptions instead of function Signed-off-by: Steven Armstrong --- bin/cdist | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bin/cdist b/bin/cdist index fe3c17fd..e3d63458 100755 --- a/bin/cdist +++ b/bin/cdist @@ -81,9 +81,9 @@ def file_to_list(filename): return lines -def exit_error(*args): - log.error(*args) - sys.exit(1) +class CdistError(Exception): + """Base exception class for this project""" + pass class Cdist: """Cdist main class to hold arbitrary data""" @@ -185,9 +185,9 @@ class Cdist: print(script_fd.read()) script_fd.close() - exit_error("Command failed (shell): " + " ".join(*args)) + raise CdistError("Command failed (shell): " + " ".join(*args)) except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) + raise CdistError(" ".join(*args) + ": " + error.args[1]) def run_or_fail(self, *args, **kargs): if "remote" in kargs: @@ -200,9 +200,9 @@ class Cdist: try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - exit_error("Command failed: " + " ".join(*args)) + raise CdistError("Command failed: " + " ".join(*args)) except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) + raise CdistError(" ".join(*args) + ": " + error.args[1]) def remove_remote_dir(self, destination): @@ -368,7 +368,7 @@ class Cdist: """Run global explorers""" explorers = self.list_global_explorers() if(len(explorers) == 0): - exit_error("No explorers found in", self.global_explorer_dir) + raise CdistError("No explorers found in", self.global_explorer_dir) self.transfer_global_explorers() for explorer in explorers: @@ -674,7 +674,7 @@ def emulator(): try: os.makedirs(param_out_dir, exist_ok=True) except OSError as error: - exit_error(param_out_dir + ": " + error.args[1]) + raise CdistError(param_out_dir + ": " + error.args[1]) # Record parameter params = vars(args) @@ -790,3 +790,6 @@ if __name__ == "__main__": commandline() except KeyboardInterrupt: sys.exit(0) + except CdistError as e: + log.error(e) + sys.exit(1) From b72fab3c01804b19f00744321f8e8e854e10c290 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:00:48 +0200 Subject: [PATCH 21/55] begin split into path module Signed-off-by: Nico Schottelius --- module/cdist/__init__.py | 0 module/cdist/path.py | 11 +++++++++++ 2 files changed, 11 insertions(+) create mode 100644 module/cdist/__init__.py create mode 100644 module/cdist/path.py diff --git a/module/cdist/__init__.py b/module/cdist/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/module/cdist/path.py b/module/cdist/path.py new file mode 100644 index 00000000..37d92940 --- /dev/null +++ b/module/cdist/path.py @@ -0,0 +1,11 @@ +class Path: + """Class that handles path related configurations""" + + def __init__(self, home=None): + if home: + self.base_dir = home + else: + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + + + print("Base:" + self.base_dir) From 3ace43ff01528717528034bb827ccc03b056b715 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:00:59 +0200 Subject: [PATCH 22/55] ignore pycache everywhere Signed-off-by: Nico Schottelius --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 259e3be3..bd4ecaaa 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ doc/man/man*/docbook-xsl.css cache/ # Python -*/__pycache__/ +**/__pycache__/ From a2f3246758cf954e1eecf682c16992cbf9a6026d Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 23 Sep 2011 17:03:00 +0200 Subject: [PATCH 23/55] fix typo /nun_or_fail/run_or_fail/ Signed-off-by: Steven Armstrong --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 0aa661a3..fde7a251 100755 --- a/bin/cdist +++ b/bin/cdist @@ -153,7 +153,7 @@ class Cdist: def remote_mkdir(self, directory): """Create directory on remote side""" - self.nun_or_fail(["mkdir", "-p", directory], remote=True) + self.run_or_fail(["mkdir", "-p", directory], remote=True) def remote_cat(filename): """Use cat on the remote side for output""" From 747517f06782db10da115f3f66f2115cf7e4fa48 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:06:42 +0200 Subject: [PATCH 24/55] import path information from current cdist Signed-off-by: Nico Schottelius --- module/cdist/path.py | 56 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/module/cdist/path.py b/module/cdist/path.py index 37d92940..ee492cc5 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -1,11 +1,59 @@ class Path: """Class that handles path related configurations""" - def __init__(self, home=None): + def __init__(self, target_host, base_dir=None): + # Base and Temp Base if home: - self.base_dir = home + self.base_dir = base_dir else: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - - print("Base:" + self.base_dir) + self.temp_dir = tempfile.mkdtemp() + + self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, target_host) + self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") + + self.out_dir = os.path.join(self.temp_dir, "out") + os.mkdir(self.out_dir) + + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + os.mkdir(self.global_explorer_out_dir) + + self.object_base_dir = os.path.join(self.out_dir, "object") + + # Setup binary directory + contents + self.bin_dir = os.path.join(self.out_dir, "bin") + os.mkdir(self.bin_dir) + self.link_type_to_emulator() + + # List of type explorers transferred + self.type_explorers_transferred = {} + + # objects + self.objects_prepared = [] + + self.remote_user = remote_user + + # Mostly static, but can be overwritten on user demand + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(self.manifest_dir, "init") + + def cleanup(self): + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # + log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) + # Remove previous cache + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + shutil.move(self.temp_dir, self.cache_dir) + From 26772939ca8dce14507358f100cbfd9bdeacdb4d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:11:03 +0200 Subject: [PATCH 25/55] import whole cdist into path.py and strip down (most stuff is moved here anyway) Signed-off-by: Nico Schottelius --- module/cdist/path.py | 665 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 665 insertions(+) diff --git a/module/cdist/path.py b/module/cdist/path.py index ee492cc5..387481ca 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -1,3 +1,28 @@ +# Given paths from installation +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") +REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") + +CODE_HEADER = "#!/bin/sh -e\n" +DOT_CDIST = ".cdist" +TYPE_PREFIX = "__" + +def file_to_list(filename): + """Return list from \n seperated file""" + if os.path.isfile(filename): + file_fd = open(filename, "r") + lines = file_fd.readlines() + file_fd.close() + + # Remove \n from all lines + lines = map(lambda s: s.strip(), lines) + else: + lines = [] + + return lines + class Path: """Class that handles path related configurations""" @@ -57,3 +82,643 @@ class Path: shutil.rmtree(self.cache_dir) shutil.move(self.temp_dir, self.cache_dir) + + def remote_mkdir(self, directory): + """Create directory on remote side""" + self.run_or_fail(["mkdir", "-p", directory], remote=True) + + def remote_cat(filename): + """Use cat on the remote side for output""" + self.run_or_fail(["cat", filename], remote=True) + + def shell_run_or_debug_fail(self, script, *args, **kargs): + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0][:0] = [ "/bin/sh", "-e" ] + + remote = False + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + remote = true + + del kargs["remote"] + + log.debug("Shell exec cmd: %s", args) + log.debug("Shell exec env: %s", kargs['env']) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + log.error("Code that raised the error:\n") + if remote: + remote_cat(script) + else: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + + exit_error("Command failed (shell): " + " ".join(*args)) + except OSError as error: + exit_error(" ".join(*args) + ": " + error.args[1]) + + def run_or_fail(self, *args, **kargs): + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + + del kargs["remote"] + + log.debug("Exec: " + " ".join(*args)) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + exit_error("Command failed: " + " ".join(*args)) + except OSError as error: + exit_error(" ".join(*args) + ": " + error.args[1]) + + + def remove_remote_dir(self, destination): + self.run_or_fail(["rm", "-rf", destination], remote=True) + + def transfer_dir(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_dir(destination) + self.run_or_fail(["scp", "-qr", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def transfer_file(self, source, destination): + """Transfer file""" + self.run_or_fail(["scp", "-q", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def global_explorer_output_path(self, explorer): + """Returns path of the output for a global explorer""" + return os.path.join(self.global_explorer_out_dir, explorer) + + def type_explorer_output_dir(self, cdist_object): + """Returns and creates dir of the output for a type explorer""" + dir = os.path.join(self.object_dir(cdist_object), "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + + return dir + + def remote_global_explorer_path(self, explorer): + """Returns path to the remote explorer""" + return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + + def list_global_explorers(self): + """Return list of available explorers""" + return os.listdir(self.global_explorer_dir) + + def list_type_explorers(self, type): + """Return list of available explorers for a specific type""" + dir = self.type_dir(type, "explorer") + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + log.debug("Explorers for %s in %s: %s", type, dir, list) + + return list + + def list_types(self): + return os.listdir(self.type_base_dir) + + def list_object_paths(self, starting_point): + """Return list of paths of existing objects""" + object_paths = [] + + for content in os.listdir(starting_point): + full_path = os.path.join(starting_point, content) + if os.path.isdir(full_path): + object_paths.extend(self.list_object_paths(starting_point = full_path)) + + # Directory contains .cdist -> is an object + if content == DOT_CDIST: + object_paths.append(starting_point) + + return object_paths + + def get_type_from_object(self, cdist_object): + """Returns the first part (i.e. type) of an object""" + return cdist_object.split(os.sep)[0] + + def get_object_id_from_object(self, cdist_object): + """Returns everything but the first part (i.e. object_id) of an object""" + return os.sep.join(cdist_object.split(os.sep)[1:]) + + def object_dir(self, cdist_object): + """Returns the full path to the object (including .cdist)""" + return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + + def remote_object_dir(self, cdist_object): + """Returns the remote full path to the object (including .cdist)""" + return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + + def object_parameter_dir(self, cdist_object): + """Returns the dir to the object parameter""" + return os.path.join(self.object_dir(cdist_object), "parameter") + + def remote_object_parameter_dir(self, cdist_object): + """Returns the remote dir to the object parameter""" + return os.path.join(self.remote_object_dir(cdist_object), "parameter") + + def object_code_paths(self, cdist_object): + """Return paths to code scripts of object""" + return [os.path.join(self.object_dir(cdist_object), "code-local"), + os.path.join(self.object_dir(cdist_object), "code-remote")] + + def list_objects(self): + """Return list of existing objects""" + + objects = [] + if os.path.isdir(self.object_base_dir): + object_paths = self.list_object_paths(self.object_base_dir) + + for path in object_paths: + objects.append(os.path.relpath(path, self.object_base_dir)) + + return objects + + def type_dir(self, type, *args): + """Return directory the type""" + return os.path.join(self.type_base_dir, type, *args) + + def remote_type_explorer_dir(self, type): + """Return remote directory that holds the explorers of a type""" + return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + + def transfer_object_parameter(self, cdist_object): + """Transfer the object parameter to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) + + def transfer_global_explorers(self): + """Transfer the global explorers""" + self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) + self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + + def transfer_type_explorers(self, type): + """Transfer explorers of a type, but only once""" + if type in self.type_explorers_transferred: + log.debug("Skipping retransfer for explorers of %s", type) + return + else: + # Do not retransfer + self.type_explorers_transferred[type] = 1 + + src = self.type_dir(type, "explorer") + remote_base = os.path.join(REMOTE_TYPE_DIR, type) + dst = self.remote_type_explorer_dir(type) + + # Only continue, if there is at least the directory + if os.path.isdir(src): + # Ensure that the path exists + self.remote_mkdir(remote_base) + self.transfer_dir(src, dst) + + + def link_type_to_emulator(self): + """Link type names to cdist-type-emulator""" + source = os.path.abspath(sys.argv[0]) + for type in self.list_types(): + destination = os.path.join(self.bin_dir, type) + log.debug("Linking %s to %s", source, destination) + os.symlink(source, destination) + + def run_global_explores(self): + """Run global explorers""" + explorers = self.list_global_explorers() + if(len(explorers) == 0): + exit_error("No explorers found in", self.global_explorer_dir) + + self.transfer_global_explorers() + for explorer in explorers: + output = self.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.remote_global_explorer_path(explorer)) + + self.run_or_fail(cmd, stdout=output_fd, remote=True) + output_fd.close() + + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + # Based on bin/cdist-object-explorer-run + + # Transfering explorers for this type + type = self.get_type_from_object(cdist_object) + self.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.transfer_object_parameter(cdist_object) + + explorers = self.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) + output_fd.close() + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.remove_remote_dir(REMOTE_BASE_DIR) + self.remote_mkdir(REMOTE_BASE_DIR) + + def run_initial_manifest(self): + """Run the initial manifest""" + env = { "__manifest" : self.manifest_dir } + self.run_manifest(self.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.get_type_from_object(cdist_object) + manifest = self.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.object_dir(cdist_object), + "__object_id": self.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + + # Legacy stuff to make cdist-type-emulator work + env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") + env['__cdist_local_base_dir'] = self.temp_dir + + # Submit information to new type emulator + env['__cdist_manifest'] = manifest + env['__cdist_type_base_dir'] = self.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.object_dir(cdist_object), "require") + requirements = file_to_list(file) + type = self.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + env["__object"] = self.object_dir(cdist_object) + env["__object_id"] = self.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.type_dir(type) + + if mode == "gencode": + paths = [ + self.type_dir(type, "gencode-local"), + self.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.object_dir(cdist_object) + remote_dir = self.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + self.run_or_fail([bin], remote=False) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.transfer_file(local_remote_code, remote_remote_code) + self.run_or_fail([remote_remote_code], remote=True) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + old_objects = [] + objects = self.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + log.debug("Prepare stage") + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.debug("Actual run objects") + # Now do the final steps over the existing objects + for cdist_object in self.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() + +def banner(args): + """Guess what :-)""" + print(BANNER) + sys.exit(0) + +def config(args): + """Configure remote system""" + process = {} + + time_start = datetime.datetime.now() + + for host in args.host: + c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) + if args.parallel: + log.debug("Creating child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + else: + c.deploy_and_cleanup() + + if args.parallel: + for p in process.keys(): + log.debug("Joining %s", p) + process[p].join() + + time_end = datetime.datetime.now() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) + +def install(args): + """Install remote system""" + process = {} + +def emulator(): + """Emulate type commands (i.e. __file and co)""" + type = os.path.basename(sys.argv[0]) + type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) + param_dir = os.path.join(type_dir, "parameter") + global_dir = os.environ['__global'] + object_source = os.environ['__cdist_manifest'] + + parser = argparse.ArgumentParser(add_help=False) + + # Setup optional parameters + for parameter in file_to_list(os.path.join(param_dir, "optional")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=False) + + # Setup required parameters + for parameter in file_to_list(os.path.join(param_dir, "required")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=True) + + # Setup positional parameter, if not singleton + + if not os.path.isfile(os.path.join(type_dir, "singleton")): + parser.add_argument("object_id", nargs=1) + + # And finally verify parameter + args = parser.parse_args(sys.argv[1:]) + + # Setup object_id + if os.path.isfile(os.path.join(type_dir, "singleton")): + object_id = "singleton" + else: + object_id = args.object_id[0] + del args.object_id + + # FIXME: / hardcoded - better portable solution available? + if object_id[0] == '/': + object_id = object_id[1:] + + # FIXME: verify object id + log.debug(args) + + object_dir = os.path.join(global_dir, "object", type, + object_id, DOT_CDIST) + param_out_dir = os.path.join(object_dir, "parameter") + + object_source_file = os.path.join(object_dir, "source") + + if os.path.exists(param_out_dir): + object_exists = True + old_object_source_fd = open(object_source_file, "r") + old_object_source = old_object_source_fd.readlines() + old_object_source_fd.close() + + else: + object_exists = False + try: + os.makedirs(param_out_dir, exist_ok=True) + except OSError as error: + exit_error(param_out_dir + ": " + error.args[1]) + + # Record parameter + params = vars(args) + for param in params: + value = getattr(args, param) + if value: + file = os.path.join(param_out_dir, param) + log.debug(file + "<-" + param + " = " + value) + + # Already exists, verify all parameter are the same + if object_exists: + if not os.path.isfile(file): + print("New parameter + " + param + "specified, aborting") + print("Source = " + old_object_source + "new =" + object_source) + sys.exit(1) + else: + param_fd = open(file, "r") + param_old = param_fd.readlines() + param_fd.close() + + if(param_old != param): + print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + print("Sources: " + " ".join(old_object_source) + " and " + object_source) + sys.exit(1) + else: + param_fd = open(file, "w") + param_fd.writelines(value) + param_fd.close() + + # Record requirements + if "__require" in os.environ: + requirements = os.environ['__require'] + print(object_id + ":Writing requirements: " + requirements) + require_fd = open(os.path.join(object_dir, "require"), "a") + require_fd.writelines(requirements.split(" ")) + require_fd.close() + + # Record / Append source + source_fd = open(os.path.join(object_dir, "source"), "a") + source_fd.writelines(object_source) + source_fd.close() + + # sys.exit(1) + print("Finished " + type + "/" + object_id + repr(params)) + + +def commandline(): + """Parse command line""" + # Construct parser others can reuse + parser = {} + # Options _all_ parsers have in common + parser['most'] = argparse.ArgumentParser(add_help=False) + parser['most'].add_argument('-d', '--debug', + help='Set log level to debug', action='store_true') + + # Main subcommand parser + parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) + parser['main'].add_argument('-V', '--version', + help='Show version', action='version', + version='%(prog)s ' + VERSION) + parser['sub'] = parser['main'].add_subparsers(title="Commands") + + # Banner + parser['banner'] = parser['sub'].add_parser('banner', + add_help=False) + parser['banner'].set_defaults(func=banner) + + # Config and install (common stuff) + parser['configinstall'] = argparse.ArgumentParser(add_help=False) + parser['configinstall'].add_argument('host', nargs='+', + help='one or more hosts to operate on') + parser['configinstall'].add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store') + parser['configinstall'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest', + dest='manifest', required=False) + parser['configinstall'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['configinstall'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') + + # Config + parser['config'] = parser['sub'].add_parser('config', + parents=[parser['most'], parser['configinstall']]) + parser['config'].set_defaults(func=config) + + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['most'], parser['configinstall']]) + parser['install'].set_defaults(func=install) + + for p in parser: + parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" + + args = parser['main'].parse_args(sys.argv[1:]) + + # Most subcommands have --debug, so handle it here + if 'debug' in args: + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) + + args.func(args) + +if __name__ == "__main__": + try: + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): + emulator() + else: + commandline() + except KeyboardInterrupt: + sys.exit(0) From 02b9b71ffeb38e26af06b7cf06dc8e9e494f08f9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:11:39 +0200 Subject: [PATCH 26/55] remove exec functions from path Signed-off-by: Nico Schottelius --- module/cdist/path.py | 46 -------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/module/cdist/path.py b/module/cdist/path.py index 387481ca..964e29b8 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -91,52 +91,6 @@ class Path: """Use cat on the remote side for output""" self.run_or_fail(["cat", filename], remote=True) - def shell_run_or_debug_fail(self, script, *args, **kargs): - # Manually execute /bin/sh, because sh -e does what we want - # and sh -c -e does not exit if /bin/false called - args[0][:0] = [ "/bin/sh", "-e" ] - - remote = False - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - remote = true - - del kargs["remote"] - - log.debug("Shell exec cmd: %s", args) - log.debug("Shell exec env: %s", kargs['env']) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - log.error("Code that raised the error:\n") - if remote: - remote_cat(script) - else: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() - - exit_error("Command failed (shell): " + " ".join(*args)) - except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) - - def run_or_fail(self, *args, **kargs): - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - - del kargs["remote"] - - log.debug("Exec: " + " ".join(*args)) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - exit_error("Command failed: " + " ".join(*args)) - except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) - - def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) From 7bc4f74d50d0ff115e62981d17ed6b732590fa69 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:17:59 +0200 Subject: [PATCH 27/55] remove non path stuff from path Signed-off-by: Nico Schottelius --- module/cdist/path.py | 431 +------------------------------------------ 1 file changed, 4 insertions(+), 427 deletions(-) diff --git a/module/cdist/path.py b/module/cdist/path.py index 964e29b8..c72d29a8 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -23,6 +23,9 @@ def file_to_list(filename): return lines +# FIXME: self.run_or_fail needs to be elsewhere! +# Exec? + class Path: """Class that handles path related configurations""" @@ -159,6 +162,7 @@ class Path: return object_paths + # FIXME def get_type_from_object(self, cdist_object): """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] @@ -249,430 +253,3 @@ class Path: destination = os.path.join(self.bin_dir, type) log.debug("Linking %s to %s", source, destination) os.symlink(source, destination) - - def run_global_explores(self): - """Run global explorers""" - explorers = self.list_global_explorers() - if(len(explorers) == 0): - exit_error("No explorers found in", self.global_explorer_dir) - - self.transfer_global_explorers() - for explorer in explorers: - output = self.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.remote_global_explorer_path(explorer)) - - self.run_or_fail(cmd, stdout=output_fd, remote=True) - output_fd.close() - - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" - # Based on bin/cdist-object-explorer-run - - # Transfering explorers for this type - type = self.get_type_from_object(cdist_object) - self.transfer_type_explorers(type) - - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) - - # Need to transfer at least the parameters for objects to be useful - self.transfer_object_parameter(cdist_object) - - explorers = self.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) - output_fd.close() - - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.remove_remote_dir(REMOTE_BASE_DIR) - self.remote_mkdir(REMOTE_BASE_DIR) - - def run_initial_manifest(self): - """Run the initial manifest""" - env = { "__manifest" : self.manifest_dir } - self.run_manifest(self.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.get_type_from_object(cdist_object) - manifest = self.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.object_dir(cdist_object), - "__object_id": self.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) - - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.bin_dir + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - - # Legacy stuff to make cdist-type-emulator work - env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") - env['__cdist_local_base_dir'] = self.temp_dir - - # Submit information to new type emulator - env['__cdist_manifest'] = manifest - env['__cdist_type_base_dir'] = self.type_base_dir - - # Other environment stuff - if extra_env: - env.update(extra_env) - - self.shell_run_or_debug_fail(manifest, [manifest], env=env) - - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.object_dir(cdist_object), "require") - requirements = file_to_list(file) - type = self.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - env["__object"] = self.object_dir(cdist_object) - env["__object_id"] = self.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.type_dir(type) - - if mode == "gencode": - paths = [ - self.type_dir(type, "gencode-local"), - self.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.object_dir(cdist_object), - os.path.basename(bin)[3:]) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - # Mark object as changed - open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() - - - if mode == "code": - local_dir = self.object_dir(cdist_object) - remote_dir = self.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - self.run_or_fail([bin], remote=False) - - - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.transfer_file(local_remote_code, remote_remote_code) - self.run_or_fail([remote_remote_code], remote=True) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - old_objects = [] - objects = self.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - log.debug("Prepare stage") - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) - - objects = self.list_objects() - - def stage_run(self): - """The final (and real) step of deployment""" - log.debug("Actual run objects") - # Now do the final steps over the existing objects - for cdist_object in self.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() - - self.stage_prepare() - self.stage_run() - - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() - -def banner(args): - """Guess what :-)""" - print(BANNER) - sys.exit(0) - -def config(args): - """Configure remote system""" - process = {} - - time_start = datetime.datetime.now() - - for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) - if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) - process[host].start() - else: - c.deploy_and_cleanup() - - if args.parallel: - for p in process.keys(): - log.debug("Joining %s", p) - process[p].join() - - time_end = datetime.datetime.now() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start).total_seconds()) - -def install(args): - """Install remote system""" - process = {} - -def emulator(): - """Emulate type commands (i.e. __file and co)""" - type = os.path.basename(sys.argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) - param_dir = os.path.join(type_dir, "parameter") - global_dir = os.environ['__global'] - object_source = os.environ['__cdist_manifest'] - - parser = argparse.ArgumentParser(add_help=False) - - # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=False) - - # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=True) - - # Setup positional parameter, if not singleton - - if not os.path.isfile(os.path.join(type_dir, "singleton")): - parser.add_argument("object_id", nargs=1) - - # And finally verify parameter - args = parser.parse_args(sys.argv[1:]) - - # Setup object_id - if os.path.isfile(os.path.join(type_dir, "singleton")): - object_id = "singleton" - else: - object_id = args.object_id[0] - del args.object_id - - # FIXME: / hardcoded - better portable solution available? - if object_id[0] == '/': - object_id = object_id[1:] - - # FIXME: verify object id - log.debug(args) - - object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) - param_out_dir = os.path.join(object_dir, "parameter") - - object_source_file = os.path.join(object_dir, "source") - - if os.path.exists(param_out_dir): - object_exists = True - old_object_source_fd = open(object_source_file, "r") - old_object_source = old_object_source_fd.readlines() - old_object_source_fd.close() - - else: - object_exists = False - try: - os.makedirs(param_out_dir, exist_ok=True) - except OSError as error: - exit_error(param_out_dir + ": " + error.args[1]) - - # Record parameter - params = vars(args) - for param in params: - value = getattr(args, param) - if value: - file = os.path.join(param_out_dir, param) - log.debug(file + "<-" + param + " = " + value) - - # Already exists, verify all parameter are the same - if object_exists: - if not os.path.isfile(file): - print("New parameter + " + param + "specified, aborting") - print("Source = " + old_object_source + "new =" + object_source) - sys.exit(1) - else: - param_fd = open(file, "r") - param_old = param_fd.readlines() - param_fd.close() - - if(param_old != param): - print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) - print("Sources: " + " ".join(old_object_source) + " and " + object_source) - sys.exit(1) - else: - param_fd = open(file, "w") - param_fd.writelines(value) - param_fd.close() - - # Record requirements - if "__require" in os.environ: - requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) - require_fd = open(os.path.join(object_dir, "require"), "a") - require_fd.writelines(requirements.split(" ")) - require_fd.close() - - # Record / Append source - source_fd = open(os.path.join(object_dir, "source"), "a") - source_fd.writelines(object_source) - source_fd.close() - - # sys.exit(1) - print("Finished " + type + "/" + object_id + repr(params)) - - -def commandline(): - """Parse command line""" - # Construct parser others can reuse - parser = {} - # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') - - # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) - parser['main'].add_argument('-V', '--version', - help='Show version', action='version', - version='%(prog)s ' + VERSION) - parser['sub'] = parser['main'].add_subparsers(title="Commands") - - # Banner - parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) - parser['banner'].set_defaults(func=banner) - - # Config and install (common stuff) - parser['configinstall'] = argparse.ArgumentParser(add_help=False) - parser['configinstall'].add_argument('host', nargs='+', - help='one or more hosts to operate on') - parser['configinstall'].add_argument('-c', '--cdist-home', - help='Change cdist home (default: .. from bin directory)', - action='store') - parser['configinstall'].add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest', - dest='manifest', required=False) - parser['configinstall'].add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['configinstall'].add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') - - # Config - parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) - parser['config'].set_defaults(func=config) - - # Install - parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) - parser['install'].set_defaults(func=install) - - for p in parser: - parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" - - args = parser['main'].parse_args(sys.argv[1:]) - - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) - - args.func(args) - -if __name__ == "__main__": - try: - if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - emulator() - else: - commandline() - except KeyboardInterrupt: - sys.exit(0) From ceaf82f6df5494f96d31aab271a690e8bb5e13dc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:19:16 +0200 Subject: [PATCH 28/55] better comment Signed-off-by: Nico Schottelius --- module/cdist/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/cdist/path.py b/module/cdist/path.py index c72d29a8..e18f6582 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -1,4 +1,4 @@ -# Given paths from installation +# Hardcoded paths usually not changable REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") From 91022c3f7e7102833a4416cb45ae171f6e8d4b1e Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 23 Sep 2011 17:19:45 +0200 Subject: [PATCH 29/55] handle ioerror if script does not exist Signed-off-by: Steven Armstrong --- bin/cdist | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index fde7a251..d9f88b48 100755 --- a/bin/cdist +++ b/bin/cdist @@ -181,9 +181,12 @@ class Cdist: if remote: remote_cat(script) else: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() + try: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + except IOError as error: + raise CdistError(str(error)) raise CdistError("Command failed (shell): " + " ".join(*args)) except OSError as error: From 8e59f97800a0c9fecccbdb9517b1a8a581e5c933 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 18:55:23 +0200 Subject: [PATCH 30/55] begin to used cdist.path Signed-off-by: Nico Schottelius --- bin/cdist | 13 ++++++++----- {module => lib}/cdist/__init__.py | 0 {module => lib}/cdist/path.py | 0 3 files changed, 8 insertions(+), 5 deletions(-) rename {module => lib}/cdist/__init__.py (100%) rename {module => lib}/cdist/path.py (100%) diff --git a/bin/cdist b/bin/cdist index d9f88b48..69e0623f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -20,6 +20,10 @@ # # +# Add our own library path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib/python'))) + import argparse import datetime import logging @@ -32,6 +36,8 @@ import stat import sys import tempfile +import cdist.path + BANNER = """ .. . .x+=:. s dF @88> z` ^% :8 @@ -62,11 +68,6 @@ VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() -# Begin to split into parts -sys.path.insert(0, - os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib/python'))) - def file_to_list(filename): """Return list from \n seperated file""" if os.path.isfile(filename): @@ -94,6 +95,8 @@ class Cdist: self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] + self.path_info = cdist.path.Path(target_host, home) + # Setup directory paths self.temp_dir = tempfile.mkdtemp() diff --git a/module/cdist/__init__.py b/lib/cdist/__init__.py similarity index 100% rename from module/cdist/__init__.py rename to lib/cdist/__init__.py diff --git a/module/cdist/path.py b/lib/cdist/path.py similarity index 100% rename from module/cdist/path.py rename to lib/cdist/path.py From 2aa9f2ab8c8cb4d1349ea6ce98a9e3df8e5b0851 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 18:56:29 +0200 Subject: [PATCH 31/55] test: things to do Signed-off-by: Nico Schottelius --- doc/dev/todo/TAKEME | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index 5439a1b9..8be1da54 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -9,6 +9,12 @@ CORE - allow cdist to run without $PATH setup: ./bin/cdist-deploy-to - support non-ssh access? +TESTS +----- +- multiple defines of object: + - fail if different parameters + - succeed if same parameters + USER INTERFACE -------------- - add support $__tmp? From 6139fab354b52ca56e5d4ffaea281d83b020a21e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 19:10:07 +0200 Subject: [PATCH 32/55] import cdist.path works Signed-off-by: Nico Schottelius --- bin/cdist | 10 +++++----- lib/cdist/path.py | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 69e0623f..18e7845b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -20,10 +20,6 @@ # # -# Add our own library path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib/python'))) - import argparse import datetime import logging @@ -36,6 +32,10 @@ import stat import sys import tempfile +# Add our own library path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib'))) + import cdist.path BANNER = """ @@ -95,7 +95,7 @@ class Cdist: self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] - self.path_info = cdist.path.Path(target_host, home) + self.path_info = cdist.path.Path(target_host, base_dir=home) # Setup directory paths self.temp_dir = tempfile.mkdtemp() diff --git a/lib/cdist/path.py b/lib/cdist/path.py index e18f6582..5d72a23a 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -1,3 +1,7 @@ +import os +import sys +import tempfile + # Hardcoded paths usually not changable REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") @@ -31,7 +35,7 @@ class Path: def __init__(self, target_host, base_dir=None): # Base and Temp Base - if home: + if base_dir: self.base_dir = base_dir else: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) From 26fad6f23ce1b656618eb64e1d367cf5757cf79f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 19:13:36 +0200 Subject: [PATCH 33/55] correct base dir, now relative to lib Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 5d72a23a..69bc5496 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -38,7 +38,7 @@ class Path: if base_dir: self.base_dir = base_dir else: - self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) self.temp_dir = tempfile.mkdtemp() From 79173488ffd697ef89d8a387034eb3447cf3da61 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:08:55 +0200 Subject: [PATCH 34/55] cdist runs with path module loaded (unusued though) Signed-off-by: Nico Schottelius --- bin/cdist | 13 ++++++++----- lib/cdist/path.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 18e7845b..f6156825 100755 --- a/bin/cdist +++ b/bin/cdist @@ -32,11 +32,6 @@ import stat import sys import tempfile -# Add our own library path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib'))) - -import cdist.path BANNER = """ .. . .x+=:. s @@ -788,11 +783,19 @@ def commandline(): args.func(args) + if __name__ == "__main__": try: if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): emulator() else: + # Add our own library path + sys.path.insert(0, + os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib'))) + + import cdist.path + commandline() except KeyboardInterrupt: sys.exit(0) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 69bc5496..2050a1a3 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -1,3 +1,25 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging import os import sys import tempfile @@ -13,6 +35,10 @@ CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" TYPE_PREFIX = "__" +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +log = logging.getLogger() + + def file_to_list(filename): """Return list from \n seperated file""" if os.path.isfile(filename): @@ -33,7 +59,10 @@ def file_to_list(filename): class Path: """Class that handles path related configurations""" - def __init__(self, target_host, base_dir=None): + def __init__(self, target_host, + initial_manifest=False, remote_user="root", + base_dir=None, debug=False): + # Base and Temp Base if base_dir: self.base_dir = base_dir From d4406cece3dc07e8a1f7a248b32c98c92f2895cb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:17:17 +0200 Subject: [PATCH 35/55] begin to replace first function with path module Signed-off-by: Nico Schottelius --- bin/cdist | 67 +++++++------------------------------------------------ 1 file changed, 8 insertions(+), 59 deletions(-) diff --git a/bin/cdist b/bin/cdist index f6156825..137f701f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -90,72 +90,21 @@ class Cdist: self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] - self.path_info = cdist.path.Path(target_host, base_dir=home) - - # Setup directory paths - self.temp_dir = tempfile.mkdtemp() + self.path = cdist.path.Path(target_host, + initial_manifest=initial_manifest, + remote_user=remote_user, + base_dir=home, + debug=debug) self.debug = debug - if home: - self.base_dir = home - else: - self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - - self.conf_dir = os.path.join(self.base_dir, "conf") - self.cache_base_dir = os.path.join(self.base_dir, "cache") - self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) - self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") - self.lib_dir = os.path.join(self.base_dir, "lib") - self.manifest_dir = os.path.join(self.conf_dir, "manifest") - self.type_base_dir = os.path.join(self.conf_dir, "type") - - self.out_dir = os.path.join(self.temp_dir, "out") - os.mkdir(self.out_dir) - - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") - os.mkdir(self.global_explorer_out_dir) - - self.object_base_dir = os.path.join(self.out_dir, "object") - - # Setup binary directory + contents - self.bin_dir = os.path.join(self.out_dir, "bin") - os.mkdir(self.bin_dir) - self.link_type_to_emulator() - - # List of type explorers transferred - self.type_explorers_transferred = {} - # objects self.objects_prepared = [] self.remote_user = remote_user - # Mostly static, but can be overwritten on user demand - if initial_manifest: - self.initial_manifest = initial_manifest - else: - self.initial_manifest = os.path.join(self.manifest_dir, "init") - def cleanup(self): - # Do not use in __del__: - # http://docs.python.org/reference/datamodel.html#customization - # "other globals referenced by the __del__() method may already have been deleted - # or in the process of being torn down (e.g. the import machinery shutting down)" - # - log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) - # Remove previous cache - if os.path.exists(self.cache_dir): - shutil.rmtree(self.cache_dir) - shutil.move(self.temp_dir, self.cache_dir) - - def remote_mkdir(self, directory): - """Create directory on remote side""" - self.run_or_fail(["mkdir", "-p", directory], remote=True) - - def remote_cat(filename): - """Use cat on the remote side for output""" - self.run_or_fail(["cat", filename], remote=True) + self.path.cleanup() def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want @@ -415,8 +364,8 @@ class Cdist: """Ensure the base directories are cleaned up""" log.debug("Creating clean directory structure") - self.remove_remote_dir(REMOTE_BASE_DIR) - self.remote_mkdir(REMOTE_BASE_DIR) + self.path.remove_remote_dir(REMOTE_BASE_DIR) + self.path.remote_mkdir(REMOTE_BASE_DIR) def run_initial_manifest(self): """Run the initial manifest""" From e14848ec1f88e1e79aade52fa4180d4a8d33db9e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:21:10 +0200 Subject: [PATCH 36/55] remove run* from cdist and put it into lib/exec Signed-off-by: Nico Schottelius --- bin/cdist | 49 --------------------------------- lib/cdist/exec.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 49 deletions(-) create mode 100644 lib/cdist/exec.py diff --git a/bin/cdist b/bin/cdist index 137f701f..30fec278 100755 --- a/bin/cdist +++ b/bin/cdist @@ -106,55 +106,6 @@ class Cdist: def cleanup(self): self.path.cleanup() - def shell_run_or_debug_fail(self, script, *args, **kargs): - # Manually execute /bin/sh, because sh -e does what we want - # and sh -c -e does not exit if /bin/false called - args[0][:0] = [ "/bin/sh", "-e" ] - - remote = False - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - remote = true - - del kargs["remote"] - - log.debug("Shell exec cmd: %s", args) - log.debug("Shell exec env: %s", kargs['env']) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - log.error("Code that raised the error:\n") - if remote: - remote_cat(script) - else: - try: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() - except IOError as error: - raise CdistError(str(error)) - - raise CdistError("Command failed (shell): " + " ".join(*args)) - except OSError as error: - raise CdistError(" ".join(*args) + ": " + error.args[1]) - - def run_or_fail(self, *args, **kargs): - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - - del kargs["remote"] - - log.debug("Exec: " + " ".join(*args)) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - raise CdistError("Command failed: " + " ".join(*args)) - except OSError as error: - raise CdistError(" ".join(*args) + ": " + error.args[1]) - - def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py new file mode 100644 index 00000000..7a1421e0 --- /dev/null +++ b/lib/cdist/exec.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +def shell_run_or_debug_fail(self, script, *args, **kargs): + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0][:0] = [ "/bin/sh", "-e" ] + + remote = False + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + remote = true + + del kargs["remote"] + + log.debug("Shell exec cmd: %s", args) + log.debug("Shell exec env: %s", kargs['env']) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + log.error("Code that raised the error:\n") + if remote: + remote_cat(script) + else: + try: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + except IOError as error: + raise CdistError(str(error)) + + raise CdistError("Command failed (shell): " + " ".join(*args)) + except OSError as error: + raise CdistError(" ".join(*args) + ": " + error.args[1]) + + +def run_or_fail(self, *args, **kargs): + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + + del kargs["remote"] + + log.debug("Exec: " + " ".join(*args)) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + raise CdistError("Command failed: " + " ".join(*args)) + except OSError as error: + raise CdistError(" ".join(*args) + ": " + error.args[1]) + From 3925ba1c6e0bb7276a374836ec98f4f9e2c81552 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:35:13 +0200 Subject: [PATCH 37/55] pass remote_prefix to run_or_fail/shell Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 7a1421e0..a4cf4592 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -19,7 +19,8 @@ # # -def shell_run_or_debug_fail(self, script, *args, **kargs): + +def shell_run_or_debug_fail(script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] @@ -27,10 +28,11 @@ def shell_run_or_debug_fail(self, script, *args, **kargs): remote = False if "remote" in kargs: if kargs["remote"]: - args[0][:0] = self.remote_prefix + args[0][:0] = kargs["remote_prefix"] remote = true del kargs["remote"] + del kargs["remote_prefix"] log.debug("Shell exec cmd: %s", args) log.debug("Shell exec env: %s", kargs['env']) @@ -56,9 +58,10 @@ def shell_run_or_debug_fail(self, script, *args, **kargs): def run_or_fail(self, *args, **kargs): if "remote" in kargs: if kargs["remote"]: - args[0][:0] = self.remote_prefix + args[0][:0] = kargs["remote_prefix"] del kargs["remote"] + del kargs["remote_prefix"] log.debug("Exec: " + " ".join(*args)) try: @@ -67,4 +70,3 @@ def run_or_fail(self, *args, **kargs): raise CdistError("Command failed: " + " ".join(*args)) except OSError as error: raise CdistError(" ".join(*args) + ": " + error.args[1]) - From 8af45f83b20bd8a0e48f58eb5e7e9105bc76edda Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:53:09 +0200 Subject: [PATCH 38/55] rearange remote_user Signed-off-by: Nico Schottelius --- bin/cdist | 17 ----------------- lib/cdist/path.py | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/bin/cdist b/bin/cdist index 30fec278..67b96a09 100755 --- a/bin/cdist +++ b/bin/cdist @@ -88,7 +88,6 @@ class Cdist: initial_manifest=False, remote_user="root", home=None, debug=False): self.target_host = target_host - self.remote_prefix = ["ssh", "root@" + self.target_host] self.path = cdist.path.Path(target_host, initial_manifest=initial_manifest, @@ -98,7 +97,6 @@ class Cdist: self.debug = debug - # objects self.objects_prepared = [] self.remote_user = remote_user @@ -109,21 +107,6 @@ class Cdist: def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) - def transfer_dir(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) - - def transfer_file(self, source, destination): - """Transfer file""" - self.run_or_fail(["scp", "-q", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) - def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 2050a1a3..1e9d7195 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -38,6 +38,7 @@ TYPE_PREFIX = "__" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() +import cdist.exec def file_to_list(filename): """Return list from \n seperated file""" @@ -53,9 +54,6 @@ def file_to_list(filename): return lines -# FIXME: self.run_or_fail needs to be elsewhere! -# Exec? - class Path: """Class that handles path related configurations""" @@ -70,6 +68,10 @@ class Path: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) self.temp_dir = tempfile.mkdtemp() + self.target_host = target_host + + self.remote_user = remote_user + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] self.conf_dir = os.path.join(self.base_dir, "conf") self.cache_base_dir = os.path.join(self.base_dir, "cache") @@ -98,8 +100,6 @@ class Path: # objects self.objects_prepared = [] - self.remote_user = remote_user - # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest @@ -121,26 +121,26 @@ class Path: def remote_mkdir(self, directory): """Create directory on remote side""" - self.run_or_fail(["mkdir", "-p", directory], remote=True) + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote=True) def remote_cat(filename): """Use cat on the remote side for output""" - self.run_or_fail(["cat", filename], remote=True) + cdist.exec.run_or_fail(["cat", filename], remote=True) def remove_remote_dir(self, destination): - self.run_or_fail(["rm", "-rf", destination], remote=True) + cdist.exec.run_or_fail(["rm", "-rf", destination], remote=True) def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, + cdist.exec.run_or_fail(["scp", "-qr", source, self.remote_user + "@" + self.target_host + ":" + destination]) def transfer_file(self, source, destination): """Transfer file""" - self.run_or_fail(["scp", "-q", source, + cdist.exec.run_or_fail(["scp", "-q", source, self.remote_user + "@" + self.target_host + ":" + destination]) From 7b39169e3e55231eef729d83eea32be97ff49841 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Sep 2011 15:25:38 +0200 Subject: [PATCH 39/55] ignore all python cache dirs Signed-off-by: Nico Schottelius --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d606aec7..fb9c495c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,5 @@ doc/man/man*/docbook-xsl.css # Ignore cache for version control cache/ -# Python -bin/__pycache__/ +# Python / cache +__pycache__/ From a5bfd4119521b6758b7893945812ae2be2ad70dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Sep 2011 15:26:54 +0200 Subject: [PATCH 40/55] Detect owl Signed-off-by: Nico Schottelius --- conf/explorer/os | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/explorer/os b/conf/explorer/os index e922c067..268dae24 100755 --- a/conf/explorer/os +++ b/conf/explorer/os @@ -65,6 +65,11 @@ if [ -f /etc/SuSE-release ]; then exit 0 fi +if uname -r | grep -s '.owl' >/dev/null 2>&1; then + echo owl + exit 0 +fi + if [ -f /etc/cdist-preos ]; then echo preos exit 0 From c10fedaf8c9f87c36e0a0bb2602de8a6ff46edec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Sep 2011 21:10:18 +0200 Subject: [PATCH 41/55] it's not preos, but cdist-preos Signed-off-by: Nico Schottelius --- conf/explorer/os | 2 +- doc/changelog | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/explorer/os b/conf/explorer/os index 268dae24..3e1582ec 100755 --- a/conf/explorer/os +++ b/conf/explorer/os @@ -71,7 +71,7 @@ if uname -r | grep -s '.owl' >/dev/null 2>&1; then fi if [ -f /etc/cdist-preos ]; then - echo preos + echo cdist-preos exit 0 fi diff --git a/doc/changelog b/doc/changelog index b1149eb1..a08efb34 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,3 +1,6 @@ +2.0.2: + * Add support for detection of OpenWall Linux (Matthias Teege) + 2.0.1: 2011-09-23 * Bugfix core: Always print source of error in case of exec errors * Bugfix core: Various smaller bugs in string concatenation From 98f4ec9f3e6334768e3f7065dfd66ef9e13c772c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 09:24:49 +0200 Subject: [PATCH 42/55] document asciidoc requirement Signed-off-by: Nico Schottelius --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index 5e60a093..b8181757 100644 --- a/README +++ b/README @@ -78,6 +78,7 @@ cdist was tested or is know to run on at least * A posix like shell * Python (>= 3.2 required) * SSH-Client + * Asciidoc (for building the manpages) ### Client ("target host") From fd9fb13606aa54687d36f9b70965cb6a9e086829 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 09:26:58 +0200 Subject: [PATCH 43/55] and remove asciidoc/gmake dep from the comment Signed-off-by: Nico Schottelius --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index b8181757..daa12f70 100644 --- a/README +++ b/README @@ -99,7 +99,7 @@ To install cdist, execute the following commands: cd cdist export PATH=$PATH:$(pwd -P)/bin - # If you want the manpages (requires gmake and asciidoc to be installed) + # If you want the manpages ./build.sh man export MANPATH=$MANPATH:$(pwd -P)/doc/man From bc9bc37aab80bc85d991621da9addd016ac9d0d4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 10:17:02 +0200 Subject: [PATCH 44/55] use remote_prefix internally Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 26 +++++++++++++------------- lib/cdist/path.py | 6 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index a4cf4592..aca1d689 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -19,19 +19,21 @@ # # +import logging +import subprocess + +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +log = logging.getLogger() + def shell_run_or_debug_fail(script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] - remote = False - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = kargs["remote_prefix"] - remote = true - - del kargs["remote"] + if "remote_prefix" in kargs: + remote = True + args[0][:0] = kargs["remote_prefix"] del kargs["remote_prefix"] log.debug("Shell exec cmd: %s", args) @@ -41,6 +43,7 @@ def shell_run_or_debug_fail(script, *args, **kargs): except subprocess.CalledProcessError: log.error("Code that raised the error:\n") if remote: + # FIXME: included in Path! remote_cat(script) else: try: @@ -55,12 +58,9 @@ def shell_run_or_debug_fail(script, *args, **kargs): raise CdistError(" ".join(*args) + ": " + error.args[1]) -def run_or_fail(self, *args, **kargs): - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = kargs["remote_prefix"] - - del kargs["remote"] +def run_or_fail(*args, **kargs): + if "remote_prefix" in kargs: + args[0][:0] = kargs["remote_prefix"] del kargs["remote_prefix"] log.debug("Exec: " + " ".join(*args)) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 1e9d7195..c2cef1a6 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -121,14 +121,14 @@ class Path: def remote_mkdir(self, directory): """Create directory on remote side""" - cdist.exec.run_or_fail(["mkdir", "-p", directory], remote=True) + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix) def remote_cat(filename): """Use cat on the remote side for output""" - cdist.exec.run_or_fail(["cat", filename], remote=True) + cdist.exec.run_or_fail(["cat", filename], remote_prefix=self.remote_prefix) def remove_remote_dir(self, destination): - cdist.exec.run_or_fail(["rm", "-rf", destination], remote=True) + cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix) def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" From 8f2e5bb8c869bd0cce84ae13ccf84756c70298cb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 10:25:51 +0200 Subject: [PATCH 45/55] cleanup and move error class to init Signed-off-by: Nico Schottelius --- bin/cdist | 150 +----------------------------------------- lib/cdist/__init__.py | 24 +++++++ 2 files changed, 25 insertions(+), 149 deletions(-) diff --git a/bin/cdist b/bin/cdist index 67b96a09..369fbd29 100755 --- a/bin/cdist +++ b/bin/cdist @@ -77,10 +77,6 @@ def file_to_list(filename): return lines -class CdistError(Exception): - """Base exception class for this project""" - pass - class Cdist: """Cdist main class to hold arbitrary data""" @@ -104,153 +100,9 @@ class Cdist: def cleanup(self): self.path.cleanup() - def remove_remote_dir(self, destination): - self.run_or_fail(["rm", "-rf", destination], remote=True) - - def global_explorer_output_path(self, explorer): - """Returns path of the output for a global explorer""" - return os.path.join(self.global_explorer_out_dir, explorer) - - def type_explorer_output_dir(self, cdist_object): - """Returns and creates dir of the output for a type explorer""" - dir = os.path.join(self.object_dir(cdist_object), "explorer") - if not os.path.isdir(dir): - os.mkdir(dir) - - return dir - - def remote_global_explorer_path(self, explorer): - """Returns path to the remote explorer""" - return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) - - def list_global_explorers(self): - """Return list of available explorers""" - return os.listdir(self.global_explorer_dir) - - def list_type_explorers(self, type): - """Return list of available explorers for a specific type""" - dir = self.type_dir(type, "explorer") - if os.path.isdir(dir): - list = os.listdir(dir) - else: - list = [] - - log.debug("Explorers for %s in %s: %s", type, dir, list) - - return list - - def list_types(self): - return os.listdir(self.type_base_dir) - - def list_object_paths(self, starting_point): - """Return list of paths of existing objects""" - object_paths = [] - - for content in os.listdir(starting_point): - full_path = os.path.join(starting_point, content) - if os.path.isdir(full_path): - object_paths.extend(self.list_object_paths(starting_point = full_path)) - - # Directory contains .cdist -> is an object - if content == DOT_CDIST: - object_paths.append(starting_point) - - return object_paths - - def get_type_from_object(self, cdist_object): - """Returns the first part (i.e. type) of an object""" - return cdist_object.split(os.sep)[0] - - def get_object_id_from_object(self, cdist_object): - """Returns everything but the first part (i.e. object_id) of an object""" - return os.sep.join(cdist_object.split(os.sep)[1:]) - - def object_dir(self, cdist_object): - """Returns the full path to the object (including .cdist)""" - return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) - - def remote_object_dir(self, cdist_object): - """Returns the remote full path to the object (including .cdist)""" - return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) - - def object_parameter_dir(self, cdist_object): - """Returns the dir to the object parameter""" - return os.path.join(self.object_dir(cdist_object), "parameter") - - def remote_object_parameter_dir(self, cdist_object): - """Returns the remote dir to the object parameter""" - return os.path.join(self.remote_object_dir(cdist_object), "parameter") - - def object_code_paths(self, cdist_object): - """Return paths to code scripts of object""" - return [os.path.join(self.object_dir(cdist_object), "code-local"), - os.path.join(self.object_dir(cdist_object), "code-remote")] - - def list_objects(self): - """Return list of existing objects""" - - objects = [] - if os.path.isdir(self.object_base_dir): - object_paths = self.list_object_paths(self.object_base_dir) - - for path in object_paths: - objects.append(os.path.relpath(path, self.object_base_dir)) - - return objects - - def type_dir(self, type, *args): - """Return directory the type""" - return os.path.join(self.type_base_dir, type, *args) - - def remote_type_explorer_dir(self, type): - """Return remote directory that holds the explorers of a type""" - return os.path.join(REMOTE_TYPE_DIR, type, "explorer") - - def transfer_object_parameter(self, cdist_object): - """Transfer the object parameter to the remote destination""" - # Create base path before using mkdir -p - self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) - - # Synchronise parameter dir afterwards - self.transfer_dir(self.object_parameter_dir(cdist_object), - self.remote_object_parameter_dir(cdist_object)) - - def transfer_global_explorers(self): - """Transfer the global explorers""" - self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) - self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) - - def transfer_type_explorers(self, type): - """Transfer explorers of a type, but only once""" - if type in self.type_explorers_transferred: - log.debug("Skipping retransfer for explorers of %s", type) - return - else: - # Do not retransfer - self.type_explorers_transferred[type] = 1 - - src = self.type_dir(type, "explorer") - remote_base = os.path.join(REMOTE_TYPE_DIR, type) - dst = self.remote_type_explorer_dir(type) - - # Only continue, if there is at least the directory - if os.path.isdir(src): - # Ensure that the path exists - self.remote_mkdir(remote_base) - self.transfer_dir(src, dst) - - - def link_type_to_emulator(self): - """Link type names to cdist-type-emulator""" - source = os.path.abspath(sys.argv[0]) - for type in self.list_types(): - destination = os.path.join(self.bin_dir, type) - log.debug("Linking %s to %s", source, destination) - os.symlink(source, destination) - def run_global_explores(self): """Run global explorers""" - explorers = self.list_global_explorers() + explorers = self.path.list_global_explorers() if(len(explorers) == 0): raise CdistError("No explorers found in", self.global_explorer_dir) diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index e69de29b..a1afebbd 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +class Error(Exception): + """Base exception class for this project""" + pass From e6a903fd967be6cde4e7b6585aceaa42954f4049 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 10:28:57 +0200 Subject: [PATCH 46/55] cleanup logger stuff, remove more path stuff from cdist bin Signed-off-by: Nico Schottelius --- bin/cdist | 10 ---------- lib/cdist/exec.py | 4 +--- lib/cdist/path.py | 3 +-- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/bin/cdist b/bin/cdist index 369fbd29..ed71249c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -48,16 +48,6 @@ BANNER = """ "P' "" "" """ -# Given paths from installation -REMOTE_BASE_DIR = "/var/lib/cdist" -REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") -REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") -REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") -REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") - -CODE_HEADER = "#!/bin/sh -e\n" -DOT_CDIST = ".cdist" -TYPE_PREFIX = "__" VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index aca1d689..4b61a097 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -22,9 +22,7 @@ import logging import subprocess -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -log = logging.getLogger() - +log = logging.getLogger(__name__) def shell_run_or_debug_fail(script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want diff --git a/lib/cdist/path.py b/lib/cdist/path.py index c2cef1a6..401e3c46 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -35,8 +35,7 @@ CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" TYPE_PREFIX = "__" -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -log = logging.getLogger() +log = logging.getLogger(__name__) import cdist.exec From cf920ca3e96d302e00b6827968ff1a387c574bed Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 10:53:34 +0200 Subject: [PATCH 47/55] prefix issues Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 91dbc72f..4e07dd96 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,16 +1,3 @@ -2.0.1: - -- Rewrite cdist-type-emulator - - Remove legacy code in cdist - - Remove cdist-config - - Remove man1/cdist-type-emulator.text - - Remove the PATH=... part from the README - - - how to access output dir? - - Test: - __cdist_type_base_dir=$(pwd -P)/conf/type __file - - Fix / rewrite cdist-quickstart - write tutorial!!!!!!!!! @@ -49,3 +36,7 @@ http://www.youtube.com/watch?v=PRMjzy48eTI - Setup __debug, if -d is given, so other tools can reuse it + (-> non core feature! + +- remote_prefix: + scp vs. ssh issue From 16d58dcac73f71352d5b77bbf5df2d78988bd68e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:05:18 +0200 Subject: [PATCH 48/55] move out banner, fail at emulator Signed-off-by: Nico Schottelius --- bin/cdist | 204 ++++++++++++++++++------------------------ bin/cdist.py | 1 - lib/cdist/__init__.py | 3 + lib/cdist/banner.py | 46 ++++++++++ lib/cdist/path.py | 8 +- 5 files changed, 140 insertions(+), 122 deletions(-) delete mode 120000 bin/cdist.py create mode 100644 lib/cdist/banner.py diff --git a/bin/cdist b/bin/cdist index ed71249c..15c3ff62 100755 --- a/bin/cdist +++ b/bin/cdist @@ -32,40 +32,7 @@ import stat import sys import tempfile - -BANNER = """ - .. . .x+=:. s - dF @88> z` ^% :8 - '88bu. %8P . . +# +# + +import logging +import sys + +log = logging.getLogger(__name__) + +BANNER = """ + .. . .x+=:. s + dF @88> z` ^% :8 + '88bu. %8P . Date: Mon, 26 Sep 2011 11:07:45 +0200 Subject: [PATCH 49/55] move TYPE_PREFIX back into main, as it's only needed there and should work without module Signed-off-by: Nico Schottelius --- bin/cdist | 4 +++- lib/cdist/__init__.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 15c3ff62..f4d67616 100755 --- a/bin/cdist +++ b/bin/cdist @@ -34,6 +34,8 @@ import tempfile log = logging.getLogger(__name__) +TYPE_PREFIX = "__" + class Cdist: """Cdist main class to hold arbitrary data""" @@ -482,7 +484,7 @@ if __name__ == "__main__": '../lib'))) import cdist - if re.match(cdist.TYPE_PREFIX, os.path.basename(sys.argv[0])): + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): emulator() else: import cdist.banner diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 6bcc93d2..192e5001 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -19,7 +19,6 @@ # # -TYPE_PREFIX = "__" VERSION = "2.0.2" class Error(Exception): From 4eec4d96270464d8b5275a71e9d40e6f44f54bb4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:09:58 +0200 Subject: [PATCH 50/55] only setup library path in main, not emulator Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index f4d67616..d6a6da3b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -478,15 +478,14 @@ if __name__ == "__main__": try: logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - # Import generic cdist options - sys.path.insert(0, - os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib'))) - import cdist - if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): emulator() else: + cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib')) + sys.path.insert(0, cdist_lib) + + import cdist import cdist.banner import cdist.exec import cdist.path From 6d75016139cec7d14a9d937eecabd3c32f4bd885 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:18:36 +0200 Subject: [PATCH 51/55] make type emulator load again Signed-off-by: Nico Schottelius --- bin/cdist | 120 ++++---------------------------------- lib/cdist/emulator.py | 132 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 108 deletions(-) create mode 100755 lib/cdist/emulator.py diff --git a/bin/cdist b/bin/cdist index d6a6da3b..3b19e388 100755 --- a/bin/cdist +++ b/bin/cdist @@ -146,9 +146,16 @@ class Cdist: env['__cdist_local_base_dir'] = self.path.temp_dir # Submit information to new type emulator + + # Required for recording source env['__cdist_manifest'] = manifest + + # Required to find types env['__cdist_type_base_dir'] = self.path.type_base_dir + # Required for loading emulator :-) + env['__cdist_python_lib'] = cdist_lib + # Other environment stuff if extra_env: env.update(extra_env) @@ -306,112 +313,6 @@ def install(args): """Install remote system""" process = {} -def emulator(): - """Emulate type commands (i.e. __file and co)""" - type = os.path.basename(sys.argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) - param_dir = os.path.join(type_dir, "parameter") - global_dir = os.environ['__global'] - object_source = os.environ['__cdist_manifest'] - - parser = argparse.ArgumentParser(add_help=False) - - # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=False) - - # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=True) - - # Setup positional parameter, if not singleton - - if not os.path.isfile(os.path.join(type_dir, "singleton")): - parser.add_argument("object_id", nargs=1) - - # And finally verify parameter - args = parser.parse_args(sys.argv[1:]) - - # Setup object_id - if os.path.isfile(os.path.join(type_dir, "singleton")): - object_id = "singleton" - else: - object_id = args.object_id[0] - del args.object_id - - # FIXME: / hardcoded - better portable solution available? - if object_id[0] == '/': - object_id = object_id[1:] - - # FIXME: verify object id - log.debug(args) - - object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) - param_out_dir = os.path.join(object_dir, "parameter") - - object_source_file = os.path.join(object_dir, "source") - - if os.path.exists(param_out_dir): - object_exists = True - old_object_source_fd = open(object_source_file, "r") - old_object_source = old_object_source_fd.readlines() - old_object_source_fd.close() - - else: - object_exists = False - try: - os.makedirs(param_out_dir, exist_ok=True) - except OSError as error: - raise CdistError(param_out_dir + ": " + error.args[1]) - - # Record parameter - params = vars(args) - for param in params: - value = getattr(args, param) - if value: - file = os.path.join(param_out_dir, param) - log.debug(file + "<-" + param + " = " + value) - - # Already exists, verify all parameter are the same - if object_exists: - if not os.path.isfile(file): - print("New parameter + " + param + "specified, aborting") - print("Source = " + old_object_source + "new =" + object_source) - sys.exit(1) - else: - param_fd = open(file, "r") - param_old = param_fd.readlines() - param_fd.close() - - if(param_old != param): - print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) - print("Sources: " + " ".join(old_object_source) + " and " + object_source) - sys.exit(1) - else: - param_fd = open(file, "w") - param_fd.writelines(value) - param_fd.close() - - # Record requirements - if "__require" in os.environ: - requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) - require_fd = open(os.path.join(object_dir, "require"), "a") - require_fd.writelines(requirements.split(" ")) - require_fd.close() - - # Record / Append source - source_fd = open(os.path.join(object_dir, "source"), "a") - source_fd.writelines(object_source) - source_fd.close() - - # sys.exit(1) - print("Finished " + type + "/" + object_id + repr(params)) - - def commandline(): """Parse command line""" # Construct parser others can reuse @@ -477,9 +378,12 @@ def commandline(): if __name__ == "__main__": try: logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - emulator() + cdist_lib = os.environ["__cdist_python_lib"] + sys.path.insert(0, cdist_lib) + import cdist.emulator + cdist.emulator.emulator(sys.argv) else: cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib')) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py new file mode 100755 index 00000000..0e693dbe --- /dev/null +++ b/lib/cdist/emulator.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import argparse +import logging +import os +import sys + +log = logging.getLogger(__name__) + +def emulator(argv): + """Emulate type commands (i.e. __file and co)""" + type = os.path.basename(argv[0]) + type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) + param_dir = os.path.join(type_dir, "parameter") + global_dir = os.environ['__global'] + object_source = os.environ['__cdist_manifest'] + + parser = argparse.ArgumentParser(add_help=False) + + # Setup optional parameters + for parameter in file_to_list(os.path.join(param_dir, "optional")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=False) + + # Setup required parameters + for parameter in file_to_list(os.path.join(param_dir, "required")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=True) + + # Setup positional parameter, if not singleton + + if not os.path.isfile(os.path.join(type_dir, "singleton")): + parser.add_argument("object_id", nargs=1) + + # And finally verify parameter + args = parser.parse_args(argv[1:]) + + # Setup object_id + if os.path.isfile(os.path.join(type_dir, "singleton")): + object_id = "singleton" + else: + object_id = args.object_id[0] + del args.object_id + + # FIXME: / hardcoded - better portable solution available? + if object_id[0] == '/': + object_id = object_id[1:] + + # FIXME: verify object id + log.debug(args) + + object_dir = os.path.join(global_dir, "object", type, + object_id, DOT_CDIST) + param_out_dir = os.path.join(object_dir, "parameter") + + object_source_file = os.path.join(object_dir, "source") + + if os.path.exists(param_out_dir): + object_exists = True + old_object_source_fd = open(object_source_file, "r") + old_object_source = old_object_source_fd.readlines() + old_object_source_fd.close() + + else: + object_exists = False + try: + os.makedirs(param_out_dir, exist_ok=True) + except OSError as error: + raise CdistError(param_out_dir + ": " + error.args[1]) + + # Record parameter + params = vars(args) + for param in params: + value = getattr(args, param) + if value: + file = os.path.join(param_out_dir, param) + log.debug(file + "<-" + param + " = " + value) + + # Already exists, verify all parameter are the same + if object_exists: + if not os.path.isfile(file): + print("New parameter + " + param + "specified, aborting") + print("Source = " + old_object_source + "new =" + object_source) + sys.exit(1) + else: + param_fd = open(file, "r") + param_old = param_fd.readlines() + param_fd.close() + + if(param_old != param): + print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + print("Sources: " + " ".join(old_object_source) + " and " + object_source) + sys.exit(1) + else: + param_fd = open(file, "w") + param_fd.writelines(value) + param_fd.close() + + # Record requirements + if "__require" in os.environ: + requirements = os.environ['__require'] + print(object_id + ":Writing requirements: " + requirements) + require_fd = open(os.path.join(object_dir, "require"), "a") + require_fd.writelines(requirements.split(" ")) + require_fd.close() + + # Record / Append source + source_fd = open(os.path.join(object_dir, "source"), "a") + source_fd.writelines(object_source) + source_fd.close() + + # sys.exit(1) + print("Finished " + type + "/" + object_id + repr(params)) From 74dc5b96cb3af00f8fb6ac3dd805f69de40d2737 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:21:04 +0200 Subject: [PATCH 52/55] type emulator begins to run Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 8 +++++--- lib/cdist/exec.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 0e693dbe..ecdfff25 100755 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -24,6 +24,8 @@ import logging import os import sys +import cdist.path + log = logging.getLogger(__name__) def emulator(argv): @@ -37,12 +39,12 @@ def emulator(argv): parser = argparse.ArgumentParser(add_help=False) # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): + for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")): argument = "--" + parameter parser.add_argument(argument, action='store', required=False) # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): + for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")): argument = "--" + parameter parser.add_argument(argument, action='store', required=True) @@ -69,7 +71,7 @@ def emulator(argv): log.debug(args) object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) + object_id, cdist.path.DOT_CDIST) param_out_dir = os.path.join(object_dir, "parameter") object_source_file = os.path.join(object_dir, "source") diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 4b61a097..09e4e8a4 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -29,6 +29,7 @@ def shell_run_or_debug_fail(script, *args, **kargs): # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] + remote = False if "remote_prefix" in kargs: remote = True args[0][:0] = kargs["remote_prefix"] From 0197f8da13ffdff8103a1a35c118f82ffca6968e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:25:22 +0200 Subject: [PATCH 53/55] more cleanups from refactoring Signed-off-by: Nico Schottelius --- bin/cdist | 5 +++-- lib/cdist/path.py | 17 +++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 3b19e388..f511d78e 100755 --- a/bin/cdist +++ b/bin/cdist @@ -35,6 +35,7 @@ import tempfile log = logging.getLogger(__name__) TYPE_PREFIX = "__" +CODE_HEADER = "#!/bin/sh -e\n" class Cdist: """Cdist main class to hold arbitrary data""" @@ -102,7 +103,7 @@ class Cdist: log.debug("%s exploring %s using %s storing to %s", cdist_object, explorer, remote_cmd, output) - self.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) output_fd.close() def init_deploy(self): @@ -240,7 +241,7 @@ class Cdist: self.run_initial_manifest() old_objects = [] - objects = self.list_objects() + objects = self.path.list_objects() # Continue process until no new objects are created anymore while old_objects != objects: diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 16aa51ea..0fa753a8 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -21,6 +21,7 @@ import logging import os +import shutil import sys import tempfile @@ -31,7 +32,6 @@ REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") -CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" log = logging.getLogger(__name__) @@ -55,9 +55,13 @@ def file_to_list(filename): class Path: """Class that handles path related configurations""" - def __init__(self, target_host, - initial_manifest=False, remote_user="root", - remote_prefix=False, base_dir=None, debug=False): + def __init__(self, + target_host, + remote_user, + remote_prefix, + initial_manifest=False, + base_dir=None, + debug=False): # Base and Temp Base if base_dir: @@ -69,10 +73,7 @@ class Path: self.target_host = target_host self.remote_user = remote_user - if remote_prefix: - self.remote_prefix = remote_prefix - else: - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + self.remote_prefix = remote_prefix self.conf_dir = os.path.join(self.base_dir, "conf") self.cache_base_dir = os.path.join(self.base_dir, "cache") From 35e33570d18eadd88d929a3b213a504218152a72 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:45:19 +0200 Subject: [PATCH 54/55] also move out config Signed-off-by: Nico Schottelius --- bin/cdist | 286 +------------------------------- lib/cdist/config.py | 393 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 399 insertions(+), 280 deletions(-) create mode 100755 lib/cdist/config.py diff --git a/bin/cdist b/bin/cdist index f511d78e..4dd4f971 100755 --- a/bin/cdist +++ b/bin/cdist @@ -33,283 +33,14 @@ import sys import tempfile log = logging.getLogger(__name__) +real_me = os.path.dirname(os.path.realpath(__file__)) +cdist_lib = os.path.abspath(os.path.join(real_me, '../lib')) + +sys.path.insert(0, cdist_lib) TYPE_PREFIX = "__" CODE_HEADER = "#!/bin/sh -e\n" -class Cdist: - """Cdist main class to hold arbitrary data""" - - def __init__(self, target_host, - initial_manifest=False, remote_user="root", - home=None, debug=False): - - self.target_host = target_host - self.debug = debug - self.remote_user = remote_user - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] - - self.path = cdist.path.Path(self.target_host, - initial_manifest=initial_manifest, - remote_user=self.remote_user, - remote_prefix=self.remote_prefix, - base_dir=home, - debug=debug) - - - self.objects_prepared = [] - - def cleanup(self): - self.path.cleanup() - - def run_global_explores(self): - """Run global explorers""" - explorers = self.path.list_global_explorers() - if(len(explorers) == 0): - raise CdistError("No explorers found in", self.path.global_explorer_dir) - - self.path.transfer_global_explorers() - for explorer in explorers: - output = self.path.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.path.remote_global_explorer_path(explorer)) - - cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() - - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" - - type = self.path.get_type_from_object(cdist_object) - self.path.transfer_type_explorers(type) - - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) - - # Need to transfer at least the parameters for objects to be useful - self.path.transfer_object_parameter(cdist_object) - - explorers = self.path.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() - - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) - self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - - def run_initial_manifest(self): - """Run the initial manifest""" - env = { "__manifest" : self.path.manifest_dir } - self.run_manifest(self.path.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.path.get_type_from_object(cdist_object) - manifest = self.path.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.path.object_dir(cdist_object), - "__object_id": self.path.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.path.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) - - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.path.bin_dir + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - - # Legacy stuff to make cdist-type-emulator work - env['__cdist_core_dir'] = os.path.join(self.path.base_dir, "core") - env['__cdist_local_base_dir'] = self.path.temp_dir - - # Submit information to new type emulator - - # Required for recording source - env['__cdist_manifest'] = manifest - - # Required to find types - env['__cdist_type_base_dir'] = self.path.type_base_dir - - # Required for loading emulator :-) - env['__cdist_python_lib'] = cdist_lib - - # Other environment stuff - if extra_env: - env.update(extra_env) - - cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) - - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.path.object_dir(cdist_object), "require") - requirements = cdist.path.file_to_list(file) - type = self.path.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - env["__object"] = self.path.object_dir(cdist_object) - env["__object_id"] = self.path.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.path.type_dir(type) - - if mode == "gencode": - paths = [ - self.path.type_dir(type, "gencode-local"), - self.path.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.path.object_dir(cdist_object), - os.path.basename(bin)[3:]) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - # Mark object as changed - open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() - - - if mode == "code": - local_dir = self.path.object_dir(cdist_object) - remote_dir = self.path.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - cdist.exec.run_or_fail([bin]) - - - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.path.transfer_file(local_remote_code, remote_remote_code) - # FIXME: remote_prefix - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - old_objects = [] - objects = self.path.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - log.debug("Prepare stage") - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) - - objects = self.path.list_objects() - - def stage_run(self): - """The final (and real) step of deployment""" - log.debug("Actual run objects") - # Now do the final steps over the existing objects - for cdist_object in self.path.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() - - self.stage_prepare() - self.stage_run() - - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() - -def config(args): - """Configure remote system""" - process = {} - - time_start = datetime.datetime.now() - - for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) - if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) - process[host].start() - else: - c.deploy_and_cleanup() - - if args.parallel: - for p in process.keys(): - log.debug("Joining %s", p) - process[p].join() - - time_end = datetime.datetime.now() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start).total_seconds()) - def install(args): """Install remote system""" process = {} @@ -355,7 +86,7 @@ def commandline(): # Config parser['config'] = parser['sub'].add_parser('config', parents=[parser['most'], parser['configinstall']]) - parser['config'].set_defaults(func=config) + parser['config'].set_defaults(func=cdist.config.config) # Install parser['install'] = parser['sub'].add_parser('install', @@ -381,17 +112,12 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - cdist_lib = os.environ["__cdist_python_lib"] - sys.path.insert(0, cdist_lib) import cdist.emulator cdist.emulator.emulator(sys.argv) else: - cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib')) - sys.path.insert(0, cdist_lib) - import cdist import cdist.banner + import cdist.config import cdist.exec import cdist.path diff --git a/lib/cdist/config.py b/lib/cdist/config.py new file mode 100755 index 00000000..a5a2252f --- /dev/null +++ b/lib/cdist/config.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import datetime +import logging +import os +import stat + +log = logging.getLogger(__name__) + +import cdist.path + +CODE_HEADER = "#!/bin/sh -e\n" + +class Config: + """Cdist main class to hold arbitrary data""" + + def __init__(self, target_host, + initial_manifest=False, remote_user="root", + home=None, debug=False): + + self.target_host = target_host + self.debug = debug + self.remote_user = remote_user + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + + self.path = cdist.path.Path(self.target_host, + initial_manifest=initial_manifest, + remote_user=self.remote_user, + remote_prefix=self.remote_prefix, + base_dir=home, + debug=debug) + + self.objects_prepared = [] + + def cleanup(self): + self.path.cleanup() + + def run_global_explores(self): + """Run global explorers""" + explorers = self.path.list_global_explorers() + if(len(explorers) == 0): + raise CdistError("No explorers found in", self.path.global_explorer_dir) + + self.path.transfer_global_explorers() + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.path.remote_global_explorer_path(explorer)) + + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + + type = self.path.get_type_from_object(cdist_object) + self.path.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.path.transfer_object_parameter(cdist_object) + + explorers = self.path.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + + def run_initial_manifest(self): + """Run the initial manifest""" + env = { "__manifest" : self.path.manifest_dir } + self.run_manifest(self.path.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.path.get_type_from_object(cdist_object) + manifest = self.path.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.path.object_dir(cdist_object), + "__object_id": self.path.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.path.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.path.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + + # Legacy stuff to make cdist-type-emulator work + env['__cdist_core_dir'] = os.path.join(self.path.base_dir, "core") + env['__cdist_local_base_dir'] = self.path.temp_dir + + # Submit information to new type emulator + + # Required for recording source + env['__cdist_manifest'] = manifest + + # Required to find types + env['__cdist_type_base_dir'] = self.path.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.path.object_dir(cdist_object), "require") + requirements = cdist.path.file_to_list(file) + type = self.path.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + env["__object"] = self.path.object_dir(cdist_object) + env["__object_id"] = self.path.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.path.type_dir(type) + + if mode == "gencode": + paths = [ + self.path.type_dir(type, "gencode-local"), + self.path.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.path.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.path.object_dir(cdist_object) + remote_dir = self.path.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + cdist.exec.run_or_fail([bin]) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.path.transfer_file(local_remote_code, remote_remote_code) + # FIXME: remote_prefix + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + old_objects = [] + objects = self.path.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + log.debug("Prepare stage") + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.path.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.debug("Actual run objects") + # Now do the final steps over the existing objects + for cdist_object in self.path.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() + +def config(args): + """Configure remote system""" + process = {} + + time_start = datetime.datetime.now() + + for host in args.host: + c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) + if args.parallel: + log.debug("Creating child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + else: + c.deploy_and_cleanup() + + if args.parallel: + for p in process.keys(): + log.debug("Joining %s", p) + process[p].join() + + time_end = datetime.datetime.now() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) + +def install(args): + """Install remote system""" + process = {} + +def commandline(): + """Parse command line""" + # Construct parser others can reuse + parser = {} + # Options _all_ parsers have in common + parser['most'] = argparse.ArgumentParser(add_help=False) + parser['most'].add_argument('-d', '--debug', + help='Set log level to debug', action='store_true') + + # Main subcommand parser + parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION) + parser['main'].add_argument('-V', '--version', + help='Show version', action='version', + version='%(prog)s ' + cdist.VERSION) + parser['sub'] = parser['main'].add_subparsers(title="Commands") + + # Banner + parser['banner'] = parser['sub'].add_parser('banner', + add_help=False) + parser['banner'].set_defaults(func=cdist.banner.banner) + + # Config and install (common stuff) + parser['configinstall'] = argparse.ArgumentParser(add_help=False) + parser['configinstall'].add_argument('host', nargs='+', + help='one or more hosts to operate on') + parser['configinstall'].add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store') + parser['configinstall'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest', + dest='manifest', required=False) + parser['configinstall'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['configinstall'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') + + # Config + parser['config'] = parser['sub'].add_parser('config', + parents=[parser['most'], parser['configinstall']]) + parser['config'].set_defaults(func=config) + + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['most'], parser['configinstall']]) + parser['install'].set_defaults(func=install) + + for p in parser: + parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" + + args = parser['main'].parse_args(sys.argv[1:]) + + # Most subcommands have --debug, so handle it here + if 'debug' in args: + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) + + args.func(args) + + +if __name__ == "__main__": + try: + logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): + cdist_lib = os.environ["__cdist_python_lib"] + sys.path.insert(0, cdist_lib) + import cdist.emulator + cdist.emulator.emulator(sys.argv) + else: + cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib')) + sys.path.insert(0, cdist_lib) + + import cdist + import cdist.banner + import cdist.exec + import cdist.path + + commandline() + except KeyboardInterrupt: + sys.exit(0) + except cdist.Error as e: + log.error(e) + sys.exit(1) From 27b4b9cd039f30ad71f7533da79f195c82c1fbdc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:53:09 +0200 Subject: [PATCH 55/55] also move out install and cleanup library path code Signed-off-by: Nico Schottelius --- bin/cdist | 16 ++++++---------- lib/cdist/install.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 10 deletions(-) create mode 100755 lib/cdist/install.py diff --git a/bin/cdist b/bin/cdist index 4dd4f971..5e1b96bf 100755 --- a/bin/cdist +++ b/bin/cdist @@ -33,17 +33,13 @@ import sys import tempfile log = logging.getLogger(__name__) -real_me = os.path.dirname(os.path.realpath(__file__)) -cdist_lib = os.path.abspath(os.path.join(real_me, '../lib')) -sys.path.insert(0, cdist_lib) +# Ensure our /lib/ is included into PYTHON_PATH +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')) +) TYPE_PREFIX = "__" -CODE_HEADER = "#!/bin/sh -e\n" - -def install(args): - """Install remote system""" - process = {} def commandline(): """Parse command line""" @@ -91,7 +87,7 @@ def commandline(): # Install parser['install'] = parser['sub'].add_parser('install', parents=[parser['most'], parser['configinstall']]) - parser['install'].set_defaults(func=install) + parser['install'].set_defaults(func=cdist.install.install) for p in parser: parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" @@ -106,7 +102,6 @@ def commandline(): args.func(args) - if __name__ == "__main__": try: logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') @@ -119,6 +114,7 @@ if __name__ == "__main__": import cdist.banner import cdist.config import cdist.exec + import cdist.install import cdist.path commandline() diff --git a/lib/cdist/install.py b/lib/cdist/install.py new file mode 100755 index 00000000..98b388ec --- /dev/null +++ b/lib/cdist/install.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging + +log = logging.getLogger(__name__) + +def install(args): + """Install remote system""" + process = {} +