From 1b84a2851146c78fe1c122afc216ec0f8d7b4a07 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 00:41:37 +0200 Subject: [PATCH 01/87] add cdist Signed-off-by: Nico Schottelius --- bin/cdist | 438 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 438 insertions(+) create mode 100644 bin/cdist diff --git a/bin/cdist b/bin/cdist new file mode 100644 index 00000000..a70a11f8 --- /dev/null +++ b/bin/cdist @@ -0,0 +1,438 @@ +#!python3 +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +__cdist_version="1.7.0" + +# Fail if something bogus is going on +set -u + +################################################################################ +# cconf standard vars prefixed with cdist + +__cdist_pwd="$(pwd -P)" +__cdist_mydir="${0%/*}"; +__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)" +__cdist_myname=${0##*/}; +__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" + +################################################################################ +# Names / Constants +# +# Most values can be overriden from outside, so you can +# customise paths as you like (for distributors, geeks and hackers) +# + +: ${__cdist_name_bin:=bin} +: ${__cdist_name_cache:=cache} +: ${__cdist_name_code:=code} +: ${__cdist_name_conf_dir:=conf} +: ${__cdist_name_dot_cdist:=.cdist} +: ${__cdist_name_explorer:=explorer} +: ${__cdist_name_gencode:=gencode} +: ${__cdist_name_gencode_local:=local} +: ${__cdist_name_gencode_remote:=remote} +: ${__cdist_name_global:=global} +: ${__cdist_name_host:=host} +: ${__cdist_name_init:=init} +: ${__cdist_name_manifest:=manifest} +: ${__cdist_name_object:=object} +: ${__cdist_name_object_finished:=done} +: ${__cdist_name_object_prepared:=prepared} +: ${__cdist_name_object_id:=object_id} +: ${__cdist_name_object_source:=source} +: ${__cdist_name_objects_created:=.objects_created} +: ${__cdist_name_out_dir:=out} +: ${__cdist_name_parameter:=parameter} +: ${__cdist_name_parameter_required:=required} +: ${__cdist_name_parameter_optional:=optional} +: ${__cdist_name_require:=require} +: ${__cdist_name_self:=self} +: ${__cdist_name_singleton:=singleton} +: ${__cdist_name_target_host:=target_host} +: ${__cdist_name_target_user:=target_user} +: ${__cdist_name_type:=type} +: ${__cdist_name_type_bin:=type_bin} +: ${__cdist_name_type_explorer:=type_explorer} +: ${__cdist_name_type_explorer_pushed:=.explorer_pushed} + +# Used for IDs: Allow everything not starting with - and . +: ${__cdist_sane_regexp:=[^-\.].*} + +# Default remote user +: ${__cdist_remote_user:=root} + + +################################################################################ +# Exported variable names (usable for non core +# +: ${__cdist_name_var_explorer:=__$__cdist_name_explorer} +: ${__cdist_name_var_type_explorer:=__$__cdist_name_type_explorer} +: ${__cdist_name_var_global:=__$__cdist_name_global} +: ${__cdist_name_var_manifest:=__$__cdist_name_manifest} +: ${__cdist_name_var_target_host:=__$__cdist_name_target_host} +: ${__cdist_name_var_target_user:=__$__cdist_name_target_user} +: ${__cdist_name_var_object:=__$__cdist_name_object} +: ${__cdist_name_var_object_id:=__$__cdist_name_object_id} +: ${__cdist_name_var_self:=__$__cdist_name_self} +: ${__cdist_name_var_type:=__$__cdist_name_type} + + +################################################################################ +# Tempfiles +# +: ${__cdist_tmp_base_dir=/tmp} +__cdist_tmp_dir=$(mktemp -d "$__cdist_tmp_base_dir/cdist.XXXXXXXXXXXX") +__cdist_tmp_file=$(mktemp "$__cdist_tmp_dir/cdist.XXXXXXXXXXXX") + +################################################################################ +# Local Base +# +: ${__cdist_local_base_dir:=$__cdist_tmp_dir} + +# Cache may *NOT* be below __cdist_local_base_dir! +: ${__cdist_local_base_cache_dir:=$__cdist_abs_mydir/../$__cdist_name_cache} + +: ${__cdist_conf_dir:="$(cd "$__cdist_abs_mydir/../conf" && pwd -P)"} + +: ${__cdist_explorer_dir:=$__cdist_conf_dir/$__cdist_name_explorer} +: ${__cdist_manifest_dir:=$__cdist_conf_dir/$__cdist_name_manifest} +: ${__cdist_manifest_init:=$__cdist_manifest_dir/$__cdist_name_init} +: ${__cdist_type_dir:=$__cdist_conf_dir/$__cdist_name_type} + +################################################################################ +# Local output +# +: ${__cdist_out_dir:=$__cdist_local_base_dir/$__cdist_name_out_dir} +: ${__cdist_out_explorer_dir:=$__cdist_out_dir/$__cdist_name_explorer} +: ${__cdist_out_object_dir:=$__cdist_out_dir/$__cdist_name_object} +: ${__cdist_out_type_dir:=$__cdist_out_dir/$__cdist_name_type} +: ${__cdist_out_type_bin_dir:=$__cdist_out_dir/$__cdist_name_type_bin} + +: ${__cdist_objects_created:=$__cdist_out_object_dir/$__cdist_name_objects_created} + +################################################################################ +# Remote base +# +: ${__cdist_remote_base_dir:=/var/lib/cdist} +: ${__cdist_remote_bin_dir:=$__cdist_remote_base_dir/$__cdist_name_bin} +: ${__cdist_remote_conf_dir:=$__cdist_remote_base_dir/$__cdist_name_conf_dir} + +: ${__cdist_remote_explorer_dir:=$__cdist_remote_conf_dir/$__cdist_name_explorer} +: ${__cdist_remote_type_dir:=$__cdist_remote_conf_dir/$__cdist_name_type} + +################################################################################ +# Remote output +# +: ${__cdist_remote_out_dir:=$__cdist_remote_base_dir/$__cdist_name_out_dir} +: ${__cdist_remote_out_explorer_dir:=$__cdist_remote_out_dir/$__cdist_name_explorer} +: ${__cdist_remote_out_object_dir:=$__cdist_remote_out_dir/$__cdist_name_object} + + +################################################################################ +# Internal functions +# +__cdist_echo() +{ + __cdist_echo_type="$1"; shift + + set +u + if [ "$__cdist_object_self" ]; then + __cdist_echo_prefix="${__cdist_object_self}:" + else + __cdist_echo_prefix="core: " + fi + set -u + + case "$__cdist_echo_type" in + debug) + set +u + if [ "$__cdist_debug" ]; then + echo $__cdist_echo_prefix "Debug: $@" + fi + set -u + ;; + info) + echo $__cdist_echo_prefix "$@" + ;; + warn) + echo $__cdist_echo_prefix "Warning: $@" + ;; + error) + echo $__cdist_echo_prefix "Error: $@" >&2 + ;; + *) + echo "CORE BUG, who created the broken commit in $0?" >&2 + exit 23 + ;; + esac +} + +__cdist_exec_fail_on_error() +{ + set +e + sh -e "$@" + if [ "$?" -ne 0 ]; then + __cdist_echo error "$1 exited non-zero" + __cdist_echo warn "Faulty code:" + cat "$1" + __cdist_exit_err "Aborting due to non-zero exit code." + fi +} + +__cdist_exit_err() +{ + __cdist_echo error "$@" + exit 1 +} + +__cdist_usage() +{ + __cdist_exit_err "$__cdist_myname: $@" +} + +__cdist_init_deploy() +{ + __cdist_echo info "Creating clean directory structure " + + # Ensure there is no old stuff, neither local nor remote + rm -rf "$__cdist_local_base_dir" + ssh "${__cdist_remote_user}@$1" "rm -rf ${__cdist_remote_base_dir}" + + # Init base + mkdir -p "$__cdist_local_base_dir" + ssh "${__cdist_remote_user}@$1" "mkdir -p ${__cdist_remote_base_dir}" + + # Link configuration source directory - consistent with remote + ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" +} + +################################################################################ +# Cache +# +__cdist_cache_dir() +{ + cd "${__cdist_local_base_cache_dir}" && pwd -P +} + +__cdist_host_cache_dir() +{ + echo "$(__cdist_cache_dir)/$1" +} + +################################################################################ +# Object +# + +__cdist_object_code() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_code}-$2" +} + +__cdist_object_prepared() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_object_prepared}" +} + +__cdist_object_finished() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_object_finished}" +} + +__cdist_object_dir() +{ + echo "$(__cdist_object_base_dir "$1")/${__cdist_name_dot_cdist}" +} + +__cdist_object_base_dir() +{ + echo "${__cdist_out_object_dir}/$1" +} + + +__cdist_object_id_from_object() +{ + echo "${1#*/}" +} + +# Find objects, remove ./ and /MARKER +__cdist_object_list() +{ + local basedir="$1"; shift + + # Use subshell to prevent changing cwd in program + ( + cd "${basedir}" + + find . -name "$__cdist_name_dot_cdist" | \ + sed -e 's;^./;;' -e "s;/${__cdist_name_dot_cdist}\$;;" + ) +} + +__cdist_object_parameter_dir() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_parameter}" +} + +__cdist_object_require() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_require}" +} + +__cdist_object_source_name() +{ + echo "$1/${__cdist_name_object_source}" +} + +__cdist_object_source() +{ + cat "$(__cdist_object_source_name "$1")" +} + +__cdist_object_source_add() +{ + echo "$__cdist_manifest" >> "$(__cdist_object_source_name "$1")" +} + +__cdist_object_type_explorer_dir() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_explorer}" +} + +################################################################################ +# Remote +# + +__cdist_remote_object_base_dir() +{ + echo "${__cdist_remote_out_object_dir}/$1" +} + +__cdist_remote_object_dir() +{ + echo "$(__cdist_remote_object_base_dir "$1")/${__cdist_name_dot_cdist}" +} + +__cdist_remote_object_parameter_dir() +{ + echo "$(__cdist_remote_object_dir "$1")/${__cdist_name_parameter}" +} + +__cdist_remote_object_type_explorer_dir() +{ + echo "$(__cdist_remote_object_dir "$1")/${__cdist_name_explorer}" +} + + +__cdist_remote_type_explorer_dir() +{ + echo "${__cdist_remote_type_dir}/$1/${__cdist_name_explorer}" +} + + +################################################################################ +# Traps +# +__cdist_tmp_removal() +{ + rm -rf "${__cdist_tmp_dir}" +} + +# Does not work in children, will be called again in every script! +# Use only in interactive "front end" scripts +__cdist_kill_on_interrupt() +{ + __cdist_tmp_removal + kill 0 + exit 1 +} + +# Remove tempfiles at normal exit +trap __cdist_tmp_removal EXIT + + +################################################################################ +# Type +# +__cdist_type_dir() +{ + echo "${__cdist_type_dir}/$1" +} + +__cdist_type_explorer_dir() +{ + echo "${__cdist_type_dir}/$1/${__cdist_name_explorer}" +} + +__cdist_type_from_object() +{ + echo "${1%%/*}" +} + +__cdist_type_has_explorer() +{ + # We only create output, if there's at least one explorer + # and can thus be used as a boolean ;-) + if [ -d "$(__cdist_type_explorer_dir "$1")" ]; then + ls -1 "$(__cdist_type_explorer_dir "$1")" + fi +} + +__cdist_type_explorer_pushed() +{ + [ -f "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" ] \ + && grep -q "$1" "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" +} + +__cdist_type_explorer_pushed_add() +{ + [ -d "$__cdist_out_type_dir" ] || mkdir "$__cdist_out_type_dir" + echo "$1" >> "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" +} + +__cdist_type_gencode() +{ + echo "${__cdist_type_dir}/$1/${__cdist_name_gencode}-$2" +} + +__cdist_type_manifest() +{ + echo "${__cdist_type_dir}/$1/${__cdist_name_manifest}" +} + +__cdist_type_parameter_dir() +{ + echo "$(__cdist_type_dir "$1")/${__cdist_name_parameter}" +} + +__cdist_type_parameter_optional() +{ + echo "$(__cdist_type_parameter_dir "$1")/$__cdist_name_parameter_optional" +} + +__cdist_type_parameter_required() +{ + echo "$(__cdist_type_parameter_dir "$1")/$__cdist_name_parameter_required" +} + +__cdist_type_singleton() +{ + echo "${__cdist_type_dir}/$1/${__cdist_name_singleton}" +} From d359f69f4bd907e88103aa41217724707a9c0325 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 00:49:29 +0200 Subject: [PATCH 02/87] +minimal __cdist_echo Signed-off-by: Nico Schottelius --- core/__cdist_cache | 34 ++++++++++++ core/__cdist_dir | 46 ++++++++++++++++ core/__cdist_dir_listing | 30 +++++++++++ core/__cdist_echo | 56 +++++++++++++++++++ core/__cdist_exit_err | 28 ++++++++++ core/__cdist_explorer_run | 79 +++++++++++++++++++++++++++ core/__cdist_explorer_run_global | 32 +++++++++++ core/__cdist_init_deploy | 40 ++++++++++++++ core/__cdist_is_executable | 44 +++++++++++++++ core/__cdist_kill_on_interrupt | 31 +++++++++++ core/__cdist_manifest_run | 53 ++++++++++++++++++ core/__cdist_manifest_run_init | 32 +++++++++++ core/__cdist_object_all | 60 +++++++++++++++++++++ core/__cdist_object_code_run | 54 +++++++++++++++++++ core/__cdist_object_explorer_run | 89 +++++++++++++++++++++++++++++++ core/__cdist_object_gencode | 66 +++++++++++++++++++++++ core/__cdist_object_gencode_run | 36 +++++++++++++ core/__cdist_object_list | 36 +++++++++++++ core/__cdist_object_manifest_run | 59 ++++++++++++++++++++ core/__cdist_object_prepare | 47 ++++++++++++++++ core/__cdist_object_run | 74 +++++++++++++++++++++++++ core/__cdist_run | 27 ++++++++++ core/__cdist_run_remote | 32 +++++++++++ core/__cdist_run_shell | 34 ++++++++++++ core/__cdist_tmp_removal | 27 ++++++++++ core/__cdist_type_build_emulation | 49 +++++++++++++++++ core/__cdist_usage | 27 ++++++++++ 27 files changed, 1222 insertions(+) create mode 100755 core/__cdist_cache create mode 100755 core/__cdist_dir create mode 100755 core/__cdist_dir_listing create mode 100755 core/__cdist_echo create mode 100755 core/__cdist_exit_err create mode 100755 core/__cdist_explorer_run create mode 100755 core/__cdist_explorer_run_global create mode 100755 core/__cdist_init_deploy create mode 100755 core/__cdist_is_executable create mode 100644 core/__cdist_kill_on_interrupt create mode 100755 core/__cdist_manifest_run create mode 100755 core/__cdist_manifest_run_init create mode 100755 core/__cdist_object_all create mode 100755 core/__cdist_object_code_run create mode 100755 core/__cdist_object_explorer_run create mode 100755 core/__cdist_object_gencode create mode 100755 core/__cdist_object_gencode_run create mode 100755 core/__cdist_object_list create mode 100755 core/__cdist_object_manifest_run create mode 100755 core/__cdist_object_prepare create mode 100755 core/__cdist_object_run create mode 100755 core/__cdist_run create mode 100755 core/__cdist_run_remote create mode 100755 core/__cdist_run_shell create mode 100755 core/__cdist_tmp_removal create mode 100755 core/__cdist_type_build_emulation create mode 100755 core/__cdist_usage diff --git a/core/__cdist_cache b/core/__cdist_cache new file mode 100755 index 00000000..95764d3d --- /dev/null +++ b/core/__cdist_cache @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2010 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Save the configuration tree into the local cache +# + +__cdist_cache() +{ + # Create base to move into + mkdir -p "${__cdist_local_base_cache_dir}" + + __cdist_echo info \ + "Caching to $(__cdist_host_cache_dir "$__cdist_target_host")" + rm -rf "$(__cdist_host_cache_dir "$__cdist_target_host")" + mv "$__cdist_local_base_dir" \ + "$(__cdist_host_cache_dir "$__cdist_target_host")" +} diff --git a/core/__cdist_dir b/core/__cdist_dir new file mode 100755 index 00000000..32ee0075 --- /dev/null +++ b/core/__cdist_dir @@ -0,0 +1,46 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Push a directory to a target, both sides have the same name (i.e. explorers) +# or +# Pull a directory from a target, both sides have the same name (i.e. explorers) +# + +__cdist_dir() +{ + [ $# -eq 3 ] || __cdist_usage " " + + # ${3%/*} will be the destination directory, so no subdirectories + # of the same name are created, if the directory is already existing + + if [ "$1" = "push" ]; then + # FIXME: add error handling with __cdist_run_remote_... or so + ssh "${__cdist_remote_user}@${__cdist_target_host}" \ + "mkdir -p \"$3\"" + scp -qr "$2" \ + "${__cdist_remote_user}@${__cdist_target_host}:${3%/*}" + elif [ "$1" = "pull" ]; then + mkdir -p "$3" + scp -qr "${__cdist_remote_user}@${__cdist_target_host}:$2" \ + "${3%/*}" + else + __cdist_exit_err "Unknown action $1" + fi +} diff --git a/core/__cdist_dir_listing b/core/__cdist_dir_listing new file mode 100755 index 00000000..f4aa2320 --- /dev/null +++ b/core/__cdist_dir_listing @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# List files in a directory, if it exists +# +# We only create output, if there's at least one entry +# and can thus be used as a boolean ;-) +# + +__cdist_dir_listing() +{ + [ -d "$1" ] && ls -1 "$1" +} diff --git a/core/__cdist_echo b/core/__cdist_echo new file mode 100755 index 00000000..a89d1821 --- /dev/null +++ b/core/__cdist_echo @@ -0,0 +1,56 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# echo / syslog alike function +# + +__cdist_echo() +{ + __cdist_echo_type="$1"; shift + + set +u + if [ "$__cdist_object_self" ]; then + __cdist_echo_prefix="${__cdist_object_self}:" + else + __cdist_echo_prefix="core: " + fi + set -u + + case "$__cdist_echo_type" in + debug) + if [ "$__cdist_debug" = 1 ]; then + echo $__cdist_echo_prefix "DEBUG: $@" + fi + ;; + info) + echo $__cdist_echo_prefix "$@" + ;; + warn) + echo $__cdist_echo_prefix "Warning: $@" + ;; + error) + echo $__cdist_echo_prefix "Error: $@" >&2 + ;; + *) + echo "CORE BUG, who created the broken commit in $0?" >&2 + exit 23 + ;; + esac +} diff --git a/core/__cdist_exit_err b/core/__cdist_exit_err new file mode 100755 index 00000000..303dbf20 --- /dev/null +++ b/core/__cdist_exit_err @@ -0,0 +1,28 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Print error and exit (perror() alike) +# + +__cdist_exit_err() +{ + __cdist_echo error "$@" + exit 1 +} diff --git a/core/__cdist_explorer_run b/core/__cdist_explorer_run new file mode 100755 index 00000000..9e58fa09 --- /dev/null +++ b/core/__cdist_explorer_run @@ -0,0 +1,79 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run explorers - FIXME: this function is ugly +# + +__cdist_explorer_run() +{ + [ $# -eq 5 ] || __cdist_usage " " + + # Ensure there is at least one explorer + num="$(ls -1 "$__cdist_explorer_dir" | wc -l)" + if [ "$num" -lt 1 ]; then + __cdist_exit_err "${__cdist_explorer_dir}: Contains no explorers" + fi + + # Check whether to setup variable for type explorer + case "$1" in + global) + ;; + type) + # FIXME: think about how and where this gets setup! + "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" + ;; + esac + + # Transfer explorers + __cdist_dir push "$2" "$3" + + # Create output directory + __cdist_run_remote mkdir -p "$4" + + # Execute all explorers - FIXME: isolate cd call? + cd "$2"; + # FIXME: cleanup double variable, no need when in directory + for __cdist_explorer_run_explorer in *; do + __cdist_explorer_explorer_name="${__cdist_explorer_run_explorer##*/}" + + if [ -f "$__cdist_explorer_run_explorer" ]; then + if [ ! -x "$__cdist_explorer_run_explorer" ]; then + __cdist_exit_err "Explorer \"$__cdist_explorer_run_explorer\" exists, but is not executable." + fi + + else + if [ -e "$__cdist_explorer_run_explorer" ]; then + __cdist_exit_err "Explorer \"$__cdist_explorer_run_explorer\" exists, but is not a file." + fi + fi + + # FIXME: no need for remote out dir probably? + # or should we leave it and continue using __cdist_dir pull? + __cdist_run_remote \ + "export $__cdist_name_var_explorer=\"$__cdist_remote_explorer_dir\";" \ + "export $__cdist_name_var_global=\"$__cdist_remote_out_dir\";" \ + "$3/$__cdist_explorer_run_explorer" ">" \ + "$4/$__cdist_explorer_run_explorer" || \ + __cdist_exit_err "Explorer $__cdist_explorer_run_explorer failed." + done + + # Transfer results back + __cdist_dir pull "$4" "$5" +} diff --git a/core/__cdist_explorer_run_global b/core/__cdist_explorer_run_global new file mode 100755 index 00000000..27359713 --- /dev/null +++ b/core/__cdist_explorer_run_global @@ -0,0 +1,32 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Copy & run the global explorers, i.e. not bound to types +# + +__cdist_explorer_run_global() +{ + __cdist_echo info "Running global explorers " + + # run the global explorers remotely + __cdist_explorer_run global \ + "$__cdist_explorer_dir" "$__cdist_remote_explorer_dir" \ + "$__cdist_remote_out_explorer_dir" "$__cdist_out_explorer_dir" +} diff --git a/core/__cdist_init_deploy b/core/__cdist_init_deploy new file mode 100755 index 00000000..4ac3168c --- /dev/null +++ b/core/__cdist_init_deploy @@ -0,0 +1,40 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Prepare deployment +# + +__cdist_init_deploy() +{ + __cdist_echo info "Creating clean directory structure " + + # Ensure there is no old stuff, neither local nor remote + rm -rf "$__cdist_local_base_dir" + ssh "${__cdist_remote_user}@${__cdist_target_host}" \ + "rm -rf ${__cdist_remote_base_dir}" + + # Init base + mkdir -p "$__cdist_local_base_dir" + ssh "${__cdist_remote_user}@${__cdist_target_host}" \ + "mkdir -p ${__cdist_remote_base_dir}" + + # Link configuration source directory - consistent with remote + ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" +} diff --git a/core/__cdist_is_executable b/core/__cdist_is_executable new file mode 100755 index 00000000..a7a6d174 --- /dev/null +++ b/core/__cdist_is_executable @@ -0,0 +1,44 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Test whether something is executable (that should be executable) or +# is missing +# + +__cdist_is_executable() +{ + [ $# -eq 1 ] || __cdist_exit_err "" + + if [ -e "$1" ]; then + if [ -f "$1" ]; then + if [ -x "$1" ]; then + # Exists and is a correct executable + true + else + __cdist_exit_err "$1 exists, but is not executable." + fi + else + __cdist_exit_err "$1 exists, but is not a file." + fi + else + # Does not exist + false + fi +} diff --git a/core/__cdist_kill_on_interrupt b/core/__cdist_kill_on_interrupt new file mode 100644 index 00000000..7cb711fa --- /dev/null +++ b/core/__cdist_kill_on_interrupt @@ -0,0 +1,31 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run the given command for each created object. +# + +# Does not work in children, will be called again in every script! +# Use only in interactive "front end" scripts +__cdist_kill_on_interrupt() +{ + __cdist_tmp_removal + kill 0 + exit 1 +} diff --git a/core/__cdist_manifest_run b/core/__cdist_manifest_run new file mode 100755 index 00000000..cf85d646 --- /dev/null +++ b/core/__cdist_manifest_run @@ -0,0 +1,53 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Let's build a cconfig tree from a configuration +# And save it into the cache tree +# + +__cdist_manifest_run() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_manifest="$1"; shift + + ################################################################################ + # Export information for cdist-type-emulator or manifest + # + + # Config dir should not get reset - FIXME: why did I do this? + export __cdist_conf_dir + + # Used to record the source in the object + export __cdist_manifest + + # Export information for manifests - __cdist_out_dir comes from cdist-config + export $__cdist_name_var_global="$__cdist_out_dir" + + ################################################################################ + # The actual run + # + + # Ensure binaries are existing - FIXME: move error handling into __cdist_type_build_emulation + __cdist_type_build_emulation \ + || __cdist_exit_err "Failed to build type emulation binaries" + + __cdist_run_shell "${__cdist_manifest}" +} diff --git a/core/__cdist_manifest_run_init b/core/__cdist_manifest_run_init new file mode 100755 index 00000000..e8fa63de --- /dev/null +++ b/core/__cdist_manifest_run_init @@ -0,0 +1,32 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Let's build a cconfig tree from a configuration +# And save it into the cache tree +# + +__cdist_manifest_run_init() +{ + # FIXME: probably do not export but always set explicitly? + export $__cdist_name_var_manifest="$__cdist_manifest_dir" + + __cdist_echo info "Running initial manifest for $__cdist_target_host " + __cdist_manifest_run "$__cdist_manifest_init" +} diff --git a/core/__cdist_object_all b/core/__cdist_object_all new file mode 100755 index 00000000..965d08f6 --- /dev/null +++ b/core/__cdist_object_all @@ -0,0 +1,60 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run the given command for each created object. +# + +__cdist_object_all() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_object_all_object_all_command="$1"; shift + + __cdist_object_all_object_all_objects="$__cdist_tmp_dir/objects" + + # Ensure object dir exists, so marker can be created + mkdir -p "${__cdist_out_object_dir}" + + # FIXME: : - why do we use a file? + # core/__cdist_object_manifest_run: touch "$__cdist_objects_created" + + # Loop until we do not create new objects anymore + # which is equal to all objects have been run + touch "$__cdist_objects_created" + while [ -f "$__cdist_objects_created" ]; do + # Assume we're done after this run + rm "$__cdist_objects_created" + + # Get listing of objects + __cdist_object_list "$__cdist_out_object_dir" > \ + "$__cdist_object_all_object_all_objects" + + # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP + while read __cdist_object_all_object; do + set -- "$@" "$__cdist_object_all_object" + done < "$__cdist_object_all_object_all_objects" + + while [ $# -gt 0 ]; do + __cdist_object_all_object="$1"; shift + $__cdist_object_all_object_all_command "$__cdist_object_all_object" + done + done +} diff --git a/core/__cdist_object_code_run b/core/__cdist_object_code_run new file mode 100755 index 00000000..8af67ab8 --- /dev/null +++ b/core/__cdist_object_code_run @@ -0,0 +1,54 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Exec the code for the given object locally and remote +# + +__cdist_object_code_run() +{ + [ $# -eq 1 ] || __cdist_exit_err "" + + + if [ ! -d "$(__cdist_object_dir "$1")" ]; then + __cdist_exit_err "Object undefined" + fi + + # Code local + export __cdist_out_object_dir="$__cdist_out_object_dir" + __cdist_echo debug "Trying to run local code" + if __cdist_is_executable \ + "$(__cdist_object_code "$1" "${__cdist_name_gencode_local}")"; then + __cdist_run_shell \ + "$(__cdist_object_code "$1" "${__cdist_name_gencode_local}")" + else + __cdist_echo debug "Local code: none" + fi + + # Code remote + __cdist_echo debug "Trying to run remote code" + if __cdist_is_executable \ + "$(__cdist_object_code "$1" "${__cdist_name_gencode_remote}")"; then + + __cdist_run_remote $(__cdist_remote_object_code "$1") + else + __cdist_echo debug "Remote code: none" + fi +} diff --git a/core/__cdist_object_explorer_run b/core/__cdist_object_explorer_run new file mode 100755 index 00000000..da59d6c3 --- /dev/null +++ b/core/__cdist_object_explorer_run @@ -0,0 +1,89 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run the explorers for the given object on the target host. +# + +# FIXME: many cleanups needed before going production! + +__cdist_object_explorer_run() +{ + __cdist_object_self="$1"; shift + + __cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" + __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" + + # Check if type of object has >= 1 explorer + __cdist_has_explorer="$(__cdist_dir_listing "$(__cdist_type_explorer_dir "$__cdist_type")" | wc -l)" + # Run the type explorers for the current object if any + if [ "$__cdist_has_explorer" -ge 1 ]; then + if ! __cdist_type_explorer_pushed "$__cdist_type"; then + # FIXME: variables! + src_dir="$(__cdist_type_explorer_dir "$__cdist_type")" + dst_dir="$(__cdist_remote_type_explorer_dir "$__cdist_type")" + __cdist_echo info "Transfering explorers for $__cdist_type " + __cdist_dir push "$src_dir" "$dst_dir" + __cdist_type_explorer_pushed_add "$__cdist_type" + fi + + __cdist_echo info "Running explorers" + # Copy object parameters + __cdist_dir push \ + "$(__cdist_object_parameter_dir "$__cdist_object_self")" \ + "$(__cdist_remote_object_parameter_dir "$__cdist_object_self")" + + # Execute explorers + # FIXME: STOPPED: + # - remove cdist-remote-explorer-run + # - problem: new variables / need to run explorer directly? + # -> or put logic into __cdist_explorer_run + # -> think about having _one_ wrapper script for remote to execute + # shell functions + + # Create remote output directory + __cdist_run_remote mkdir -p "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" + + cd "$(__cdist_type_explorer_dir "$__cdist_type")" + + + for __cdist_object_explorer_run_explorer in *; do + __cdist_run_remote \ + "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \ + "$__cdist_name_var_object_id=\"$__cdist_object_id\"" \ + "$__cdist_name_var_self=\"$__cdist_object_self\"" \ + "$(__cdist_remote_type_explorer_dir "$__cdist_type")/$__cdist_object_explorer_run_explorer" \ + ">" "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")/$__cdist_object_explorer_run_explorer" + done + +# __cdist_run_remote \ +# "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \ +# "$__cdist_name_var_object_id=\"$__cdist_object_id\"" \ +# "$__cdist_name_var_self=\"$__cdist_object_self\"" \ +# cdist-remote-explorer-run \ +# "$__cdist_name_var_type_explorer" \ +# "$(__cdist_remote_type_explorer_dir "$__cdist_type")" \ +# "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" + + # Copy back results + __cdist_dir pull "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" \ + "$(__cdist_object_type_explorer_dir "$__cdist_object_self")" + fi +} diff --git a/core/__cdist_object_gencode b/core/__cdist_object_gencode new file mode 100755 index 00000000..08ef8b7d --- /dev/null +++ b/core/__cdist_object_gencode @@ -0,0 +1,66 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Generate code from one object (object must be relative path!) +# WARNING: OUTPUT ON STDOUT, ERRORS NEED TO BE ON STDERR! +# + +# FIXME: check variable names: +# either prefix or use global or use functions directly +# functions looks good, they are cheap anyway! + +__cdist_object_gencode() +{ + [ $# -eq 2 ] || __cdist_usage "" "" + + __cdist_object_self="$1"; shift + __cdist_gencode_type="$1"; shift + + __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" + __cdist_type_gencode="$(__cdist_type_gencode "$__cdist_type" "$__cdist_gencode_type")" + __cdist_code_output="$(__cdist_object_code "$__cdist_object_self" "$__cdist_gencode_type")" + + # export variables for the gencode script + export __object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" + export __object="$(__cdist_object_dir "$__cdist_object_self")" + export __global="$__cdist_out_dir" + + if [ -x "$__cdist_type_gencode" ]; then + __cdist_run_shell "$__cdist_type_gencode" > "$__cdist_tmp_file" + else + if [ -e "$__cdist_type_gencode" ]; then + __cdist_exit_err "$__cdist_type_gencode exists, but is not executable" + fi + + # Ensure it's empty, if there is no gencode + : > "$__cdist_tmp_file" + fi + + # Only create code, if gencode created output + if [ "$(wc -l < "$__cdist_tmp_file")" -gt 0 ]; then + cat - "$__cdist_tmp_file" << eof > "$__cdist_code_output" +# +# The following code was generated by $__cdist_type_gencode +# + +eof + chmod u+x "${__cdist_code_output}" + fi +} diff --git a/core/__cdist_object_gencode_run b/core/__cdist_object_gencode_run new file mode 100755 index 00000000..308f5f33 --- /dev/null +++ b/core/__cdist_object_gencode_run @@ -0,0 +1,36 @@ +#!/bin/sh +# +# 2010 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# For the given object create the code to be executed on the target. +# + +__cdist_object_gencode_run() +{ + __cdist_object_gencode_run_object="$1"; shift + + __cdist_echo info "Generating local code " + __cdist_object_gencode "$__cdist_object_gencode_run_object" \ + "${__cdist_name_gencode_local}" + + __cdist_echo info "Generating remote code " + __cdist_object_gencode "$__cdist_object_gencode_run_object" \ + "${__cdist_name_gencode_remote}" +} diff --git a/core/__cdist_object_list b/core/__cdist_object_list new file mode 100755 index 00000000..f2785c30 --- /dev/null +++ b/core/__cdist_object_list @@ -0,0 +1,36 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Print error and exit (perror() alike) +# + +__cdist_object_list() +{ + # FIXME: no local in posix + local basedir="$1"; shift + + # Use subshell to prevent changing cwd in program + ( + cd "${basedir}" + + find . -name "$__cdist_name_dot_cdist" | \ + sed -e 's;^./;;' -e "s;/${__cdist_name_dot_cdist}\$;;" + ) +} diff --git a/core/__cdist_object_manifest_run b/core/__cdist_object_manifest_run new file mode 100755 index 00000000..efc85539 --- /dev/null +++ b/core/__cdist_object_manifest_run @@ -0,0 +1,59 @@ +#!/bin/sh +# +# 2010 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run the manifest for the given object. +# + + +__cdist_object_manifest_run() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_object_self="$1"; shift + + # FIXME: rename to __cdist_object_dir (everywhere!) + __cdist_cur_object_dir="$(__cdist_object_dir "$__cdist_object_self")" + __cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" + + __cdist_echo info "Checking manifest " + + __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" + __cdist_manifest="$(__cdist_type_manifest "$__cdist_type")" + + if [ -f "$__cdist_manifest" ]; then + if [ -x "$__cdist_manifest" ]; then + # Make __cdist_manifest available for cdist-type-emulator + export __cdist_manifest + + __cdist_echo info "Executing manifest " + export $__cdist_name_var_object="$__cdist_cur_object_dir" + export $__cdist_name_var_object_id="$__cdist_object_id" + export $__cdist_name_var_type="$(__cdist_type_dir "$__cdist_type")" + + __cdist_manifest_run "$__cdist_manifest" + + # Tell cdist-object-run-all that there may be new objects + touch "$__cdist_objects_created" + else + __cdist_exit_err "${__cdist_manifest} needs to be executable." + fi + fi +} diff --git a/core/__cdist_object_prepare b/core/__cdist_object_prepare new file mode 100755 index 00000000..24039be0 --- /dev/null +++ b/core/__cdist_object_prepare @@ -0,0 +1,47 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# For the given object: +# - run type explorers +# - run type manifest +# + +__cdist_object_prepare() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_object_self="$1"; shift + __cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")" + [ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined" + + # Export to non-core for use in manifest and gencode scripts + export $__cdist_name_var_self=$__cdist_object_self + + __cdist_object_prepared="$(__cdist_object_prepared "$__cdist_object_self")" + if [ ! -f "$__cdist_object_prepared" ]; then + __cdist_echo info "Preparing object" + __cdist_object_explorer_run "$__cdist_object_self" + __cdist_object_manifest_run "$__cdist_object_self" + + # Mark this object as prepared + touch "$__cdist_object_prepared" + fi +} diff --git a/core/__cdist_object_run b/core/__cdist_object_run new file mode 100755 index 00000000..d2c7df6e --- /dev/null +++ b/core/__cdist_object_run @@ -0,0 +1,74 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# For the given object: +# - run type explorers +# - run type manifest +# - generate code +# - copy object to target +# - execute code on target +# + +__cdist_object_run() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_object_self="$1"; shift + __cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")" + [ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined" + + # Export to non-core for use in manifest and gencode scripts + export $__cdist_name_var_self=$__cdist_object_self + + # FIXME: BUG: should be named differently! + # FIXME: BUG: I can be called recursively! -> variables are probably already set / overwritten! + __cdist_object_finished="$(__cdist_object_finished "$__cdist_object_self")" + if [ ! -f "$__cdist_object_finished" ]; then + # Resolve dependencies, if any + __cdist_object_require="$(__cdist_object_require "$__cdist_object_self")" + if [ -f "$__cdist_object_require" ]; then + # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP + while read __cdist_requirement; do + set -- "$@" "$__cdist_requirement" + done < "$__cdist_object_require" + + while [ $# -gt 0 ]; do + __cdist_requirement="$1"; shift + __cdist_echo info "Resolving requirement $__cdist_requirement" + # FIXME: BUG: at this point, the other __cdist_object_run may have + # overwritten all our variables! + __cdist_object_run "$__cdist_requirement" + done + fi + + __cdist_echo debug "Before gencode" + __cdist_object_gencode_run "$__cdist_object_self" + __cdist_echo debug "Before push" + __cdist_dir push "$(__cdist_object_dir "$__cdist_object_self")" \ + "$(__cdist_remote_object_dir "$__cdist_object_self")" + __cdist_echo debug "Before run" + __cdist_object_code_run "$__cdist_object_self" + __cdist_echo debug "Object run done" + + # Mark this object as done + touch "$__cdist_object_finished" + fi +} diff --git a/core/__cdist_run b/core/__cdist_run new file mode 100755 index 00000000..8febe550 --- /dev/null +++ b/core/__cdist_run @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Exit if an error occurs running something +# + +__cdist_run() +{ + "$@" || __cdist_echo error "$1 exited non-zero, aborting." +} diff --git a/core/__cdist_run_remote b/core/__cdist_run_remote new file mode 100755 index 00000000..17074049 --- /dev/null +++ b/core/__cdist_run_remote @@ -0,0 +1,32 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run a cdist binary on the remote side +# + +__cdist_run_remote() +{ + [ $# -ge 1 ] || __cdist_usage " [opts]" + + ssh "${__cdist_remote_user}@${__cdist_target_host}" \ + "export PATH=\"${__cdist_remote_bin_dir}:\$PATH\";" \ + "export __cdist_out_object_dir=\"$__cdist_remote_out_object_dir\";" \ + "$@" +} diff --git a/core/__cdist_run_shell b/core/__cdist_run_shell new file mode 100755 index 00000000..b6e0a57d --- /dev/null +++ b/core/__cdist_run_shell @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Exit if an error occurs when running a shell script +# + +__cdist_run_shell() +{ + # Prepend our path, so all cdist tools come before other tools + PATH="${__cdist_out_type_bin_dir}:$PATH" sh -e "$@" + if [ "$?" -ne 0 ]; then + __cdist_echo error "$1 exited non-zero" + __cdist_echo warn "Faulty code:" + cat "$1" + __cdist_exit_err "Aborting due to non-zero exit code." + fi +} diff --git a/core/__cdist_tmp_removal b/core/__cdist_tmp_removal new file mode 100755 index 00000000..74d74936 --- /dev/null +++ b/core/__cdist_tmp_removal @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Remove tmp dir +# + +__cdist_tmp_removal() +{ + rm -rf "${__cdist_tmp_dir}" +} diff --git a/core/__cdist_type_build_emulation b/core/__cdist_type_build_emulation new file mode 100755 index 00000000..3c7270ca --- /dev/null +++ b/core/__cdist_type_build_emulation @@ -0,0 +1,49 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Build pseudo binaries for type emulation +# + +__cdist_type_build_emulation() +{ + [ $# -eq 0 ] || __cdist_usage "No arguments" + + [ -f "${__cdist_out_type_bin_dir}/.marker" ] && return 0 + + __cdist_type_emulator="$__cdist_abs_mydir/cdist-type-emulator" + + if [ ! -d "${__cdist_type_dir}" ]; then + __cdist_exit_err "$__cdist_type_dir must exist and contain available types" + fi + + # Get Types + ( + cd "${__cdist_type_dir}" + ls -1 > "${__cdist_tmp_file}" + ) + + # Create binaries + mkdir -p "${__cdist_out_type_bin_dir}" + while read __cdist_type_build_emulation_type; do + ln -sf "${__cdist_type_emulator}" \ + "${__cdist_out_type_bin_dir}/${__cdist_type_build_emulation_type}" + done < "${__cdist_tmp_file}" + + touch "${__cdist_out_type_bin_dir}/.marker" +} diff --git a/core/__cdist_usage b/core/__cdist_usage new file mode 100755 index 00000000..9dfa30e4 --- /dev/null +++ b/core/__cdist_usage @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Print error and exit (perror() alike) +# + +__cdist_usage() +{ + __cdist_exit_err "$__cdist_myname: $@" +} From 34bb39193707b4f8a6e84db36bcc38801fa3f954 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:01:25 +0200 Subject: [PATCH 03/87] +header Signed-off-by: Nico Schottelius --- bin/cdist | 425 ++---------------------------------------------------- 1 file changed, 13 insertions(+), 412 deletions(-) mode change 100644 => 100755 bin/cdist diff --git a/bin/cdist b/bin/cdist old mode 100644 new mode 100755 index a70a11f8..31d7411b --- a/bin/cdist +++ b/bin/cdist @@ -1,4 +1,4 @@ -#!python3 +#!/usr/bin/env python3 # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) # @@ -19,420 +19,21 @@ # # -__cdist_version="1.7.0" +import sys # argv -# Fail if something bogus is going on -set -u +__cdist_version="2.0.0" -################################################################################ -# cconf standard vars prefixed with cdist +def cdist_echo(type, *args): + """Ignore type for now, support later""" + print(*args) -__cdist_pwd="$(pwd -P)" -__cdist_mydir="${0%/*}"; -__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)" -__cdist_myname=${0##*/}; -__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" +def cdist_deploy_to(hostname): + """Mimic the old deploy to: Deploy to one host""" + cdist_echo("info", "Deploying to host", hostname) -################################################################################ -# Names / Constants -# -# Most values can be overriden from outside, so you can -# customise paths as you like (for distributors, geeks and hackers) -# -: ${__cdist_name_bin:=bin} -: ${__cdist_name_cache:=cache} -: ${__cdist_name_code:=code} -: ${__cdist_name_conf_dir:=conf} -: ${__cdist_name_dot_cdist:=.cdist} -: ${__cdist_name_explorer:=explorer} -: ${__cdist_name_gencode:=gencode} -: ${__cdist_name_gencode_local:=local} -: ${__cdist_name_gencode_remote:=remote} -: ${__cdist_name_global:=global} -: ${__cdist_name_host:=host} -: ${__cdist_name_init:=init} -: ${__cdist_name_manifest:=manifest} -: ${__cdist_name_object:=object} -: ${__cdist_name_object_finished:=done} -: ${__cdist_name_object_prepared:=prepared} -: ${__cdist_name_object_id:=object_id} -: ${__cdist_name_object_source:=source} -: ${__cdist_name_objects_created:=.objects_created} -: ${__cdist_name_out_dir:=out} -: ${__cdist_name_parameter:=parameter} -: ${__cdist_name_parameter_required:=required} -: ${__cdist_name_parameter_optional:=optional} -: ${__cdist_name_require:=require} -: ${__cdist_name_self:=self} -: ${__cdist_name_singleton:=singleton} -: ${__cdist_name_target_host:=target_host} -: ${__cdist_name_target_user:=target_user} -: ${__cdist_name_type:=type} -: ${__cdist_name_type_bin:=type_bin} -: ${__cdist_name_type_explorer:=type_explorer} -: ${__cdist_name_type_explorer_pushed:=.explorer_pushed} +if __name__ == "__main__": + hostname=sys.argv[1] + cdist_echo("info", "cdist", __cdist_version, ": Configuring host", hostname) + cdist_deploy_to(hostname) -# Used for IDs: Allow everything not starting with - and . -: ${__cdist_sane_regexp:=[^-\.].*} - -# Default remote user -: ${__cdist_remote_user:=root} - - -################################################################################ -# Exported variable names (usable for non core -# -: ${__cdist_name_var_explorer:=__$__cdist_name_explorer} -: ${__cdist_name_var_type_explorer:=__$__cdist_name_type_explorer} -: ${__cdist_name_var_global:=__$__cdist_name_global} -: ${__cdist_name_var_manifest:=__$__cdist_name_manifest} -: ${__cdist_name_var_target_host:=__$__cdist_name_target_host} -: ${__cdist_name_var_target_user:=__$__cdist_name_target_user} -: ${__cdist_name_var_object:=__$__cdist_name_object} -: ${__cdist_name_var_object_id:=__$__cdist_name_object_id} -: ${__cdist_name_var_self:=__$__cdist_name_self} -: ${__cdist_name_var_type:=__$__cdist_name_type} - - -################################################################################ -# Tempfiles -# -: ${__cdist_tmp_base_dir=/tmp} -__cdist_tmp_dir=$(mktemp -d "$__cdist_tmp_base_dir/cdist.XXXXXXXXXXXX") -__cdist_tmp_file=$(mktemp "$__cdist_tmp_dir/cdist.XXXXXXXXXXXX") - -################################################################################ -# Local Base -# -: ${__cdist_local_base_dir:=$__cdist_tmp_dir} - -# Cache may *NOT* be below __cdist_local_base_dir! -: ${__cdist_local_base_cache_dir:=$__cdist_abs_mydir/../$__cdist_name_cache} - -: ${__cdist_conf_dir:="$(cd "$__cdist_abs_mydir/../conf" && pwd -P)"} - -: ${__cdist_explorer_dir:=$__cdist_conf_dir/$__cdist_name_explorer} -: ${__cdist_manifest_dir:=$__cdist_conf_dir/$__cdist_name_manifest} -: ${__cdist_manifest_init:=$__cdist_manifest_dir/$__cdist_name_init} -: ${__cdist_type_dir:=$__cdist_conf_dir/$__cdist_name_type} - -################################################################################ -# Local output -# -: ${__cdist_out_dir:=$__cdist_local_base_dir/$__cdist_name_out_dir} -: ${__cdist_out_explorer_dir:=$__cdist_out_dir/$__cdist_name_explorer} -: ${__cdist_out_object_dir:=$__cdist_out_dir/$__cdist_name_object} -: ${__cdist_out_type_dir:=$__cdist_out_dir/$__cdist_name_type} -: ${__cdist_out_type_bin_dir:=$__cdist_out_dir/$__cdist_name_type_bin} - -: ${__cdist_objects_created:=$__cdist_out_object_dir/$__cdist_name_objects_created} - -################################################################################ -# Remote base -# -: ${__cdist_remote_base_dir:=/var/lib/cdist} -: ${__cdist_remote_bin_dir:=$__cdist_remote_base_dir/$__cdist_name_bin} -: ${__cdist_remote_conf_dir:=$__cdist_remote_base_dir/$__cdist_name_conf_dir} - -: ${__cdist_remote_explorer_dir:=$__cdist_remote_conf_dir/$__cdist_name_explorer} -: ${__cdist_remote_type_dir:=$__cdist_remote_conf_dir/$__cdist_name_type} - -################################################################################ -# Remote output -# -: ${__cdist_remote_out_dir:=$__cdist_remote_base_dir/$__cdist_name_out_dir} -: ${__cdist_remote_out_explorer_dir:=$__cdist_remote_out_dir/$__cdist_name_explorer} -: ${__cdist_remote_out_object_dir:=$__cdist_remote_out_dir/$__cdist_name_object} - - -################################################################################ -# Internal functions -# -__cdist_echo() -{ - __cdist_echo_type="$1"; shift - - set +u - if [ "$__cdist_object_self" ]; then - __cdist_echo_prefix="${__cdist_object_self}:" - else - __cdist_echo_prefix="core: " - fi - set -u - - case "$__cdist_echo_type" in - debug) - set +u - if [ "$__cdist_debug" ]; then - echo $__cdist_echo_prefix "Debug: $@" - fi - set -u - ;; - info) - echo $__cdist_echo_prefix "$@" - ;; - warn) - echo $__cdist_echo_prefix "Warning: $@" - ;; - error) - echo $__cdist_echo_prefix "Error: $@" >&2 - ;; - *) - echo "CORE BUG, who created the broken commit in $0?" >&2 - exit 23 - ;; - esac -} - -__cdist_exec_fail_on_error() -{ - set +e - sh -e "$@" - if [ "$?" -ne 0 ]; then - __cdist_echo error "$1 exited non-zero" - __cdist_echo warn "Faulty code:" - cat "$1" - __cdist_exit_err "Aborting due to non-zero exit code." - fi -} - -__cdist_exit_err() -{ - __cdist_echo error "$@" - exit 1 -} - -__cdist_usage() -{ - __cdist_exit_err "$__cdist_myname: $@" -} - -__cdist_init_deploy() -{ - __cdist_echo info "Creating clean directory structure " - - # Ensure there is no old stuff, neither local nor remote - rm -rf "$__cdist_local_base_dir" - ssh "${__cdist_remote_user}@$1" "rm -rf ${__cdist_remote_base_dir}" - - # Init base - mkdir -p "$__cdist_local_base_dir" - ssh "${__cdist_remote_user}@$1" "mkdir -p ${__cdist_remote_base_dir}" - - # Link configuration source directory - consistent with remote - ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" -} - -################################################################################ -# Cache -# -__cdist_cache_dir() -{ - cd "${__cdist_local_base_cache_dir}" && pwd -P -} - -__cdist_host_cache_dir() -{ - echo "$(__cdist_cache_dir)/$1" -} - -################################################################################ -# Object -# - -__cdist_object_code() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_code}-$2" -} - -__cdist_object_prepared() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_object_prepared}" -} - -__cdist_object_finished() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_object_finished}" -} - -__cdist_object_dir() -{ - echo "$(__cdist_object_base_dir "$1")/${__cdist_name_dot_cdist}" -} - -__cdist_object_base_dir() -{ - echo "${__cdist_out_object_dir}/$1" -} - - -__cdist_object_id_from_object() -{ - echo "${1#*/}" -} - -# Find objects, remove ./ and /MARKER -__cdist_object_list() -{ - local basedir="$1"; shift - - # Use subshell to prevent changing cwd in program - ( - cd "${basedir}" - - find . -name "$__cdist_name_dot_cdist" | \ - sed -e 's;^./;;' -e "s;/${__cdist_name_dot_cdist}\$;;" - ) -} - -__cdist_object_parameter_dir() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_parameter}" -} - -__cdist_object_require() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_require}" -} - -__cdist_object_source_name() -{ - echo "$1/${__cdist_name_object_source}" -} - -__cdist_object_source() -{ - cat "$(__cdist_object_source_name "$1")" -} - -__cdist_object_source_add() -{ - echo "$__cdist_manifest" >> "$(__cdist_object_source_name "$1")" -} - -__cdist_object_type_explorer_dir() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_explorer}" -} - -################################################################################ -# Remote -# - -__cdist_remote_object_base_dir() -{ - echo "${__cdist_remote_out_object_dir}/$1" -} - -__cdist_remote_object_dir() -{ - echo "$(__cdist_remote_object_base_dir "$1")/${__cdist_name_dot_cdist}" -} - -__cdist_remote_object_parameter_dir() -{ - echo "$(__cdist_remote_object_dir "$1")/${__cdist_name_parameter}" -} - -__cdist_remote_object_type_explorer_dir() -{ - echo "$(__cdist_remote_object_dir "$1")/${__cdist_name_explorer}" -} - - -__cdist_remote_type_explorer_dir() -{ - echo "${__cdist_remote_type_dir}/$1/${__cdist_name_explorer}" -} - - -################################################################################ -# Traps -# -__cdist_tmp_removal() -{ - rm -rf "${__cdist_tmp_dir}" -} - -# Does not work in children, will be called again in every script! -# Use only in interactive "front end" scripts -__cdist_kill_on_interrupt() -{ - __cdist_tmp_removal - kill 0 - exit 1 -} - -# Remove tempfiles at normal exit -trap __cdist_tmp_removal EXIT - - -################################################################################ -# Type -# -__cdist_type_dir() -{ - echo "${__cdist_type_dir}/$1" -} - -__cdist_type_explorer_dir() -{ - echo "${__cdist_type_dir}/$1/${__cdist_name_explorer}" -} - -__cdist_type_from_object() -{ - echo "${1%%/*}" -} - -__cdist_type_has_explorer() -{ - # We only create output, if there's at least one explorer - # and can thus be used as a boolean ;-) - if [ -d "$(__cdist_type_explorer_dir "$1")" ]; then - ls -1 "$(__cdist_type_explorer_dir "$1")" - fi -} - -__cdist_type_explorer_pushed() -{ - [ -f "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" ] \ - && grep -q "$1" "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" -} - -__cdist_type_explorer_pushed_add() -{ - [ -d "$__cdist_out_type_dir" ] || mkdir "$__cdist_out_type_dir" - echo "$1" >> "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" -} - -__cdist_type_gencode() -{ - echo "${__cdist_type_dir}/$1/${__cdist_name_gencode}-$2" -} - -__cdist_type_manifest() -{ - echo "${__cdist_type_dir}/$1/${__cdist_name_manifest}" -} - -__cdist_type_parameter_dir() -{ - echo "$(__cdist_type_dir "$1")/${__cdist_name_parameter}" -} - -__cdist_type_parameter_optional() -{ - echo "$(__cdist_type_parameter_dir "$1")/$__cdist_name_parameter_optional" -} - -__cdist_type_parameter_required() -{ - echo "$(__cdist_type_parameter_dir "$1")/$__cdist_name_parameter_required" -} - -__cdist_type_singleton() -{ - echo "${__cdist_type_dir}/$1/${__cdist_name_singleton}" -} From 4f7fa6b8924ec1dc8651382a8d35fe532075f39e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:10:37 +0200 Subject: [PATCH 04/87] +/- todo Signed-off-by: Nico Schottelius --- core/__cdist_init_deploy | 40 ---------------------------------------- doc/dev/todo/niconext | 28 ++-------------------------- 2 files changed, 2 insertions(+), 66 deletions(-) delete mode 100755 core/__cdist_init_deploy diff --git a/core/__cdist_init_deploy b/core/__cdist_init_deploy deleted file mode 100755 index 4ac3168c..00000000 --- a/core/__cdist_init_deploy +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Prepare deployment -# - -__cdist_init_deploy() -{ - __cdist_echo info "Creating clean directory structure " - - # Ensure there is no old stuff, neither local nor remote - rm -rf "$__cdist_local_base_dir" - ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "rm -rf ${__cdist_remote_base_dir}" - - # Init base - mkdir -p "$__cdist_local_base_dir" - ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "mkdir -p ${__cdist_remote_base_dir}" - - # Link configuration source directory - consistent with remote - ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" -} diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 28762ef4..a59f909f 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,26 +1,2 @@ -Catch broken instances in cdist-mass-deploy -p and report broken deployements at the end! - - --------------------------------------------------------------------------------- -Bug with requirements when indirect requires is scheduled late: - -__package/collectd: Generating local code -__package/collectd: Generating remote code -__package/collectd: Transferring object -__package/collectd: Checking code-local -__package/collectd: Checking code-remote -__file/etc/collectd/collectd.conf: Generating local code -__file/etc/collectd/collectd.conf: Generating remote code -__file/etc/collectd/collectd.conf: Transferring object -__file/etc/collectd/collectd.conf: Checking code-local -__file/etc/collectd/collectd.conf: Executing code-local -scp: /etc/collectd/collectd.conf: No such file or directory -__file/etc/collectd/collectd.conf: Error: /tmp/cdist.PYKFWmj9QknE/out/object/__file/etc/collectd/collectd.conf/.cdist/code-local exited non-zero -__file/etc/collectd/collectd.conf: Warning: Faulty code: -# -# The following code was generated by /home/users/nico/oeffentlich/rechner/projekte/cdist-nutzung/conf/type/__file/gencode-local -# - -scp /home/users/nico/oeffentlich/rechner/projekte/cdist-nutzung/conf/type/__ethz_collectd/files/collectd.conf.client root@shrek08:/etc/collectd/collectd.conf -__file/etc/collectd/collectd.conf: Error: Aborting due to non-zero exit code. - +- rewrite in python? +- support non-ssh access? From 9711a5612d359919ff0a254042c4fec4ba17f1ca Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:21:15 +0200 Subject: [PATCH 05/87] partly implement run_or_fail, init_deploy Signed-off-by: Nico Schottelius --- bin/cdist | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 31d7411b..233e6fee 100755 --- a/bin/cdist +++ b/bin/cdist @@ -20,20 +20,51 @@ # import sys # argv +import subprocess # execute stuff __cdist_version="2.0.0" -def cdist_echo(type, *args): +def logger(type, *args): """Ignore type for now, support later""" print(*args) +def exit_error(*args): + logger(*args) + sys.exit(1) + + +def run_or_fail(*args): + try: + subprocess.check_call(*args) + except CalledProcessError: + exit_error("Command failed:", *args) + + def cdist_deploy_to(hostname): """Mimic the old deploy to: Deploy to one host""" - cdist_echo("info", "Deploying to host", hostname) + logger("info", "Deploying to host", hostname) + + +def init_deploy(): + logger("info", "Creating clean directory structure") + + # Ensure there is no old stuff, neither local nor remote + run_or_fail(["echo rm -rf", "$__cdist_local_base_dir"]) + +# ssh "${__cdist_remote_user}@${__cdist_target_host}" \ +# "rm -rf ${__cdist_remote_base_dir}" +# +# # Init base +# mkdir -p "$__cdist_local_base_dir" +# ssh "${__cdist_remote_user}@${__cdist_target_host}" \ +# "mkdir -p ${__cdist_remote_base_dir}" +# +# # Link configuraion source directory - consistent with remote +# ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" if __name__ == "__main__": hostname=sys.argv[1] - cdist_echo("info", "cdist", __cdist_version, ": Configuring host", hostname) + logger("info", "cdist", __cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) From a90751dcff360e6ba2ceb5902ba90f43bdc6686e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:36:22 +0200 Subject: [PATCH 06/87] begin to implmenent remote execution Signed-off-by: Nico Schottelius --- bin/cdist | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 233e6fee..d680eaa6 100755 --- a/bin/cdist +++ b/bin/cdist @@ -34,25 +34,34 @@ def exit_error(*args): def run_or_fail(*args): + newargs = ["echo"] + newargs.extend(*args) + try: - subprocess.check_call(*args) + subprocess.check_call(newargs) except CalledProcessError: - exit_error("Command failed:", *args) + exit_error("Command failed:", newargs) + +def remote_run_or_fail(hostname, *args): + newargs = ["ssh", hostname] + newargs.extend(*args) + + run_or_fail(newargs) def cdist_deploy_to(hostname): """Mimic the old deploy to: Deploy to one host""" logger("info", "Deploying to host", hostname) + init_deploy(hostname) -def init_deploy(): +def init_deploy(hostname): logger("info", "Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote - run_or_fail(["echo rm -rf", "$__cdist_local_base_dir"]) + run_or_fail(["echo", "rm -rf", "$__cdist_local_base_dir"]) -# ssh "${__cdist_remote_user}@${__cdist_target_host}" \ -# "rm -rf ${__cdist_remote_base_dir}" + remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) # # # Init base # mkdir -p "$__cdist_local_base_dir" From 89964b32dd70c587f2b94ef9a4e027680ffb1274 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:41:54 +0200 Subject: [PATCH 07/87] in theory, init_deploy is done Signed-off-by: Nico Schottelius --- bin/cdist | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bin/cdist b/bin/cdist index d680eaa6..2ff2e011 100755 --- a/bin/cdist +++ b/bin/cdist @@ -59,17 +59,16 @@ def init_deploy(hostname): logger("info", "Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote - run_or_fail(["echo", "rm -rf", "$__cdist_local_base_dir"]) + run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) -# -# # Init base -# mkdir -p "$__cdist_local_base_dir" -# ssh "${__cdist_remote_user}@${__cdist_target_host}" \ -# "mkdir -p ${__cdist_remote_base_dir}" -# -# # Link configuraion source directory - consistent with remote -# ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" + + # Create base directories + run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) + remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) + + # Link configuraion source directory - consistent with remote + run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) if __name__ == "__main__": From ffd7f4b251bb4a61239b6a5afa274a6d5061e70c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:48:23 +0200 Subject: [PATCH 08/87] --todo Signed-off-by: Nico Schottelius --- bin/cdist-deploy-to | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to index bf5614bc..84181bbd 100755 --- a/bin/cdist-deploy-to +++ b/bin/cdist-deploy-to @@ -43,13 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " # See cdist-stages(7) # -# Prepare local and remote directories -__cdist_init_deploy "$__cdist_target_host" - -# Transfer cdist executables -__cdist_echo info "Transferring cdist binaries to the target host " -cdist-dir push "$__cdist_target_host" \ - "${__cdist_abs_mydir}" "${__cdist_remote_bin_dir}" cdist-explorer-run-global "$__cdist_target_host" cdist-manifest-run-init "$__cdist_target_host" cdist-object-all "$__cdist_target_host" cdist-object-prepare From c2873a8fa0b132003d1cd14b5bc53a6537360c71 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:12:48 +0200 Subject: [PATCH 09/87] ipmlement base_directory Signed-off-by: Nico Schottelius --- bin/cdist | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bin/cdist b/bin/cdist index 2ff2e011..440ec459 100755 --- a/bin/cdist +++ b/bin/cdist @@ -21,6 +21,7 @@ import sys # argv import subprocess # execute stuff +import os __cdist_version="2.0.0" @@ -54,6 +55,20 @@ def cdist_deploy_to(hostname): logger("info", "Deploying to host", hostname) init_deploy(hostname) +def base_directory(): + """Returns the directory in which all cdist stuff is based in""" + os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) + return os.getcwd() + +def global_explorer_directory(): + """Returns path to directory containing the global explorers""" + +def list_global_explorers(): + """Return list of available explorers""" + os.listdir(path=global_explorer_directory()) + +def explore(hostname, type=''): + """Run explorers""" def init_deploy(hostname): logger("info", "Creating clean directory structure") @@ -75,4 +90,5 @@ if __name__ == "__main__": hostname=sys.argv[1] logger("info", "cdist", __cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) + print(base_directory()) From 11750515e1bc1bbc821082bac1c901c4eac66ec6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:16:13 +0200 Subject: [PATCH 10/87] finish global_explorer_directory() Signed-off-by: Nico Schottelius --- bin/cdist | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 440ec459..2246f943 100755 --- a/bin/cdist +++ b/bin/cdist @@ -60,8 +60,13 @@ def base_directory(): os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) return os.getcwd() +def conf_directory(): + """Returns path to main configuration directory""" + return os.path.join(base_directory(), "conf") + def global_explorer_directory(): """Returns path to directory containing the global explorers""" + return os.path.join(conf_directory(), "explorer") def list_global_explorers(): """Return list of available explorers""" @@ -90,5 +95,5 @@ if __name__ == "__main__": hostname=sys.argv[1] logger("info", "cdist", __cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) - print(base_directory()) + print(global_explorer_directory()) From 09c58e2327281d91354460012e94f8ab455a8ed0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:39:24 +0200 Subject: [PATCH 11/87] exit, if there are no global explorers Signed-off-by: Nico Schottelius --- bin/cdist | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 2246f943..1019726a 100755 --- a/bin/cdist +++ b/bin/cdist @@ -30,7 +30,7 @@ def logger(type, *args): print(*args) def exit_error(*args): - logger(*args) + logger("error", *args) sys.exit(1) @@ -50,13 +50,9 @@ def remote_run_or_fail(hostname, *args): run_or_fail(newargs) -def cdist_deploy_to(hostname): - """Mimic the old deploy to: Deploy to one host""" - logger("info", "Deploying to host", hostname) - init_deploy(hostname) - def base_directory(): """Returns the directory in which all cdist stuff is based in""" + print("Going to", __file__, os.path.join(os.path.dirname(__file__), os.pardir)) os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) return os.getcwd() @@ -70,10 +66,14 @@ def global_explorer_directory(): def list_global_explorers(): """Return list of available explorers""" - os.listdir(path=global_explorer_directory()) + return os.listdir(global_explorer_directory()) -def explore(hostname, type=''): - """Run explorers""" +def global_explore(hostname): + """Run global explorers""" + explorer = list_global_explorers() + if(len(explorer) == 0): + exit_error("No explorers found in", global_explorer_directory()) + def init_deploy(hostname): logger("info", "Creating clean directory structure") @@ -90,10 +90,16 @@ def init_deploy(hostname): # Link configuraion source directory - consistent with remote run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) +def cdist_deploy_to(hostname): + """Mimic the old deploy to: Deploy to one host""" + logger("info", "Deploying to host", hostname) + init_deploy(hostname) + global_explore(hostname) + if __name__ == "__main__": hostname=sys.argv[1] logger("info", "cdist", __cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) - print(global_explorer_directory()) + print(list_global_explorers()) From 6e3c228a6f2c1b4603e29bf682608015cefab266 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:46:09 +0200 Subject: [PATCH 12/87] transfer_dir() added Signed-off-by: Nico Schottelius --- bin/cdist | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 1019726a..767d2fe9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -36,6 +36,7 @@ def exit_error(*args): def run_or_fail(*args): newargs = ["echo"] + newargs = [] newargs.extend(*args) try: @@ -49,6 +50,8 @@ def remote_run_or_fail(hostname, *args): run_or_fail(newargs) +def transfer_dir(hostname, source, destination): + run_or_fail(["scp", "-r", source, hostname + ":" + destination]) def base_directory(): """Returns the directory in which all cdist stuff is based in""" @@ -64,16 +67,22 @@ def global_explorer_directory(): """Returns path to directory containing the global explorers""" return os.path.join(conf_directory(), "explorer") +def remote_global_explorer_directory(): + """Returns path to directory containing the global explorers""" + return os.path.join(conf_directory(), "explorer") + def list_global_explorers(): """Return list of available explorers""" return os.listdir(global_explorer_directory()) +def transfer_global_explorers(hostname): + transfer_dir(hostname, global_explorer_directory(), remote_global_explorer_directory()) + def global_explore(hostname): """Run global explorers""" explorer = list_global_explorers() if(len(explorer) == 0): exit_error("No explorers found in", global_explorer_directory()) - def init_deploy(hostname): logger("info", "Creating clean directory structure") From 94fc5b3c911de461e7437518433257e3073e431c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:51:45 +0200 Subject: [PATCH 13/87] add remove_remote_dir() Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/cdist b/bin/cdist index 767d2fe9..94154974 100755 --- a/bin/cdist +++ b/bin/cdist @@ -50,7 +50,11 @@ def remote_run_or_fail(hostname, *args): run_or_fail(newargs) +def remove_remote_dir(hostname, destination): + remote_run_or_fail(hostname, ["rm", "-rf", destination]) + def transfer_dir(hostname, source, destination): + remove_remote_dir(hostname, destination) run_or_fail(["scp", "-r", source, hostname + ":" + destination]) def base_directory(): From 78a5bbf6e89fad6850f2f1d435c081918ee8dd35 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:52:38 +0200 Subject: [PATCH 14/87] remove the currently broken init() Signed-off-by: Nico Schottelius --- bin/cdist | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 94154974..fb09e8b2 100755 --- a/bin/cdist +++ b/bin/cdist @@ -92,16 +92,16 @@ def init_deploy(hostname): logger("info", "Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote - run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) - - remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) - - # Create base directories - run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) - remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) - - # Link configuraion source directory - consistent with remote - run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) +# run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) +# +# remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) +# +# # Create base directories +# run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) +# remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) +# +# # Link configuraion source directory - consistent with remote +# run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) def cdist_deploy_to(hostname): """Mimic the old deploy to: Deploy to one host""" From d0eeafd228fae25e1e8662311b7573022ab06be5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:56:02 +0200 Subject: [PATCH 15/87] also add some bugs Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++++++++-- doc/dev/todo/niconext | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index fb09e8b2..5cabf17b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -63,17 +63,24 @@ def base_directory(): os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) return os.getcwd() +def remote_base_directory(): + return "/var/lib/cdist" + def conf_directory(): """Returns path to main configuration directory""" return os.path.join(base_directory(), "conf") +def remote_conf_directory(): + """Returns path to remote main configuration directory""" + return os.path.join(remote_base_directory(), "conf") + def global_explorer_directory(): """Returns path to directory containing the global explorers""" return os.path.join(conf_directory(), "explorer") def remote_global_explorer_directory(): - """Returns path to directory containing the global explorers""" - return os.path.join(conf_directory(), "explorer") + """Returns path to the remote directory containing the global explorers""" + return os.path.join(remote_conf_directory(), "explorer") def list_global_explorers(): """Return list of available explorers""" diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index a59f909f..fce4033e 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,2 +1,4 @@ - rewrite in python? - support non-ssh access? + +- Bug: os.path.join() may be wrong for the remote side! From ef925714d5fc4aed09981d2c6de1a2088a24c881 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:57:47 +0200 Subject: [PATCH 16/87] transfer explorer Signed-off-by: Nico Schottelius --- bin/cdist | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 5cabf17b..7d28d112 100755 --- a/bin/cdist +++ b/bin/cdist @@ -45,7 +45,7 @@ def run_or_fail(*args): exit_error("Command failed:", newargs) def remote_run_or_fail(hostname, *args): - newargs = ["ssh", hostname] + newargs = ["ssh", "root@" + hostname] newargs.extend(*args) run_or_fail(newargs) @@ -55,7 +55,7 @@ def remove_remote_dir(hostname, destination): def transfer_dir(hostname, source, destination): remove_remote_dir(hostname, destination) - run_or_fail(["scp", "-r", source, hostname + ":" + destination]) + run_or_fail(["scp", "-r", source, "root@" + hostname + ":" + destination]) def base_directory(): """Returns the directory in which all cdist stuff is based in""" @@ -95,6 +95,8 @@ def global_explore(hostname): if(len(explorer) == 0): exit_error("No explorers found in", global_explorer_directory()) + transfer_global_explorers(hostname) + def init_deploy(hostname): logger("info", "Creating clean directory structure") From 85df71c9fa511b583076d500df25ab6cf384dc81 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:23:18 +0200 Subject: [PATCH 17/87] executing explorers works Signed-off-by: Nico Schottelius --- bin/cdist | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7d28d112..d2a0abd9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -30,14 +30,15 @@ def logger(type, *args): print(*args) def exit_error(*args): - logger("error", *args) + logger("error", args) sys.exit(1) def run_or_fail(*args): - newargs = ["echo"] + # newargs = ["echo"] newargs = [] newargs.extend(*args) + print(newargs) try: subprocess.check_call(newargs) @@ -45,9 +46,9 @@ def run_or_fail(*args): exit_error("Command failed:", newargs) def remote_run_or_fail(hostname, *args): + """Run something on the remote side and fail is something breaks""" newargs = ["ssh", "root@" + hostname] newargs.extend(*args) - run_or_fail(newargs) def remove_remote_dir(hostname, destination): @@ -82,6 +83,10 @@ def remote_global_explorer_directory(): """Returns path to the remote directory containing the global explorers""" return os.path.join(remote_conf_directory(), "explorer") +def remote_global_explorer_path(explorer): + """Returns path to the remote explorer""" + return os.path.join(remote_global_explorer_directory(), explorer) + def list_global_explorers(): """Return list of available explorers""" return os.listdir(global_explorer_directory()) @@ -91,11 +96,14 @@ def transfer_global_explorers(hostname): def global_explore(hostname): """Run global explorers""" - explorer = list_global_explorers() - if(len(explorer) == 0): + explorers = list_global_explorers() + if(len(explorers) == 0): exit_error("No explorers found in", global_explorer_directory()) transfer_global_explorers(hostname) + for explorer in explorers: + remote_run_or_fail(hostname, [remote_global_explorer_path(explorer)]) + def init_deploy(hostname): logger("info", "Creating clean directory structure") From 9036f8c4cce9edb5b353928855bdd995248cdb16 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:24:27 +0200 Subject: [PATCH 18/87] executing explorers works Signed-off-by: Nico Schottelius --- bin/cdist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/cdist b/bin/cdist index d2a0abd9..339349a3 100755 --- a/bin/cdist +++ b/bin/cdist @@ -103,6 +103,8 @@ def global_explore(hostname): transfer_global_explorers(hostname) for explorer in explorers: remote_run_or_fail(hostname, [remote_global_explorer_path(explorer)]) + + remote_run_or_fail(hostname, [remote_global_explorer_path("moo")]) def init_deploy(hostname): From ff2d5629d65a3bafe90df40718fd6ab393055527 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:28:06 +0200 Subject: [PATCH 19/87] submit string on failed error Signed-off-by: Nico Schottelius --- bin/cdist | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 339349a3..ab913541 100755 --- a/bin/cdist +++ b/bin/cdist @@ -30,7 +30,7 @@ def logger(type, *args): print(*args) def exit_error(*args): - logger("error", args) + logger("error", *args) sys.exit(1) @@ -42,8 +42,8 @@ def run_or_fail(*args): try: subprocess.check_call(newargs) - except CalledProcessError: - exit_error("Command failed:", newargs) + except subprocess.CalledProcessError: + exit_error("Command failed:", " ".join(newargs)) def remote_run_or_fail(hostname, *args): """Run something on the remote side and fail is something breaks""" @@ -104,8 +104,6 @@ def global_explore(hostname): for explorer in explorers: remote_run_or_fail(hostname, [remote_global_explorer_path(explorer)]) - remote_run_or_fail(hostname, [remote_global_explorer_path("moo")]) - def init_deploy(hostname): logger("info", "Creating clean directory structure") From 7a7413f1f48c9509bdd4e390f3d86017bf010ab7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:31:58 +0200 Subject: [PATCH 20/87] pause Signed-off-by: Nico Schottelius --- bin/cdist | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/cdist b/bin/cdist index ab913541..59d99040 100755 --- a/bin/cdist +++ b/bin/cdist @@ -75,6 +75,11 @@ def remote_conf_directory(): """Returns path to remote main configuration directory""" return os.path.join(remote_base_directory(), "conf") +def out_dir(): + FIXME: stopped - probably need static temp know! + """Local directory containing output""" + return os.path.join(base_directory(), "conf") + def global_explorer_directory(): """Returns path to directory containing the global explorers""" return os.path.join(conf_directory(), "explorer") From 572401e4f8010574df45423ccb1c21f056b719b9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:56:59 +0200 Subject: [PATCH 21/87] commit broken stuff Signed-off-by: Nico Schottelius --- bin/cdist | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 59d99040..e1d62391 100755 --- a/bin/cdist +++ b/bin/cdist @@ -22,8 +22,22 @@ import sys # argv import subprocess # execute stuff import os +import tempfile +import shutil -__cdist_version="2.0.0" + +class Cdist: + """Cdist main class to hold arbitrary data""" + version="2.0.0" + + def __init__(self): + self.tempdir = tempfile.mkdtemp() + print(self.tempdir) + + def __del__(self): + print("Zerstoeren") + print(self.tempdir) + shutil.rmtree(self.tempdir) def logger(type, *args): """Ignore type for now, support later""" @@ -76,7 +90,7 @@ def remote_conf_directory(): return os.path.join(remote_base_directory(), "conf") def out_dir(): - FIXME: stopped - probably need static temp know! + # FIXME: stopped - probably need static temp know! """Local directory containing output""" return os.path.join(base_directory(), "conf") @@ -134,7 +148,9 @@ def cdist_deploy_to(hostname): if __name__ == "__main__": hostname=sys.argv[1] - logger("info", "cdist", __cdist_version, ": Configuring host", hostname) +# logger("info", "cdist", cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) print(list_global_explorers()) + c = Cdist() + From e8a6d89e3c253897dd46863348a44eb606ad97cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 04:22:21 +0200 Subject: [PATCH 22/87] wrap into a class Signed-off-by: Nico Schottelius --- bin/cdist | 187 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 98 insertions(+), 89 deletions(-) diff --git a/bin/cdist b/bin/cdist index e1d62391..53728648 100755 --- a/bin/cdist +++ b/bin/cdist @@ -30,127 +30,136 @@ class Cdist: """Cdist main class to hold arbitrary data""" version="2.0.0" - def __init__(self): + def __init__(self, hostname): self.tempdir = tempfile.mkdtemp() + self.hostname = hostname + print(self.hostname) print(self.tempdir) - def __del__(self): - print("Zerstoeren") + def cleanup(self): + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # print(self.tempdir) shutil.rmtree(self.tempdir) -def logger(type, *args): - """Ignore type for now, support later""" - print(*args) + def out_dir(self): + # FIXME: stopped - probably need static temp know! + """Local directory containing output""" + return os.path.join(base_directory(), "conf") -def exit_error(*args): - logger("error", *args) - sys.exit(1) - + def logger(self,type, *args): + """Ignore type for now, support later""" + print(*args) -def run_or_fail(*args): - # newargs = ["echo"] - newargs = [] - newargs.extend(*args) - print(newargs) + def exit_error(self,*args): + self.logger("error", *args) + sys.exit(1) + - try: - subprocess.check_call(newargs) - except subprocess.CalledProcessError: - exit_error("Command failed:", " ".join(newargs)) + def run_or_fail(self,*args): + # newargs = ["echo"] + newargs = [] + newargs.extend(*args) + print(newargs) -def remote_run_or_fail(hostname, *args): - """Run something on the remote side and fail is something breaks""" - newargs = ["ssh", "root@" + hostname] - newargs.extend(*args) - run_or_fail(newargs) + try: + subprocess.check_call(newargs) + except subprocess.CalledProcessError: + self.exit_error("Command failed:", " ".join(newargs)) -def remove_remote_dir(hostname, destination): - remote_run_or_fail(hostname, ["rm", "-rf", destination]) + def remote_run_or_fail(self, *args): + """Run something on the remote side and fail is something breaks""" + newargs = ["ssh", "root@" + self.hostname] + newargs.extend(*args) + print(newargs, "bbbbb", self.hostname) + self.run_or_fail(newargs) -def transfer_dir(hostname, source, destination): - remove_remote_dir(hostname, destination) - run_or_fail(["scp", "-r", source, "root@" + hostname + ":" + destination]) + def remove_remote_dir(self, destination): + self.remote_run_or_fail(["rm", "-rf", destination]) -def base_directory(): - """Returns the directory in which all cdist stuff is based in""" - print("Going to", __file__, os.path.join(os.path.dirname(__file__), os.pardir)) - os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) - return os.getcwd() + def transfer_dir(self, source, destination): + self.remove_remote_dir(destination) + self.run_or_fail(["scp", "-qr", source, "root@" + self.hostname + ":" + destination]) -def remote_base_directory(): - return "/var/lib/cdist" + def base_directory(self): + """Returns the directory in which all cdist stuff is based in""" + print("Going to", __file__, os.path.join(os.path.dirname(__file__), os.pardir)) + os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) + return os.getcwd() -def conf_directory(): - """Returns path to main configuration directory""" - return os.path.join(base_directory(), "conf") + def remote_base_directory(self): + return "/var/lib/cdist" -def remote_conf_directory(): - """Returns path to remote main configuration directory""" - return os.path.join(remote_base_directory(), "conf") + def conf_directory(self): + """Returns path to main configuration directory""" + return os.path.join(self.base_directory(), "conf") -def out_dir(): - # FIXME: stopped - probably need static temp know! - """Local directory containing output""" - return os.path.join(base_directory(), "conf") + def remote_conf_directory(self): + """Returns path to remote main configuration directory""" + return os.path.join(self.remote_base_directory(), "conf") -def global_explorer_directory(): - """Returns path to directory containing the global explorers""" - return os.path.join(conf_directory(), "explorer") + def global_explorer_directory(self): + """Returns path to directory containing the global explorers""" + return os.path.join(self.conf_directory(), "explorer") -def remote_global_explorer_directory(): - """Returns path to the remote directory containing the global explorers""" - return os.path.join(remote_conf_directory(), "explorer") + def remote_global_explorer_directory(self): + """Returns path to the remote directory containing the global explorers""" + return os.path.join(self.remote_conf_directory(), "explorer") -def remote_global_explorer_path(explorer): - """Returns path to the remote explorer""" - return os.path.join(remote_global_explorer_directory(), explorer) + def remote_global_explorer_path(self,explorer): + """Returns path to the remote explorer""" + return os.path.join(self.remote_global_explorer_directory(), explorer) -def list_global_explorers(): - """Return list of available explorers""" - return os.listdir(global_explorer_directory()) + def list_global_explorers(self): + """Return list of available explorers""" + return os.listdir(self.global_explorer_directory()) -def transfer_global_explorers(hostname): - transfer_dir(hostname, global_explorer_directory(), remote_global_explorer_directory()) + def transfer_global_explorers(self): + self.transfer_dir(self.global_explorer_directory(), self.remote_global_explorer_directory()) -def global_explore(hostname): - """Run global explorers""" - explorers = list_global_explorers() - if(len(explorers) == 0): - exit_error("No explorers found in", global_explorer_directory()) + def global_explore(self): + """Run global explorers""" + explorers = self.list_global_explorers() + if(len(explorers) == 0): + self.exit_error("No explorers found in", self.global_explorer_directory()) - transfer_global_explorers(hostname) - for explorer in explorers: - remote_run_or_fail(hostname, [remote_global_explorer_path(explorer)]) + self.transfer_global_explorers() + for explorer in explorers: + self.remote_run_or_fail([self.remote_global_explorer_path(explorer)]) -def init_deploy(hostname): - logger("info", "Creating clean directory structure") + def init_deploy(self): + self.logger("info", "Creating clean directory structure") - # Ensure there is no old stuff, neither local nor remote -# run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) -# -# remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) -# -# # Create base directories -# run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) -# remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) -# -# # Link configuraion source directory - consistent with remote -# run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) + # Ensure there is no old stuff, neither local nor remote + # run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) + # + # remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) + # + # # Create base directories + # run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) + # remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) + # + # # Link configuraion source directory - consistent with remote + # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) -def cdist_deploy_to(hostname): - """Mimic the old deploy to: Deploy to one host""" - logger("info", "Deploying to host", hostname) - init_deploy(hostname) - global_explore(hostname) + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + self.logger("info", "Deploying to host", self.hostname) + self.init_deploy() + self.global_explore() if __name__ == "__main__": hostname=sys.argv[1] # logger("info", "cdist", cdist_version, ": Configuring host", hostname) - cdist_deploy_to(hostname) - print(list_global_explorers()) - c = Cdist() + for host in sys.argv[1:]: + c = Cdist(host) + c.deploy_to() + print(c.list_global_explorers()) + c.cleanup() From 0dc6af251219e267791ec6d4f08667c38084f991 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 04:22:40 +0200 Subject: [PATCH 23/87] wrap into a class Signed-off-by: Nico Schottelius --- bin/cdist | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 53728648..6770c181 100755 --- a/bin/cdist +++ b/bin/cdist @@ -74,7 +74,6 @@ class Cdist: """Run something on the remote side and fail is something breaks""" newargs = ["ssh", "root@" + self.hostname] newargs.extend(*args) - print(newargs, "bbbbb", self.hostname) self.run_or_fail(newargs) def remove_remote_dir(self, destination): From e8360df96b6bb9ae4641046e862d1a414ebfc03c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 09:47:30 +0200 Subject: [PATCH 24/87] allow passing arguments to Popen() Signed-off-by: Nico Schottelius --- bin/cdist | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 6770c181..39a0b80b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -59,22 +59,22 @@ class Cdist: sys.exit(1) - def run_or_fail(self,*args): + def run_or_fail(self,*args, **kargs): # newargs = ["echo"] newargs = [] newargs.extend(*args) print(newargs) try: - subprocess.check_call(newargs) + subprocess.check_call(newargs, **kargs) except subprocess.CalledProcessError: self.exit_error("Command failed:", " ".join(newargs)) - def remote_run_or_fail(self, *args): + def remote_run_or_fail(self, *args, **kargs): """Run something on the remote side and fail is something breaks""" newargs = ["ssh", "root@" + self.hostname] newargs.extend(*args) - self.run_or_fail(newargs) + self.run_or_fail(newargs, **kargs) def remove_remote_dir(self, destination): self.remote_run_or_fail(["rm", "-rf", destination]) From 4c1939829e26ae8ba73f48793409daec8c5d0c3f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:08:44 +0200 Subject: [PATCH 25/87] import logging, begin to put constant stuff in module Signed-off-by: Nico Schottelius --- bin/cdist | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/bin/cdist b/bin/cdist index 39a0b80b..162bf1aa 100755 --- a/bin/cdist +++ b/bin/cdist @@ -19,22 +19,36 @@ # # -import sys # argv +import sys import subprocess # execute stuff import os import tempfile import shutil +import logging +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + +logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s') + class Cdist: """Cdist main class to hold arbitrary data""" version="2.0.0" def __init__(self, hostname): - self.tempdir = tempfile.mkdtemp() self.hostname = hostname - print(self.hostname) - print(self.tempdir) + + # Setup directory paths + self.temp_dir = tempfile.mkdtemp() + + self.out_dir = os.path.join(self.temp_dir, "out") + os.mkdir(self.out_dir) + + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + os.mkdir(self.global_explorer_out_dir) + + # Given paths from installation + self.conf_dir = os.path.join(self.base_dir, "conf") def cleanup(self): # Do not use in __del__: @@ -45,11 +59,6 @@ class Cdist: print(self.tempdir) shutil.rmtree(self.tempdir) - def out_dir(self): - # FIXME: stopped - probably need static temp know! - """Local directory containing output""" - return os.path.join(base_directory(), "conf") - def logger(self,type, *args): """Ignore type for now, support later""" print(*args) @@ -83,18 +92,9 @@ class Cdist: self.remove_remote_dir(destination) self.run_or_fail(["scp", "-qr", source, "root@" + self.hostname + ":" + destination]) - def base_directory(self): - """Returns the directory in which all cdist stuff is based in""" - print("Going to", __file__, os.path.join(os.path.dirname(__file__), os.pardir)) - os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) - return os.getcwd() - def remote_base_directory(self): return "/var/lib/cdist" - def conf_directory(self): - """Returns path to main configuration directory""" - return os.path.join(self.base_directory(), "conf") def remote_conf_directory(self): """Returns path to remote main configuration directory""" @@ -112,6 +112,10 @@ class Cdist: """Returns path to the remote explorer""" return os.path.join(self.remote_global_explorer_directory(), explorer) + def global_explorer_output_path(self, explorer): + """Returns path of the output for a global explorer""" + return os.path.join(self.global_explorer_out_dir, explorer) + def list_global_explorers(self): """Return list of available explorers""" return os.listdir(self.global_explorer_directory()) @@ -127,7 +131,10 @@ class Cdist: self.transfer_global_explorers() for explorer in explorers: - self.remote_run_or_fail([self.remote_global_explorer_path(explorer)]) + output = self.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + self.remote_run_or_fail([self.remote_global_explorer_path(explorer)], stdout=output_fd) + output_fd.close() def init_deploy(self): From 7d3c67c178a1996eb7a6633040c9a30668f56fda Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:44:04 +0200 Subject: [PATCH 26/87] make it work again :-) Signed-off-by: Nico Schottelius --- bin/cdist | 81 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/bin/cdist b/bin/cdist index 162bf1aa..c160b945 100755 --- a/bin/cdist +++ b/bin/cdist @@ -27,9 +27,37 @@ import shutil import logging +# Given paths from installation BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) +CONF_DIR = os.path.join(BASE_DIR, "conf") +GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") + + +#class Context(object): +# +# def __init__(self, target_host): +# self.target_host = target_host +# +# # class variable +# user_selber_shuld_wenn_aendert = 'bla' +# +# # read only, aber statisch +# @property +# def remote_base_directory(self): +# return "/var/lib/cdist" +# @property.setter +# +# @property +# def special_foo(self): +# return 'foo/{0}'.format(self.target_host) +# + logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s') +log = logging.getLogger() class Cdist: """Cdist main class to hold arbitrary data""" @@ -38,35 +66,27 @@ class Cdist: def __init__(self, hostname): self.hostname = hostname + # log.info("foobar") + # Setup directory paths self.temp_dir = tempfile.mkdtemp() - self.out_dir = os.path.join(self.temp_dir, "out") os.mkdir(self.out_dir) - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) - # Given paths from installation - self.conf_dir = os.path.join(self.base_dir, "conf") - def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization # "other globals referenced by the __del__() method may already have been deleted # or in the process of being torn down (e.g. the import machinery shutting down)" # - print(self.tempdir) - shutil.rmtree(self.tempdir) - - def logger(self,type, *args): - """Ignore type for now, support later""" - print(*args) + print(self.temp_dir) + shutil.rmtree(self.temp_dir) def exit_error(self,*args): - self.logger("error", *args) + log.error(args) sys.exit(1) - def run_or_fail(self,*args, **kargs): # newargs = ["echo"] @@ -92,42 +112,26 @@ class Cdist: self.remove_remote_dir(destination) self.run_or_fail(["scp", "-qr", source, "root@" + self.hostname + ":" + destination]) - def remote_base_directory(self): - return "/var/lib/cdist" - - - def remote_conf_directory(self): - """Returns path to remote main configuration directory""" - return os.path.join(self.remote_base_directory(), "conf") - - def global_explorer_directory(self): - """Returns path to directory containing the global explorers""" - return os.path.join(self.conf_directory(), "explorer") - - def remote_global_explorer_directory(self): - """Returns path to the remote directory containing the global explorers""" - return os.path.join(self.remote_conf_directory(), "explorer") - - def remote_global_explorer_path(self,explorer): - """Returns path to the remote explorer""" - return os.path.join(self.remote_global_explorer_directory(), explorer) - def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) + def remote_global_explorer_path(self, explorer): + """Returns path to the remote explorer""" + return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + def list_global_explorers(self): """Return list of available explorers""" - return os.listdir(self.global_explorer_directory()) + return os.listdir(GLOBAL_EXPLORER_DIR) def transfer_global_explorers(self): - self.transfer_dir(self.global_explorer_directory(), self.remote_global_explorer_directory()) + self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) def global_explore(self): """Run global explorers""" explorers = self.list_global_explorers() if(len(explorers) == 0): - self.exit_error("No explorers found in", self.global_explorer_directory()) + self.exit_error("No explorers found in", GLOBAL_EXPLORER_DIR) self.transfer_global_explorers() for explorer in explorers: @@ -138,7 +142,7 @@ class Cdist: def init_deploy(self): - self.logger("info", "Creating clean directory structure") + log.info("Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote # run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) @@ -154,14 +158,13 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - self.logger("info", "Deploying to host", self.hostname) + log.info("Deploying to host", self.hostname) self.init_deploy() self.global_explore() if __name__ == "__main__": hostname=sys.argv[1] -# logger("info", "cdist", cdist_version, ": Configuring host", hostname) for host in sys.argv[1:]: c = Cdist(host) From 8fa576152ee2666c24e6d53dd697a40d78940980 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:44:58 +0200 Subject: [PATCH 27/87] make it work again :-) Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index c160b945..1e2fe11e 100755 --- a/bin/cdist +++ b/bin/cdist @@ -158,7 +158,7 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to host", self.hostname) + log.info("Deploying to host" + self.hostname) self.init_deploy() self.global_explore() From 506e0e6c81d340f424cf589b16d7e8672a5d824f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:45:58 +0200 Subject: [PATCH 28/87] +better logging Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 1e2fe11e..56991d68 100755 --- a/bin/cdist +++ b/bin/cdist @@ -158,7 +158,7 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to host" + self.hostname) + log.info("Deploying to host " + self.hostname) self.init_deploy() self.global_explore() From 908d9d06f9737a37a1558313836c65b3a172fce2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:50:30 +0200 Subject: [PATCH 29/87] prepend __explorer on global explorer run Signed-off-by: Nico Schottelius --- bin/cdist | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 56991d68..d6421161 100755 --- a/bin/cdist +++ b/bin/cdist @@ -81,8 +81,8 @@ class Cdist: # "other globals referenced by the __del__() method may already have been deleted # or in the process of being torn down (e.g. the import machinery shutting down)" # - print(self.temp_dir) - shutil.rmtree(self.temp_dir) + print("I should cleanup " + self.temp_dir) + # shutil.rmtree(self.temp_dir) def exit_error(self,*args): log.error(args) @@ -137,7 +137,11 @@ class Cdist: for explorer in explorers: output = self.global_explorer_output_path(explorer) output_fd = open(output, mode='w') - self.remote_run_or_fail([self.remote_global_explorer_path(explorer)], stdout=output_fd) + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.remote_global_explorer_path(explorer)) + + self.remote_run_or_fail(cmd, stdout=output_fd) output_fd.close() @@ -145,12 +149,9 @@ class Cdist: log.info("Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote - # run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) - # # remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) # # # Create base directories - # run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) # remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) # # # Link configuraion source directory - consistent with remote From ddff3e8b571fd907e423ea8888c482eb1f4c05f6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:55:29 +0200 Subject: [PATCH 30/87] support changing the initial manifest Signed-off-by: Nico Schottelius --- bin/cdist | 14 +++++++++++++- bin/cdist-deploy-to | 1 - 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index d6421161..766a2acf 100755 --- a/bin/cdist +++ b/bin/cdist @@ -31,6 +31,8 @@ import logging BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) CONF_DIR = os.path.join(BASE_DIR, "conf") GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") +MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") + REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") @@ -63,7 +65,7 @@ class Cdist: """Cdist main class to hold arbitrary data""" version="2.0.0" - def __init__(self, hostname): + def __init__(self, hostname, initial_manifest=False): self.hostname = hostname # log.info("foobar") @@ -75,6 +77,12 @@ class Cdist: self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) + # Mostly static, but can be overwritten on user demand + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(MANIFEST_DIR, "init") + def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization @@ -157,11 +165,15 @@ class Cdist: # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) + def initial_manifes(self): + """Run the initial manifest""" + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to host " + self.hostname) self.init_deploy() self.global_explore() + self.initial_manifest() if __name__ == "__main__": diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to index 84181bbd..79f42931 100755 --- a/bin/cdist-deploy-to +++ b/bin/cdist-deploy-to @@ -43,7 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " # See cdist-stages(7) # -cdist-explorer-run-global "$__cdist_target_host" cdist-manifest-run-init "$__cdist_target_host" cdist-object-all "$__cdist_target_host" cdist-object-prepare cdist-object-all "$__cdist_target_host" cdist-object-run From 9533e579b390c171f8147211d63866935b2d683b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:04:30 +0200 Subject: [PATCH 31/87] finish run of initial manifest, finish shell_run_or_debug_fail() Signed-off-by: Nico Schottelius --- bin/cdist | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 766a2acf..594b5eea 100755 --- a/bin/cdist +++ b/bin/cdist @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) # @@ -96,11 +97,23 @@ class Cdist: log.error(args) sys.exit(1) - def run_or_fail(self,*args, **kargs): + def shell_run_or_debug_fail(self, script, *args, **kargs): + kargs['shell'] = True + log.debug("Shell exec: " + " ".join(*args)) + + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + # FIXME: print out shell script! + script_fd = open(script) + log.error("Code that raised the error:\n" + script_fd.read()) + script_fd.close() + self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) + + def run_or_fail(self, *args, **kargs): # newargs = ["echo"] newargs = [] newargs.extend(*args) - print(newargs) try: subprocess.check_call(newargs, **kargs) @@ -165,15 +178,18 @@ class Cdist: # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) - def initial_manifes(self): + def run_initial_manifest(self): """Run the initial manifest""" + log.info("Running the initial manifest") + self.shell_run_or_debug_fail(self.initial_manifest, [self.initial_manifest]) + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to host " + self.hostname) self.init_deploy() self.global_explore() - self.initial_manifest() + self.run_initial_manifest() if __name__ == "__main__": From c5d960438cc111c5476b0261628b0ffcb1d25dbd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:05:20 +0200 Subject: [PATCH 32/87] import parser from steven Signed-off-by: Nico Schottelius --- bin/cdist | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/bin/cdist b/bin/cdist index 594b5eea..72e95e29 100755 --- a/bin/cdist +++ b/bin/cdist @@ -201,3 +201,41 @@ if __name__ == "__main__": print(c.list_global_explorers()) c.cleanup() + +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import logging +# TODO: configure logging based on config and/or user given arguments +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +log = logging.getLogger() + +import argparse +import sys + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Some helpfull blabla') + parser.add_argument('host', nargs='+', help='one or more hosts to operate on') + parser.add_argument('-i', '--initial-manifest', + help='path to a cdist manifest or - to read from stdin', + dest='manifest', required=True) + parser.add_argument('-p', '--parallel', + help='operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser.add_argument('-s', '--sequential', + help='operate on multiple hosts sequentially', + action='store_false', dest='parallel') + parser.add_argument('-d', '--debug', help='set log level to debug', + action='store_true') + + args = parser.parse_args(sys.argv[1:]) + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug('Look ma, now whe\'re showing debug messages') + + try: + print(args) + #import cdist + #sys.exit(cdist.main(args)) + except KeyboardInterrupt: + sys.exit(0) From 979174a5685ae810327f98b04a8cf522c504f15c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:11:31 +0200 Subject: [PATCH 33/87] integrate parser Signed-off-by: Nico Schottelius --- bin/cdist | 86 +++++++++++++++++++++++-------------------------------- 1 file changed, 36 insertions(+), 50 deletions(-) diff --git a/bin/cdist b/bin/cdist index 72e95e29..a9b400ca 100755 --- a/bin/cdist +++ b/bin/cdist @@ -20,13 +20,13 @@ # # -import sys -import subprocess # execute stuff -import os -import tempfile -import shutil +import argparse import logging - +import os +import subprocess +import shutil +import sys +import tempfile # Given paths from installation BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) @@ -37,6 +37,7 @@ MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") +VERSION = "2.0.0" #class Context(object): @@ -64,7 +65,6 @@ log = logging.getLogger() class Cdist: """Cdist main class to hold arbitrary data""" - version="2.0.0" def __init__(self, hostname, initial_manifest=False): self.hostname = hostname @@ -195,47 +195,33 @@ class Cdist: if __name__ == "__main__": hostname=sys.argv[1] - for host in sys.argv[1:]: - c = Cdist(host) - c.deploy_to() - print(c.list_global_explorers()) - c.cleanup() + parser = argparse.ArgumentParser(description='cdist ' + VERSION) + parser.add_argument('host', nargs='+', help='one or more hosts to operate on') + parser.add_argument('-d', '--debug', help='set log level to debug', + action='store_true') + parser.add_argument('-i', '--initial-manifest', + help='path to a cdist manifest or - to read from stdin', + dest='manifest', required=True) + parser.add_argument('-p', '--parallel', + help='operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser.add_argument('-s', '--sequential', + help='operate on multiple hosts sequentially', + action='store_false', dest='parallel') + + args = parser.parse_args(sys.argv[1:]) + if args.debug: + logging.root.setLevel(logging.DEBUG) + + try: + print(args) + import cdist + sys.exit(cdist.main(args)) - -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import logging -# TODO: configure logging based on config and/or user given arguments -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -log = logging.getLogger() - -import argparse -import sys - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Some helpfull blabla') - parser.add_argument('host', nargs='+', help='one or more hosts to operate on') - parser.add_argument('-i', '--initial-manifest', - help='path to a cdist manifest or - to read from stdin', - dest='manifest', required=True) - parser.add_argument('-p', '--parallel', - help='operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser.add_argument('-s', '--sequential', - help='operate on multiple hosts sequentially', - action='store_false', dest='parallel') - parser.add_argument('-d', '--debug', help='set log level to debug', - action='store_true') - - args = parser.parse_args(sys.argv[1:]) - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug('Look ma, now whe\'re showing debug messages') - - try: - print(args) - #import cdist - #sys.exit(cdist.main(args)) - except KeyboardInterrupt: - sys.exit(0) + for host in sys.argv[1:]: + c = Cdist(host) + c.deploy_to() + print(c.list_global_explorers()) + c.cleanup() + except KeyboardInterrupt: + sys.exit(0) From 1d367d5f581e246793e44016ec1df0836b822e6e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:17:44 +0200 Subject: [PATCH 34/87] make cdist localhost work again Signed-off-by: Nico Schottelius --- bin/cdist | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index a9b400ca..58ddcb3f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -201,7 +201,7 @@ if __name__ == "__main__": action='store_true') parser.add_argument('-i', '--initial-manifest', help='path to a cdist manifest or - to read from stdin', - dest='manifest', required=True) + dest='manifest', required=False) parser.add_argument('-p', '--parallel', help='operate on multiple hosts in parallel', action='store_true', dest='parallel') @@ -214,11 +214,9 @@ if __name__ == "__main__": logging.root.setLevel(logging.DEBUG) try: - print(args) - import cdist - sys.exit(cdist.main(args)) + log.debug(args) - for host in sys.argv[1:]: + for host in args.host: c = Cdist(host) c.deploy_to() print(c.list_global_explorers()) From c8ce7e98ec2da0bfad43272adab71a2b43df2303 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:45:54 +0200 Subject: [PATCH 35/87] use -e to shell Signed-off-by: Nico Schottelius --- bin/cdist | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 58ddcb3f..a5902472 100755 --- a/bin/cdist +++ b/bin/cdist @@ -101,6 +101,10 @@ class Cdist: kargs['shell'] = True log.debug("Shell exec: " + " ".join(*args)) + # Fail if the script fails + args[0].insert(0,"-e") + print(args) + try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: @@ -219,7 +223,6 @@ if __name__ == "__main__": for host in args.host: c = Cdist(host) c.deploy_to() - print(c.list_global_explorers()) c.cleanup() except KeyboardInterrupt: sys.exit(0) From 1a96f886567f676f0ea149d73fc7ef9546496b55 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:50:45 +0200 Subject: [PATCH 36/87] name variable target_host for consistency and easy handling Signed-off-by: Nico Schottelius --- bin/cdist | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index a5902472..323d2a96 100755 --- a/bin/cdist +++ b/bin/cdist @@ -66,8 +66,8 @@ log = logging.getLogger() class Cdist: """Cdist main class to hold arbitrary data""" - def __init__(self, hostname, initial_manifest=False): - self.hostname = hostname + def __init__(self, target_host, initial_manifest=False): + self.target_host = target_host # log.info("foobar") @@ -126,7 +126,7 @@ class Cdist: def remote_run_or_fail(self, *args, **kargs): """Run something on the remote side and fail is something breaks""" - newargs = ["ssh", "root@" + self.hostname] + newargs = ["ssh", "root@" + self.target_host] newargs.extend(*args) self.run_or_fail(newargs, **kargs) @@ -135,7 +135,7 @@ class Cdist: def transfer_dir(self, source, destination): self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, "root@" + self.hostname + ":" + destination]) + self.run_or_fail(["scp", "-qr", source, "root@" + self.target_host + ":" + destination]) def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" @@ -185,20 +185,23 @@ class Cdist: def run_initial_manifest(self): """Run the initial manifest""" log.info("Running the initial manifest") - self.shell_run_or_debug_fail(self.initial_manifest, [self.initial_manifest]) + env = os.environ.copy() + env['__target_host'] = self.target_host + + self.shell_run_or_debug_fail(self.initial_manifest, + [self.initial_manifest], + env=env) def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to host " + self.hostname) + log.info("Deploying to host " + self.target_host) self.init_deploy() self.global_explore() self.run_initial_manifest() if __name__ == "__main__": - hostname=sys.argv[1] - parser = argparse.ArgumentParser(description='cdist ' + VERSION) parser.add_argument('host', nargs='+', help='one or more hosts to operate on') parser.add_argument('-d', '--debug', help='set log level to debug', From 4fd8a16e6cd618ed61c9e312d5e3827bb4d15b2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:52:42 +0200 Subject: [PATCH 37/87] use *args not args :-) Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 323d2a96..7059f73b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -93,8 +93,8 @@ class Cdist: print("I should cleanup " + self.temp_dir) # shutil.rmtree(self.temp_dir) - def exit_error(self,*args): - log.error(args) + def exit_error(self, *args): + log.error(*args) sys.exit(1) def shell_run_or_debug_fail(self, script, *args, **kargs): From 2c2a234c74ce1a5641d3683f6ef947627585b007 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:13:05 +0200 Subject: [PATCH 38/87] use sh -e instead of shipped shell=true (see included comment) Signed-off-by: Nico Schottelius --- bin/cdist | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7059f73b..1216edab 100755 --- a/bin/cdist +++ b/bin/cdist @@ -98,13 +98,12 @@ class Cdist: sys.exit(1) def shell_run_or_debug_fail(self, script, *args, **kargs): - kargs['shell'] = True + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0].insert(0,"/bin/sh") + args[0].insert(1,"-e") log.debug("Shell exec: " + " ".join(*args)) - # Fail if the script fails - args[0].insert(0,"-e") - print(args) - try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: @@ -115,14 +114,12 @@ class Cdist: self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) def run_or_fail(self, *args, **kargs): - # newargs = ["echo"] - newargs = [] - newargs.extend(*args) + log.debug("Exec: " + " ".join(*args)) try: - subprocess.check_call(newargs, **kargs) + subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - self.exit_error("Command failed:", " ".join(newargs)) + self.exit_error("Command failed:", " ".join(*args)) def remote_run_or_fail(self, *args, **kargs): """Run something on the remote side and fail is something breaks""" From 97f7513f6eae5796fc9046530fe528d3e083dd6d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:14:28 +0200 Subject: [PATCH 39/87] -old fixme Signed-off-by: Nico Schottelius --- bin/cdist | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 1216edab..857e127d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -107,7 +107,6 @@ class Cdist: try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - # FIXME: print out shell script! script_fd = open(script) log.error("Code that raised the error:\n" + script_fd.read()) script_fd.close() From d6db6d0c7e22b45e970299bcfcbb641dfdcc774a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:20:25 +0200 Subject: [PATCH 40/87] begin move of cdist-type-emulator to lib/ and submit initial_manifest to cdist Signed-off-by: Nico Schottelius --- bin/cdist | 3 ++- {bin => lib}/cdist-type-emulator | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename {bin => lib}/cdist-type-emulator (100%) diff --git a/bin/cdist b/bin/cdist index 857e127d..6e13188d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -32,6 +32,7 @@ import tempfile BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) CONF_DIR = os.path.join(BASE_DIR, "conf") GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") +LIB_DIR = os.path.join(BASE_DIR, "lib") MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") REMOTE_BASE_DIR = "/var/lib/cdist" @@ -220,7 +221,7 @@ if __name__ == "__main__": log.debug(args) for host in args.host: - c = Cdist(host) + c = Cdist(host, initial_manifest=args.manifest) c.deploy_to() c.cleanup() except KeyboardInterrupt: diff --git a/bin/cdist-type-emulator b/lib/cdist-type-emulator similarity index 100% rename from bin/cdist-type-emulator rename to lib/cdist-type-emulator From 9ce26e5817907105c20ab9503933578717d14352 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:35:29 +0200 Subject: [PATCH 41/87] include cdist-type-emulator into path again Signed-off-by: Nico Schottelius --- bin/cdist | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bin/cdist b/bin/cdist index 6e13188d..0cc4ac7f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -34,6 +34,7 @@ CONF_DIR = os.path.join(BASE_DIR, "conf") GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") LIB_DIR = os.path.join(BASE_DIR, "lib") MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") +TYPE_DIR = os.path.join(CONF_DIR, "type") REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") @@ -64,6 +65,10 @@ VERSION = "2.0.0" logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s') log = logging.getLogger() +# List types +def list_types(): + return os.listdir(TYPE_DIR) + class Cdist: """Cdist main class to hold arbitrary data""" @@ -74,11 +79,18 @@ class Cdist: # Setup directory paths self.temp_dir = tempfile.mkdtemp() + self.out_dir = os.path.join(self.temp_dir, "out") os.mkdir(self.out_dir) + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) + # Setup binary directory + contents + self.bin_dir = os.path.join(self.out_dir, "bin") + os.mkdir(self.bin_dir) + self.link_type_to_emulator() + # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest @@ -149,6 +161,14 @@ class Cdist: def transfer_global_explorers(self): self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) + def link_type_to_emulator(self): + """Link type names to cdist-type-emulator""" + for type in list_types(): + source = os.path.join(LIB_DIR, "cdist-type-emulator") + destination = os.path.join(self.bin_dir, type) + log.debug("Linking %s to %s", source, destination) + os.symlink(source, destination) + def global_explore(self): """Run global explorers""" explorers = self.list_global_explorers() @@ -184,6 +204,7 @@ class Cdist: log.info("Running the initial manifest") env = os.environ.copy() env['__target_host'] = self.target_host + env['PATH'] = self.bin_dir + ":" + env['PATH'] self.shell_run_or_debug_fail(self.initial_manifest, [self.initial_manifest], From bf0f2b8f12949bfcd988f95533987f23d54d9695 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:39:21 +0200 Subject: [PATCH 42/87] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index fce4033e..972525aa 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,4 +1,5 @@ - rewrite in python? + - also do with cdist-type-emulator, which had quirks applied from outside to run - support non-ssh access? - Bug: os.path.join() may be wrong for the remote side! From dbbbf822fe55f714c115fa4e610b62361dfd614e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:52:34 +0200 Subject: [PATCH 43/87] make cdist-type-emulator work Signed-off-by: Nico Schottelius --- bin/cdist | 15 ++++++++++++--- bin/cdist-deploy-to | 1 - 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 0cc4ac7f..57df474d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -169,7 +169,7 @@ class Cdist: log.debug("Linking %s to %s", source, destination) os.symlink(source, destination) - def global_explore(self): + def run_global_explores(self): """Run global explorers""" explorers = self.list_global_explorers() if(len(explorers) == 0): @@ -186,6 +186,7 @@ class Cdist: self.remote_run_or_fail(cmd, stdout=output_fd) output_fd.close() + # def run_type_explorer(self): def init_deploy(self): log.info("Creating clean directory structure") @@ -203,9 +204,17 @@ class Cdist: """Run the initial manifest""" log.info("Running the initial manifest") env = os.environ.copy() - env['__target_host'] = self.target_host env['PATH'] = self.bin_dir + ":" + env['PATH'] + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + + # Legacy stuff to make cdist-type-emulator work + env['__cdist_conf_dir'] = CONF_DIR + env['__cdist_core_dir'] = os.path.join(BASE_DIR, "core") + env['__cdist_local_base_dir'] = self.temp_dir + env['__cdist_manifest'] = self.initial_manifest + self.shell_run_or_debug_fail(self.initial_manifest, [self.initial_manifest], env=env) @@ -215,7 +224,7 @@ class Cdist: """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to host " + self.target_host) self.init_deploy() - self.global_explore() + self.run_global_explores() self.run_initial_manifest() diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to index 79f42931..96650f90 100755 --- a/bin/cdist-deploy-to +++ b/bin/cdist-deploy-to @@ -43,7 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " # See cdist-stages(7) # -cdist-manifest-run-init "$__cdist_target_host" cdist-object-all "$__cdist_target_host" cdist-object-prepare cdist-object-all "$__cdist_target_host" cdist-object-run cdist-cache "$__cdist_target_host" From 0f36ddd649d5646e2955ff219524a31f1707f61e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:54:10 +0200 Subject: [PATCH 44/87] support object dir Signed-off-by: Nico Schottelius --- bin/cdist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/cdist b/bin/cdist index 57df474d..d7e2f697 100755 --- a/bin/cdist +++ b/bin/cdist @@ -86,6 +86,8 @@ class Cdist: self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) + self.object_dir = os.path.join(self.out_dir, "object") + # Setup binary directory + contents self.bin_dir = os.path.join(self.out_dir, "bin") os.mkdir(self.bin_dir) From 03e2db83cd1234e55ca2b4801f31e3094e3516b8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 17:01:22 +0200 Subject: [PATCH 45/87] implement list of objects Signed-off-by: Nico Schottelius --- bin/cdist | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/bin/cdist b/bin/cdist index d7e2f697..8655dcad 100755 --- a/bin/cdist +++ b/bin/cdist @@ -160,6 +160,44 @@ class Cdist: """Return list of available explorers""" return os.listdir(GLOBAL_EXPLORER_DIR) + def list_object_paths(self, starting_point = False): + """Return list of paths of existing objects""" + object_paths = [] + + if not starting_point: + starting_point = self.object_dir + + for content in os.listdir(starting_point): + full_path = os.path.join(starting_point, content) + print(full_path) + if os.path.isdir(full_path): + log.debug("Recursing for %s", full_path) + object_paths.extend(self.list_object_paths(starting_point = full_path)) + + # Directory contains .cdist -> is an object + if content == ".cdist": + log.debug("Adding Object Path %s", starting_point) + object_paths.append(starting_point) + + return object_paths + + def list_objects(self, starting_point = False): + """Return list of existing objects""" + + if not starting_point: + starting_point = self.object_dir + + object_paths = self.list_object_paths(starting_point) + objects = [] + log.debug("Paths recieved: %s", object_paths) + log.debug("And te starting point: %s", starting_point) + + for path in object_paths: + objects.append(os.path.relpath(path, starting_point)) + + return objects + + def transfer_global_explorers(self): self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) From 4d765a702406de7b351174421e6679cb6507cdd3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 17:45:07 +0200 Subject: [PATCH 46/87] begin type explorer transfer Signed-off-by: Nico Schottelius --- bin/cdist | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 8655dcad..4e62bd64 100755 --- a/bin/cdist +++ b/bin/cdist @@ -38,7 +38,10 @@ TYPE_DIR = os.path.join(CONF_DIR, "type") REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") + +DOT_CDIST = ".cdist" VERSION = "2.0.0" @@ -93,6 +96,9 @@ class Cdist: os.mkdir(self.bin_dir) self.link_type_to_emulator() + # List of type explorers transferred + self.type_explorers_transferred = {} + # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest @@ -175,12 +181,15 @@ class Cdist: object_paths.extend(self.list_object_paths(starting_point = full_path)) # Directory contains .cdist -> is an object - if content == ".cdist": + if content == DOT_CDIST: log.debug("Adding Object Path %s", starting_point) object_paths.append(starting_point) return object_paths + def get_type_from_object(cdist_object): + return cdist_object.split(os.sep)[0] + def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -197,10 +206,20 @@ class Cdist: return objects - def transfer_global_explorers(self): self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) + def transfer_type_explorers(self, type): + """Transfer explorers of a type, but only once""" + if self.type_explorers_transferred[type] == 1 + return + else + self.type_explorers_transferred[type] = 1 + + src = os.path.join(TYPE_DIR, type) + dst = os.path.join(REMOTE_TYPE_DIR, type, "explorer") + self.transfer_dir(src, dst) + def link_type_to_emulator(self): """Link type names to cdist-type-emulator""" for type in list_types(): @@ -226,7 +245,11 @@ class Cdist: self.remote_run_or_fail(cmd, stdout=output_fd) output_fd.close() - # def run_type_explorer(self): + def run_type_explorer(self, object): + """Run type specific explorers for objects""" + # Based on bin/cdist-object-explorer-run + + # Transfering explorers for this type def init_deploy(self): log.info("Creating clean directory structure") From 97ac276433bd15722d3ada5891d47d00ddb90856 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 17:52:15 +0200 Subject: [PATCH 47/87] implement transfer of type explorers Signed-off-by: Nico Schottelius --- bin/cdist | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 4e62bd64..56327fb4 100755 --- a/bin/cdist +++ b/bin/cdist @@ -211,15 +211,20 @@ class Cdist: def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" - if self.type_explorers_transferred[type] == 1 + if type in self.type_explorers_transferred: return - else - self.type_explorers_transferred[type] = 1 src = os.path.join(TYPE_DIR, type) - dst = os.path.join(REMOTE_TYPE_DIR, type, "explorer") + base = os.path.join(REMOTE_TYPE_DIR, type) + dst = os.path.join(base, "explorer") + + # Ensure the path path exists + self.remote_run_or_fail(["mkdir", "-p", base]) self.transfer_dir(src, dst) + # Do not retransfer + self.type_explorers_transferred[type] = 1 + def link_type_to_emulator(self): """Link type names to cdist-type-emulator""" for type in list_types(): From f31f33418ff646f9107e72c33dc87c1b4d51644e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 18:03:19 +0200 Subject: [PATCH 48/87] begin to actually prepare the object/type explorers Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 56327fb4..0a7011b4 100755 --- a/bin/cdist +++ b/bin/cdist @@ -187,7 +187,7 @@ class Cdist: return object_paths - def get_type_from_object(cdist_object): + def get_type_from_object(self, cdist_object): return cdist_object.split(os.sep)[0] def list_objects(self, starting_point = False): @@ -250,11 +250,13 @@ class Cdist: self.remote_run_or_fail(cmd, stdout=output_fd) output_fd.close() - def run_type_explorer(self, object): + def run_type_explorer(self, cdist_object): """Run type specific explorers for objects""" # Based on bin/cdist-object-explorer-run # Transfering explorers for this type + type = self.get_type_from_object(cdist_object) + self.transfer_type_explorers(type) def init_deploy(self): log.info("Creating clean directory structure") @@ -294,6 +296,11 @@ class Cdist: self.init_deploy() self.run_global_explores() self.run_initial_manifest() + + objects = self.list_objects() + + for cdist_object in objects: + self.run_type_explorer(cdist_object) if __name__ == "__main__": From ab0cd9430fc270f0f99d21e16c5360173db9b914 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 22:40:12 +0200 Subject: [PATCH 49/87] add support for the banner :-) Signed-off-by: Nico Schottelius --- bin/cdist | 61 ++++++++++++++++++++++++--- conf/type/__directory/explorer/exists | 30 ------------- 2 files changed, 56 insertions(+), 35 deletions(-) delete mode 100755 conf/type/__directory/explorer/exists diff --git a/bin/cdist b/bin/cdist index 0a7011b4..ad87b38c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -28,6 +28,21 @@ import shutil import sys import tempfile +BANNER = """ + .. . .x+=:. s + dF @88> z` ^% :8 + '88bu. %8P . . -# -# -# Check whether file exists or not -# - -destination="/$__object_id" - -if [ -e "$destination" ]; then - echo yes -else - echo no -fi From 7e8362eeedd24f7df7c983d4e698dfdccb162f76 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 22:44:25 +0200 Subject: [PATCH 50/87] allow zero hosts to be specified, fixup exit_error Signed-off-by: Nico Schottelius --- bin/cdist | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index ad87b38c..8f556b1e 100755 --- a/bin/cdist +++ b/bin/cdist @@ -160,7 +160,7 @@ class Cdist: try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - self.exit_error("Command failed:", " ".join(*args)) + self.exit_error("Command failed: " + " ".join(*args)) def remote_run_or_fail(self, *args, **kargs): """Run something on the remote side and fail is something breaks""" @@ -349,20 +349,20 @@ class Cdist: if __name__ == "__main__": parser = argparse.ArgumentParser(description='cdist ' + VERSION) - parser.add_argument('host', nargs='+', help='one or more hosts to operate on') + parser.add_argument('host', nargs='*', help='one or more hosts to operate on') parser.add_argument('-b', '--banner', help='Show cdist banner', action='store_true', dest='banner') - parser.add_argument('-d', '--debug', help='set log level to debug', + parser.add_argument('-d', '--debug', help='Set log level to debug', action='store_true') parser.add_argument('-i', '--initial-manifest', - help='path to a cdist manifest or - to read from stdin', + help='Path to a cdist manifest or - to read from stdin', dest='manifest', required=False) parser.add_argument('-p', '--parallel', - help='operate on multiple hosts in parallel', + help='Operate on multiple hosts in parallel', action='store_true', dest='parallel') parser.add_argument('-s', '--sequential', - help='operate on multiple hosts sequentially', + help='Operate on multiple hosts sequentially', action='store_false', dest='parallel') args = parser.parse_args(sys.argv[1:]) From cbdb5cd05d0ea4a98d54cc83d6b03f4be136cbc7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 23:01:11 +0200 Subject: [PATCH 51/87] fixup type explorer run Signed-off-by: Nico Schottelius --- bin/cdist | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/bin/cdist b/bin/cdist index 8f556b1e..3f57de40 100755 --- a/bin/cdist +++ b/bin/cdist @@ -189,7 +189,13 @@ class Cdist: def list_type_explorers(self, type): """Return list of available explorers for a specific type""" - return os.listdir(type_explorer_dir(type)) + dir = self.type_explorer_dir(type) + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + return list def list_object_paths(self, starting_point = False): """Return list of paths of existing objects""" @@ -234,7 +240,7 @@ class Cdist: """Return directory that holds the explorers of a type""" return os.path.join(TYPE_DIR, type, "explorer") - def remote_type_explorer_dir(type): + def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" return os.path.join(REMOTE_TYPE_DIR, type, "explorer") @@ -246,17 +252,20 @@ class Cdist: if type in self.type_explorers_transferred: log.debug("Skipping retransfer for %s", type) return + else: + # Do not retransfer + self.type_explorers_transferred[type] = 1 src = self.type_explorer_dir(type) remote_base = os.path.join(REMOTE_TYPE_DIR, type) - dst = self.type_explorer_dir(type) + dst = self.remote_type_explorer_dir(type) - # Ensure the path path exists - self.remote_run_or_fail(["mkdir", "-p", remote_base]) - self.transfer_dir(src, dst) + # Only continue, if there is at least the directory + if os.path.isdir(src): + # Ensure that the path exists + self.remote_run_or_fail(["mkdir", "-p", remote_base]) + self.transfer_dir(src, dst) - # Do not retransfer - self.type_explorers_transferred[type] = 1 def link_type_to_emulator(self): """Link type names to cdist-type-emulator""" @@ -293,12 +302,12 @@ class Cdist: cmd = [] cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + remote_type_explorer_dir(type)) + cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - explorers = list_type_explorers(type) + explorers = self.list_type_explorers(type) for explorer in explorers: remote_cmd = cmd - remote_cmd.append(os.path.join(remote_type_explorer_dir(type), explorer)) + remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) self.remote_run_or_fail(remote_cmd) From 8889e4a5b0b0e14ecfc6752edc000bcb917fb618 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 08:06:16 +0200 Subject: [PATCH 52/87] prepare saving output of type explorers Signed-off-by: Nico Schottelius --- bin/cdist | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 3f57de40..b35b8c17 100755 --- a/bin/cdist +++ b/bin/cdist @@ -309,7 +309,12 @@ class Cdist: remote_cmd = cmd remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) - self.remote_run_or_fail(remote_cmd) + output = self.type_explorer_output_path(cdist_object) + output_fd = open(output, mode='w') + + self.remote_run_or_fail(remote_cmd, stdout=output_fd) + + output_fd.close() def init_deploy(self): log.info("Creating clean directory structure") From 894c39cd210ea0d312cd07aaf3f01d6e99ca0144 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 12:18:25 +0200 Subject: [PATCH 53/87] store results of type explorer Signed-off-by: Nico Schottelius --- bin/cdist | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index b35b8c17..e2e53c9d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -179,6 +179,14 @@ class Cdist: """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) + def type_explorer_output_dir(self, cdist_object): + """Returns and creates dir of the output for a type explorer""" + dir = os.path.join(self.object_full_path(cdist_object), DOT_CDIST, "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + + return dir + def remote_global_explorer_path(self, explorer): """Returns path to the remote explorer""" return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) @@ -219,8 +227,13 @@ class Cdist: return object_paths def get_type_from_object(self, cdist_object): + """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] + def object_full_path(self, cdist_object): + """Returns the full path to the object (""" + return os.path.join(self.object_dir, cdist_object) + def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -304,12 +317,15 @@ class Cdist: cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) + # FIXME: need to transfer object before as well! + cmd.append("__object=" + self.remote_type_explorer_dir(type)) + explorers = self.list_type_explorers(type) for explorer in explorers: remote_cmd = cmd remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) - output = self.type_explorer_output_path(cdist_object) + output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') self.remote_run_or_fail(remote_cmd, stdout=output_fd) From b2d1d1bee703e177a952e80660782cf06df5e782 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:21:16 +0200 Subject: [PATCH 54/87] begin to modify tuple (bad idea) Signed-off-by: Nico Schottelius --- bin/cdist | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/bin/cdist b/bin/cdist index e2e53c9d..0772c703 100755 --- a/bin/cdist +++ b/bin/cdist @@ -157,19 +157,29 @@ class Cdist: def run_or_fail(self, *args, **kargs): log.debug("Exec: " + " ".join(*args)) + if "remote" in kargs: + log.debug("Remote found") + if kargs["remote"]: + # Replace the list found in the tuple at position 0 + cmd = ["ssh", "root@" + self.target_host] + cmd.extend(args[0]) + args[0] = cmd + print(newargs) + + del kargs["remote"] + + log.debug(newargs) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - self.exit_error("Command failed: " + " ".join(*args)) + self.exit_error("Command failed: " + " ".join(*newargs)) - def remote_run_or_fail(self, *args, **kargs): - """Run something on the remote side and fail is something breaks""" - newargs = ["ssh", "root@" + self.target_host] - newargs.extend(*args) - self.run_or_fail(newargs, **kargs) +# def remote_run_or_fail(self, *args, **kargs): +# """Run something on the remote side and fail is something breaks""" +# self.run_or_fail(newargs, **kargs) def remove_remote_dir(self, destination): - self.remote_run_or_fail(["rm", "-rf", destination]) + self.run_or_fail(["rm", "-rf", destination], remote=True) def transfer_dir(self, source, destination): self.remove_remote_dir(destination) From c1ad93bccc6cf93b3a5129914624ec3350ca9b14 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:27:54 +0200 Subject: [PATCH 55/87] extend run_or_fail to include remote exec Signed-off-by: Nico Schottelius --- bin/cdist | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 0772c703..5227dc06 100755 --- a/bin/cdist +++ b/bin/cdist @@ -155,20 +155,14 @@ class Cdist: self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) def run_or_fail(self, *args, **kargs): - log.debug("Exec: " + " ".join(*args)) - if "remote" in kargs: log.debug("Remote found") if kargs["remote"]: - # Replace the list found in the tuple at position 0 - cmd = ["ssh", "root@" + self.target_host] - cmd.extend(args[0]) - args[0] = cmd - print(newargs) + args[0][:0] = ["ssh", "root@" + self.target_host] del kargs["remote"] - log.debug(newargs) + log.debug("Exec: " + " ".join(*args)) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: From 20aafe62f578bf51512507d53d9f385e1b94dba7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:30:53 +0200 Subject: [PATCH 56/87] make use of remote=True flag Signed-off-by: Nico Schottelius --- bin/cdist | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/bin/cdist b/bin/cdist index 5227dc06..68424cfd 100755 --- a/bin/cdist +++ b/bin/cdist @@ -168,10 +168,6 @@ class Cdist: except subprocess.CalledProcessError: self.exit_error("Command failed: " + " ".join(*newargs)) -# def remote_run_or_fail(self, *args, **kargs): -# """Run something on the remote side and fail is something breaks""" -# self.run_or_fail(newargs, **kargs) - def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) @@ -280,7 +276,7 @@ class Cdist: # Only continue, if there is at least the directory if os.path.isdir(src): # Ensure that the path exists - self.remote_run_or_fail(["mkdir", "-p", remote_base]) + self.run_or_fail(["mkdir", "-p", remote_base], remote=True) self.transfer_dir(src, dst) @@ -306,7 +302,7 @@ class Cdist: cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append(self.remote_global_explorer_path(explorer)) - self.remote_run_or_fail(cmd, stdout=output_fd) + self.run_or_fail(cmd, stdout=output_fd, remote=True) output_fd.close() def run_type_explorer(self, cdist_object): @@ -332,7 +328,7 @@ class Cdist: output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') - self.remote_run_or_fail(remote_cmd, stdout=output_fd) + self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) output_fd.close() From f21ce6a0db1f778fba8c3726155fa7562c3a4635 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:51:25 +0200 Subject: [PATCH 57/87] commit broken code Signed-off-by: Nico Schottelius --- bin/cdist | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 68424cfd..a0189fbe 100755 --- a/bin/cdist +++ b/bin/cdist @@ -98,6 +98,7 @@ class Cdist: def __init__(self, target_host, initial_manifest=False): self.target_host = target_host + self.remote_prefix = ["ssh", "root@" + self.target_host] # log.info("foobar") @@ -139,26 +140,47 @@ class Cdist: log.error(*args) sys.exit(1) + def remote_cat(filename): + cmd = self.remote_prefix + cmd.append("cat") + cmd.append(filename) + try: + subprocess.call(cmd) + except subprocess.CalledProcessError: + log.error("Remote cat failed") + def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called - args[0].insert(0,"/bin/sh") - args[0].insert(1,"-e") - log.debug("Shell exec: " + " ".join(*args)) + args[0][:0] = [ "/bin/sh", "-e" ] + if "remote" in kargs: + log.debug("Remote found") + if kargs["remote"]: + args[0][:0] = self.remote_prefix + remote = true + + del kargs["remote"] + + log.debug("Shell exec: " + " ".join(*args)) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - script_fd = open(script) - log.error("Code that raised the error:\n" + script_fd.read()) - script_fd.close() + log.error("Code that raised the error:\n") + if remote: + remote_cat(script) + else: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) def run_or_fail(self, *args, **kargs): if "remote" in kargs: log.debug("Remote found") if kargs["remote"]: - args[0][:0] = ["ssh", "root@" + self.target_host] + args[0][:0] = self.remote_prefix del kargs["remote"] @@ -166,7 +188,7 @@ class Cdist: try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - self.exit_error("Command failed: " + " ".join(*newargs)) + self.exit_error("Command failed: " + " ".join(*args)) def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) @@ -322,6 +344,7 @@ class Cdist: explorers = self.list_type_explorers(type) for explorer in explorers: + # THIS IS A BUG, because remote_cmd is NOT a copy remote_cmd = cmd remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) From 2c15069d96e0e35cffc7e4523f7ddf85d7d87212 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:56:51 +0200 Subject: [PATCH 58/87] make remote_cmd a new list Signed-off-by: Nico Schottelius --- bin/cdist | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index a0189fbe..70df214c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -344,9 +344,7 @@ class Cdist: explorers = self.list_type_explorers(type) for explorer in explorers: - # THIS IS A BUG, because remote_cmd is NOT a copy - remote_cmd = cmd - remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) + remote_cmd = cmd + os.path.join(self.remote_type_explorer_dir(type), explorer) output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') From 6d5a53b7ad9368c2ee3bc704a7068fad5b52ee9c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 15:00:09 +0200 Subject: [PATCH 59/87] fixup list bug by creating a new one Signed-off-by: Nico Schottelius --- bin/cdist | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 70df214c..80d9f525 100755 --- a/bin/cdist +++ b/bin/cdist @@ -155,7 +155,6 @@ class Cdist: args[0][:0] = [ "/bin/sh", "-e" ] if "remote" in kargs: - log.debug("Remote found") if kargs["remote"]: args[0][:0] = self.remote_prefix remote = true @@ -178,7 +177,6 @@ class Cdist: def run_or_fail(self, *args, **kargs): if "remote" in kargs: - log.debug("Remote found") if kargs["remote"]: args[0][:0] = self.remote_prefix @@ -344,13 +342,10 @@ class Cdist: explorers = self.list_type_explorers(type) for explorer in explorers: - remote_cmd = cmd + os.path.join(self.remote_type_explorer_dir(type), explorer) - + remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') - self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) - output_fd.close() def init_deploy(self): From 722339f3e1164b0aac1d2e54904a2f2a510d5548 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 15:16:47 +0200 Subject: [PATCH 60/87] support getting paths of objects and their parameters Signed-off-by: Nico Schottelius --- bin/cdist | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 80d9f525..302e21be 100755 --- a/bin/cdist +++ b/bin/cdist @@ -53,6 +53,7 @@ TYPE_DIR = os.path.join(CONF_DIR, "type") REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") @@ -100,8 +101,6 @@ class Cdist: self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] - # log.info("foobar") - # Setup directory paths self.temp_dir = tempfile.mkdtemp() @@ -192,6 +191,7 @@ class Cdist: self.run_or_fail(["rm", "-rf", destination], remote=True) def transfer_dir(self, source, destination): + """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) self.run_or_fail(["scp", "-qr", source, "root@" + self.target_host + ":" + destination]) @@ -201,7 +201,7 @@ class Cdist: def type_explorer_output_dir(self, cdist_object): """Returns and creates dir of the output for a type explorer""" - dir = os.path.join(self.object_full_path(cdist_object), DOT_CDIST, "explorer") + dir = os.path.join(self.object_dir(cdist_object), "explorer") if not os.path.isdir(dir): os.mkdir(dir) @@ -250,9 +250,21 @@ class Cdist: """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] - def object_full_path(self, cdist_object): - """Returns the full path to the object (""" - return os.path.join(self.object_dir, cdist_object) + def object_dir(self, cdist_object): + """Returns the full path to the object (including .cdist)""" + return os.path.join(self.object_dir, cdist_object, DOT_CDIST) + + def remote_object_dir(self, cdist_object): + """Returns the remote full path to the object (including .cdist)""" + return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + + def object_parameter_dir(self, cdist_object): + """Returns the dir to the object parameter""" + return os.path.join(object_dir(cdist_object), "parameter") + + def remote_object_parameter_dir(self, cdist_object): + """Returns the remote dir to the object parameter""" + return os.path.join(remote_object_dir(cdist_object), "parameter") def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -277,6 +289,9 @@ class Cdist: """Return remote directory that holds the explorers of a type""" return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + def transfer_object_parameter(self, cdist_object): + self.transfer_dir(object_dir(cdist_object), ) + def transfer_global_explorers(self): self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) From db2562303ade41004fe9849074410caa18840376 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 15:28:38 +0200 Subject: [PATCH 61/87] in theory transfer the object parameters Signed-off-by: Nico Schottelius --- bin/cdist | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bin/cdist b/bin/cdist index 302e21be..b2f10ba1 100755 --- a/bin/cdist +++ b/bin/cdist @@ -110,7 +110,7 @@ class Cdist: self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) - self.object_dir = os.path.join(self.out_dir, "object") + self.object_base_dir = os.path.join(self.out_dir, "object") # Setup binary directory + contents self.bin_dir = os.path.join(self.out_dir, "bin") @@ -230,7 +230,7 @@ class Cdist: object_paths = [] if not starting_point: - starting_point = self.object_dir + starting_point = self.object_base_dir for content in os.listdir(starting_point): full_path = os.path.join(starting_point, content) @@ -252,7 +252,7 @@ class Cdist: def object_dir(self, cdist_object): """Returns the full path to the object (including .cdist)""" - return os.path.join(self.object_dir, cdist_object, DOT_CDIST) + return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) def remote_object_dir(self, cdist_object): """Returns the remote full path to the object (including .cdist)""" @@ -260,17 +260,17 @@ class Cdist: def object_parameter_dir(self, cdist_object): """Returns the dir to the object parameter""" - return os.path.join(object_dir(cdist_object), "parameter") + return os.path.join(self.object_dir(cdist_object), "parameter") def remote_object_parameter_dir(self, cdist_object): """Returns the remote dir to the object parameter""" - return os.path.join(remote_object_dir(cdist_object), "parameter") + return os.path.join(self.remote_object_dir(cdist_object), "parameter") def list_objects(self, starting_point = False): """Return list of existing objects""" if not starting_point: - starting_point = self.object_dir + starting_point = self.object_base_dir object_paths = self.list_object_paths(starting_point) objects = [] @@ -290,9 +290,12 @@ class Cdist: return os.path.join(REMOTE_TYPE_DIR, type, "explorer") def transfer_object_parameter(self, cdist_object): - self.transfer_dir(object_dir(cdist_object), ) + """Transfer the object parameter to the remote destination""" + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) def transfer_global_explorers(self): + """Transfer the global explorers""" self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) def transfer_type_explorers(self, type): @@ -352,8 +355,9 @@ class Cdist: cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - # FIXME: need to transfer object before as well! - cmd.append("__object=" + self.remote_type_explorer_dir(type)) + # Need to transfer at least the parameters for objects to be useful + self.transfer_object_parameter(cdist_object) + cmd.append("__object=" + self.remote_object_dir(cdist_object)) explorers = self.list_type_explorers(type) for explorer in explorers: From 14ac0e3ad166cc90a947d88e2b120f3746cd67b5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 20:53:09 +0200 Subject: [PATCH 62/87] introduce remote_mkdir() Signed-off-by: Nico Schottelius --- bin/cdist | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index b2f10ba1..30af2827 100755 --- a/bin/cdist +++ b/bin/cdist @@ -139,6 +139,10 @@ class Cdist: log.error(*args) sys.exit(1) + def remote_mkdir(directory): + """Create directory on remote side""" + self.run_or_fail(["mkdir", "-p", directory], remote=True) + def remote_cat(filename): cmd = self.remote_prefix cmd.append("cat") @@ -234,14 +238,11 @@ class Cdist: for content in os.listdir(starting_point): full_path = os.path.join(starting_point, content) - print(full_path) if os.path.isdir(full_path): - log.debug("Recursing for %s", full_path) object_paths.extend(self.list_object_paths(starting_point = full_path)) # Directory contains .cdist -> is an object if content == DOT_CDIST: - log.debug("Adding Object Path %s", starting_point) object_paths.append(starting_point) return object_paths @@ -314,7 +315,7 @@ class Cdist: # Only continue, if there is at least the directory if os.path.isdir(src): # Ensure that the path exists - self.run_or_fail(["mkdir", "-p", remote_base], remote=True) + self.remote_mkdir(remote_base) self.transfer_dir(src, dst) From 99268591c2e3eddbc49bb95f5b2fa31b7f8bd42f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 20:54:17 +0200 Subject: [PATCH 63/87] make more use of run_or_fail Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 30af2827..72b7a752 100755 --- a/bin/cdist +++ b/bin/cdist @@ -139,18 +139,13 @@ class Cdist: log.error(*args) sys.exit(1) - def remote_mkdir(directory): + def remote_mkdir(self, directory): """Create directory on remote side""" self.run_or_fail(["mkdir", "-p", directory], remote=True) def remote_cat(filename): - cmd = self.remote_prefix - cmd.append("cat") - cmd.append(filename) - try: - subprocess.call(cmd) - except subprocess.CalledProcessError: - log.error("Remote cat failed") + """Use cat on the remote side for output""" + self.run_or_fail(["cat", filename], remote=True) def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want From 232618a675fc90accea25646171a9db30b54a2b9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:01:16 +0200 Subject: [PATCH 64/87] make manifest run more generic Signed-off-by: Nico Schottelius --- bin/cdist | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 72b7a752..4f9e95de 100755 --- a/bin/cdist +++ b/bin/cdist @@ -287,6 +287,10 @@ class Cdist: def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards self.transfer_dir(self.object_parameter_dir(cdist_object), self.remote_object_parameter_dir(cdist_object)) @@ -375,8 +379,13 @@ class Cdist: # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) + # def run_object_manifest(self, cdist_object): def run_initial_manifest(self): """Run the initial manifest""" + self.run_manifest(self.initial_manifest) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" log.info("Running the initial manifest") env = os.environ.copy() env['PATH'] = self.bin_dir + ":" + env['PATH'] @@ -390,9 +399,11 @@ class Cdist: env['__cdist_local_base_dir'] = self.temp_dir env['__cdist_manifest'] = self.initial_manifest - self.shell_run_or_debug_fail(self.initial_manifest, - [self.initial_manifest], - env=env) + # Other environment stuff + if extra_env: + env.update(extra_env) + + self.shell_run_or_debug_fail(manifest, [manifest], env=env) def deploy_to(self): From 8dff399c9654b8983e2997f483253e162c839eb3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:09:57 +0200 Subject: [PATCH 65/87] reloop until no new objects are created anyomer Signed-off-by: Nico Schottelius --- bin/cdist | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 4f9e95de..505bb012 100755 --- a/bin/cdist +++ b/bin/cdist @@ -379,11 +379,14 @@ class Cdist: # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) - # def run_object_manifest(self, cdist_object): def run_initial_manifest(self): """Run the initial manifest""" self.run_manifest(self.initial_manifest) + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + self.run_manifest(self.initial_manifest) + def run_manifest(self, manifest, extra_env=None): """Run a manifest""" log.info("Running the initial manifest") @@ -413,10 +416,18 @@ class Cdist: self.run_global_explores() self.run_initial_manifest() + old_objects = [] objects = self.list_objects() - for cdist_object in objects: - self.run_type_explorer(cdist_object) + # Continue process until no new objects are created anymore + while not old_objects == objects: + old_objects = objects.copy() + for cdist_object in objects: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + + objects = self.list_objects() + if __name__ == "__main__": From 13e3e27679443d558ddaa9846603c4da390cfb5e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:29:51 +0200 Subject: [PATCH 66/87] changes for next Signed-off-by: Nico Schottelius --- doc/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog b/doc/changelog index 7e98277a..6d25f234 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,5 +1,6 @@ next: * New Type: __package_rubygem (Chase Allen James) + * __self replaced by __object_fq (or so) 1.7.1: 2011-07-26 * Documentation: Add explorers to reference From 7eb418c3ab6db117db2de036226d05da3033b1cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:40:45 +0200 Subject: [PATCH 67/87] correct env for the various stages Signed-off-by: Nico Schottelius --- bin/cdist | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 505bb012..af19a676 100755 --- a/bin/cdist +++ b/bin/cdist @@ -246,6 +246,10 @@ class Cdist: """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] + def get_object_id_from_object(self, cdist_object): + """Returns everything but the first part (i.e. object_id) of an object""" + return os.sep.join(cdist_object.split(os.sep)[1:]) + def object_dir(self, cdist_object): """Returns the full path to the object (including .cdist)""" return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) @@ -281,6 +285,10 @@ class Cdist: """Return directory that holds the explorers of a type""" return os.path.join(TYPE_DIR, type, "explorer") + def type_manifest_path(self, type): + """Return path to manifest of type""" + return os.path.join(TYPE_DIR, type, "manifest") + def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" return os.path.join(REMOTE_TYPE_DIR, type, "explorer") @@ -352,6 +360,7 @@ class Cdist: self.transfer_type_explorers(type) cmd = [] + # FIXME: __global::__object_id::__self:: cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) @@ -381,15 +390,27 @@ class Cdist: def run_initial_manifest(self): """Run the initial manifest""" - self.run_manifest(self.initial_manifest) + # FIXME: add support for __manifest:: + env = { "__manifest" : MANIFEST_DIR } + self.run_manifest(self.initial_manifest, extra_env=env) def run_type_manifest(self, cdist_object): """Run manifest for a specific object""" - self.run_manifest(self.initial_manifest) + type = self.get_type_from_object(cdist_object) + manifest = self.type_manifest_path(type) + + # FIXME: add more sensible checks for manifest + if os.path.exists(manifest): + env = { "__object" : self.object_dir(cdist_object), + "__object_id": self.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": type + } + self.run_manifest(self.initial_manifest, extra_env=env) def run_manifest(self, manifest, extra_env=None): """Run a manifest""" - log.info("Running the initial manifest") + log.info("Running manifest %s", manifest) env = os.environ.copy() env['PATH'] = self.bin_dir + ":" + env['PATH'] @@ -420,8 +441,10 @@ class Cdist: objects = self.list_objects() # Continue process until no new objects are created anymore - while not old_objects == objects: - old_objects = objects.copy() + while old_objects != objects: + log.debug("Prepare stage") + old_objects = list(objects) + # FIXME: do not rerun existing objects! for cdist_object in objects: self.run_type_explorer(cdist_object) self.run_type_manifest(cdist_object) From 4e2e731374a55827e4670885b6c4bf6948c3a9d5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:43:28 +0200 Subject: [PATCH 68/87] execute the correct manifest for objects Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index af19a676..ff114325 100755 --- a/bin/cdist +++ b/bin/cdist @@ -406,11 +406,11 @@ class Cdist: "__object_fq": cdist_object, "__type": type } - self.run_manifest(self.initial_manifest, extra_env=env) + self.run_manifest(manifest, extra_env=env) def run_manifest(self, manifest, extra_env=None): """Run a manifest""" - log.info("Running manifest %s", manifest) + log.info("Running manifest %s, env=%s", manifest, extra_env) env = os.environ.copy() env['PATH'] = self.bin_dir + ":" + env['PATH'] From bc6201102eb0b1d09f4328959b38f3608b7048ba Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:44:15 +0200 Subject: [PATCH 69/87] todo-- Signed-off-by: Nico Schottelius --- bin/cdist-deploy-to | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to index 96650f90..86e15800 100755 --- a/bin/cdist-deploy-to +++ b/bin/cdist-deploy-to @@ -43,7 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " # See cdist-stages(7) # -cdist-object-all "$__cdist_target_host" cdist-object-prepare cdist-object-all "$__cdist_target_host" cdist-object-run cdist-cache "$__cdist_target_host" From f5c2ae3049c48bb056fdc9570648f36a892045d1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:48:24 +0200 Subject: [PATCH 70/87] begin cleanup Signed-off-by: Nico Schottelius --- bin/cdist-cache | 39 -------------------------- bin/cdist-dir | 51 ---------------------------------- bin/cdist-run-remote | 33 ---------------------- bin/cdist-type-build-emulation | 43 ---------------------------- 4 files changed, 166 deletions(-) delete mode 100755 bin/cdist-cache delete mode 100755 bin/cdist-dir delete mode 100755 bin/cdist-run-remote delete mode 100755 bin/cdist-type-build-emulation diff --git a/bin/cdist-cache b/bin/cdist-cache deleted file mode 100755 index ee27ffb4..00000000 --- a/bin/cdist-cache +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Let's build a cconfig tree from a configuration -# And save it into the cache tree -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -u - -__cdist_target_host="$1"; shift - -# Create base to move into -mkdir -p "${__cdist_local_base_cache_dir}" - -# Now determine absolute path -__cdist_ddir="$(__cdist_host_cache_dir "$__cdist_target_host")" - -__cdist_echo info "Saving cache to $__cdist_ddir " -rm -rf "$__cdist_ddir" -mv "$__cdist_local_base_dir" "$__cdist_ddir" diff --git a/bin/cdist-dir b/bin/cdist-dir deleted file mode 100755 index 0d30e14a..00000000 --- a/bin/cdist-dir +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Push a directory to a target, both sides have the same name (i.e. explorers) -# or -# Pull a directory from a target, both sides have the same name (i.e. explorers) -# - - -. cdist-config -[ $# -eq 4 ] || __cdist_usage " " -set -ue - -__cdist_action="$1"; shift -__cdist_target_host="$1"; shift -__cdist_src_dir="$1"; shift -__cdist_dst_dir="$1"; shift - -# This will be the destination directory, so no subdirectories -# of the same name are created, if the directory is already existing -__cdist_top_dir="${__cdist_dst_dir%/*}" - -if [ "$__cdist_action" = "push" ]; then - ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "mkdir -p \"${__cdist_dst_dir}\"" - scp -qr "$__cdist_src_dir" \ - "${__cdist_remote_user}@${__cdist_target_host}:${__cdist_top_dir}" -elif [ "$__cdist_action" = "pull" ]; then - mkdir -p "${__cdist_dst_dir}" - scp -qr "${__cdist_remote_user}@${__cdist_target_host}:${__cdist_src_dir}" \ - "${__cdist_top_dir}" -else - __cdist_exit_err "Unknown action $__cdist_action" -fi diff --git a/bin/cdist-run-remote b/bin/cdist-run-remote deleted file mode 100755 index 4a4452a2..00000000 --- a/bin/cdist-run-remote +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run a cdist binary on the remote side -# - -. cdist-config -[ $# -ge 2 ] || __cdist_usage " [opts]" -set -ue - -__cdist_target_host="$1"; shift - -ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "export PATH=\"${__cdist_remote_bin_dir}:\$PATH\";" \ - "export __cdist_out_object_dir=\"$__cdist_remote_out_object_dir\";" \ - "$@" diff --git a/bin/cdist-type-build-emulation b/bin/cdist-type-build-emulation deleted file mode 100755 index 51c2f5b4..00000000 --- a/bin/cdist-type-build-emulation +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Build pseudo binaries for type emulation -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -eu - -__cdist_output_dir="$1"; shift - -__cdist_type_emulator="$__cdist_abs_mydir/cdist-type-emulator" - -if [ ! -d "${__cdist_type_dir}" ]; then - __cdist_exit_err "$__cdist_type_dir must exist and contain available types" -fi - -# Get Types -cd "${__cdist_type_dir}" -ls -1 > "${__cdist_tmp_file}" - -# Create binaries -mkdir -p "${__cdist_output_dir}" -while read type; do - ln -sf "${__cdist_type_emulator}" "${__cdist_output_dir}/${type}" -done < "${__cdist_tmp_file}" From 74c280911bb1e3bca234d72236df9a584feb29c9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:53:10 +0200 Subject: [PATCH 71/87] begin run_object_gencode() and import missing FIXME bits deploy_to() Signed-off-by: Nico Schottelius --- bin/cdist | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bin/cdist b/bin/cdist index ff114325..46422ce5 100755 --- a/bin/cdist +++ b/bin/cdist @@ -429,6 +429,23 @@ class Cdist: self.shell_run_or_debug_fail(manifest, [manifest], env=env) + def run_object_gencode(self, cdist_object): + """Run the gencode scripts for the object""" + log.info("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + + # FIXME: if -local, -remote... + + + # Other environment stuff + if extra_env: + env.update(extra_env) + + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" @@ -451,6 +468,17 @@ class Cdist: objects = self.list_objects() + # Now do the final steps over the existing objects + for cdist_object in objects: + # FIXME: Check requirements and execute those before + + # FIXME: to be implemented + # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" + # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + if __name__ == "__main__": From 4a5100692ab2374fafcc7744ca3b21b855b2da5d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 10:12:01 +0200 Subject: [PATCH 72/87] add helper methods to get paths from code/gencode Signed-off-by: Nico Schottelius --- bin/cdist | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 46422ce5..9eaf39fe 100755 --- a/bin/cdist +++ b/bin/cdist @@ -266,6 +266,11 @@ class Cdist: """Returns the remote dir to the object parameter""" return os.path.join(self.remote_object_dir(cdist_object), "parameter") + def object_code_paths(self, cdist_object): + """Return paths to code scripts of object""" + return [os.path.join(object_dir(cdist_object), "code-local"), + os.path.join(object_dir(cdist_object), "code-remote")] + def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -285,6 +290,11 @@ class Cdist: """Return directory that holds the explorers of a type""" return os.path.join(TYPE_DIR, type, "explorer") + def type_gencode_paths(self, type): + """Return paths to gencode scripts of type""" + return [os.path.join(TYPE_DIR, type, "gencode-local"), + os.path.join(TYPE_DIR, type, "gencode-remote")] + def type_manifest_path(self, type): """Return path to manifest of type""" return os.path.join(TYPE_DIR, type, "manifest") @@ -390,7 +400,6 @@ class Cdist: def run_initial_manifest(self): """Run the initial manifest""" - # FIXME: add support for __manifest:: env = { "__manifest" : MANIFEST_DIR } self.run_manifest(self.initial_manifest, extra_env=env) @@ -446,6 +455,26 @@ class Cdist: self.shell_run_or_debug_fail(manifest, [manifest], env=env) + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + # FIXME: Check requirements and execute those before + requirements = list_object_requirements(cdist_object) + + for requirement in requirements: + object_run(requirement, mode=mode) + + # Find and run all available gencode scripts + if mode == "gencode": + base_path = object_dir + if mode == "code": + + # FIXME: to be implemented + # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" + # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" @@ -470,14 +499,9 @@ class Cdist: # Now do the final steps over the existing objects for cdist_object in objects: - # FIXME: Check requirements and execute those before + object_run(cdist_object, mode="gencode") + object_run(cdist_object, mode="code") - # FIXME: to be implemented - # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) From c516d8359eb5780a5b9b0f4d4c51208ef684f03c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 13:19:32 +0200 Subject: [PATCH 73/87] support __object_* in run_type_explorer() Signed-off-by: Nico Schottelius --- bin/cdist | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 9eaf39fe..b3397fb8 100755 --- a/bin/cdist +++ b/bin/cdist @@ -370,9 +370,11 @@ class Cdist: self.transfer_type_explorers(type) cmd = [] - # FIXME: __global::__object_id::__self:: - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) + cmd.append("__global =" + self.out_dir) + cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) # Need to transfer at least the parameters for objects to be useful self.transfer_object_parameter(cdist_object) @@ -465,15 +467,20 @@ class Cdist: # Find and run all available gencode scripts if mode == "gencode": - base_path = object_dir + paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) if mode == "code": + paths = self.object_code_paths(cdist_object) # FIXME: to be implemented # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) + for bin in paths: + if os.path.isfile(bin): + if mode == "gencode": + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + if mode == "code": + self.run_or_fail(manifest, [manifest], env=env) def deploy_to(self): From beafb55e12b7df60f021183b38d0d4f001ec082a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 14:44:21 +0200 Subject: [PATCH 74/87] __global not available for type explorer (should it be?) Signed-off-by: Nico Schottelius --- doc/man/cdist-reference.text.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index 90efc0f8..9ef7e9c2 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -164,7 +164,7 @@ __manifest:: Available for: initial manifest __global:: Directory that contains generic output like explorer. - Available for: initial manifest, type manifest, type explorer, type codegen + Available for: initial manifest, type manifest, type codegen __object:: Directory that contains the current object. Available for: type manifest, type explorer, type codegen From 636995f44ef76deaec2ef531f6d9ef86cb0f0119 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 14:45:59 +0200 Subject: [PATCH 75/87] add some meat to object_run() Signed-off-by: Nico Schottelius --- bin/cdist | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/bin/cdist b/bin/cdist index b3397fb8..a13abcf4 100755 --- a/bin/cdist +++ b/bin/cdist @@ -372,13 +372,12 @@ class Cdist: cmd = [] cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - cmd.append("__global =" + self.out_dir) + cmd.append("__object=" + self.remote_object_dir(cdist_object)) cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) cmd.append("__object_fq=" + cdist_object) # Need to transfer at least the parameters for objects to be useful self.transfer_object_parameter(cdist_object) - cmd.append("__object=" + self.remote_object_dir(cdist_object)) explorers = self.list_type_explorers(type) for explorer in explorers: @@ -459,28 +458,29 @@ class Cdist: def object_run(self, cdist_object, mode): """Run gencode or code for an object""" - # FIXME: Check requirements and execute those before - requirements = list_object_requirements(cdist_object) + # FIXME: Check requirements and execute those before + requirements = self.list_object_requirements(cdist_object) - for requirement in requirements: - object_run(requirement, mode=mode) + for requirement in requirements: + self.object_run(requirement, mode=mode) - # Find and run all available gencode scripts - if mode == "gencode": - paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) - if mode == "code": - paths = self.object_code_paths(cdist_object) + # Find and run all available gencode scripts + if mode == "gencode": + paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) + if mode == "code": + paths = self.object_code_paths(cdist_object) - # FIXME: to be implemented - # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + # FIXME: to be implemented + # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" + # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - for bin in paths: - if os.path.isfile(bin): - if mode == "gencode": - self.shell_run_or_debug_fail(manifest, [manifest], env=env) - if mode == "code": - self.run_or_fail(manifest, [manifest], env=env) + for bin in paths: + FIXME + if os.path.isfile(bin): + if mode == "gencode": + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + if mode == "code": + self.run_or_fail(manifest, [manifest], env=env) def deploy_to(self): @@ -506,11 +506,8 @@ class Cdist: # Now do the final steps over the existing objects for cdist_object in objects: - object_run(cdist_object, mode="gencode") - object_run(cdist_object, mode="code") - - - + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") if __name__ == "__main__": parser = argparse.ArgumentParser(description='cdist ' + VERSION) From 5a765bfc761d9fda4f026376f470f8309af5786e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 15:09:09 +0200 Subject: [PATCH 76/87] codegen vs. gencode Signed-off-by: Nico Schottelius --- doc/man/cdist-reference.text.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index 9ef7e9c2..a823891b 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -164,19 +164,19 @@ __manifest:: Available for: initial manifest __global:: Directory that contains generic output like explorer. - Available for: initial manifest, type manifest, type codegen + Available for: initial manifest, type manifest, type gencode __object:: Directory that contains the current object. - Available for: type manifest, type explorer, type codegen + Available for: type manifest, type explorer, type gencode __object_id:: The type unique object id. - Available for: type manifest, type explorer, type codegen + Available for: type manifest, type explorer, type gencode __self:: The full qualified name of the current object. - Available for: type manifest, type explorer, type codegen + Available for: type manifest, type explorer, type gencode __target_host:: The host we are deploying to. - Available for: initial manifest, type manifest, type codegen + Available for: initial manifest, type manifest, type gencode __target_user:: User to use for authentication on remote host. Currently static in core. From ca58841696d969db8c62c6eafd0840a895cca520 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 15:36:21 +0200 Subject: [PATCH 77/87] Broken code - it's too hot DEBUG: Shell exec: /bin/sh -e /home/users/nico/p/cdist/conf/type/__directory/gencode-remote Traceback (most recent call last): File "/home/users/nico/p/cdist/bin/cdist", line 557, in c.deploy_to() File "/home/users/nico/p/cdist/bin/cdist", line 523, in deploy_to self.object_run(cdist_object, mode="gencode") File "/home/users/nico/p/cdist/bin/cdist", line 491, in object_run self.shell_run_or_debug_fail(bin, [bin], env=env) File "/home/users/nico/p/cdist/bin/cdist", line 164, in shell_run_or_debug_fail subprocess.check_call(*args, **kargs) File "/usr/lib/python3.2/subprocess.py", line 480, in check_call retcode = call(*popenargs, **kwargs) File "/usr/lib/python3.2/subprocess.py", line 467, in call return Popen(*popenargs, **kwargs).wait() File "/usr/lib/python3.2/subprocess.py", line 741, in __init__ restore_signals, start_new_session) File "/usr/lib/python3.2/subprocess.py", line 1174, in _execute_child for k, v in env.items()] File "/usr/lib/python3.2/subprocess.py", line 1174, in for k, v in env.items()] File "/usr/lib/python3.2/os.py", line 581, in fsencode raise TypeError("expect bytes or str, not %s" % type(filename).__name__) TypeError: expect bytes or str, not tuple Signed-off-by: Nico Schottelius --- bin/cdist | 58 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/bin/cdist b/bin/cdist index a13abcf4..b9c071b5 100755 --- a/bin/cdist +++ b/bin/cdist @@ -268,8 +268,8 @@ class Cdist: def object_code_paths(self, cdist_object): """Return paths to code scripts of object""" - return [os.path.join(object_dir(cdist_object), "code-local"), - os.path.join(object_dir(cdist_object), "code-remote")] + return [os.path.join(self.object_dir(cdist_object), "code-local"), + os.path.join(self.object_dir(cdist_object), "code-remote")] def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -279,7 +279,6 @@ class Cdist: object_paths = self.list_object_paths(starting_point) objects = [] - log.debug("Paths recieved: %s", object_paths) for path in object_paths: objects.append(os.path.relpath(path, starting_point)) @@ -439,22 +438,23 @@ class Cdist: self.shell_run_or_debug_fail(manifest, [manifest], env=env) - def run_object_gencode(self, cdist_object): - """Run the gencode scripts for the object""" - log.info("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() + def list_object_requirements(self, cdist_object): + """Return list of requirements for specific object""" + file=os.path.join(self.object_dir(cdist_object), "require") - env['__target_host'] = self.target_host - env['__global'] = self.out_dir + if os.path.isfile(file): + file_fd = open(file, "r") + requirements = file_fd.readlines() + file_fd.close() - # FIXME: if -local, -remote... - + # Remove \n from all lines + requirements = map(lambda s: s.strip(), requirements) - # Other environment stuff - if extra_env: - env.update(extra_env) + log.debug(requirements) + else: + requirements = [] - self.shell_run_or_debug_fail(manifest, [manifest], env=env) + return requirements def object_run(self, cdist_object, mode): """Run gencode or code for an object""" @@ -470,18 +470,32 @@ class Cdist: if mode == "code": paths = self.object_code_paths(cdist_object) - # FIXME: to be implemented - # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + env["__object"] = self.object_dir(cdist_object), + env["__object_id"] = self.get_object_id_from_object(cdist_object), + env["__object_fq"] = cdist_object for bin in paths: - FIXME + log.debug("object/bin: %s", bin) if os.path.isfile(bin): + # FIXME: to be implemented + # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" if mode == "gencode": - self.shell_run_or_debug_fail(manifest, [manifest], env=env) + # __global::__object::__object_id::__self::__target_host:: + # + self.shell_run_or_debug_fail(bin, [bin], env=env) if mode == "code": - self.run_or_fail(manifest, [manifest], env=env) - + # FIXME: to be implemented + # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + log.debug("ERROR NOT IMPLEMENTED") + sys.exit(3) + # FIXME: transfer code and add remote path! + self.run_or_fail([bin], remote=True) def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" From 151022ffd1ef322b2e7fbeb4057630ab42d08f17 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 23:46:17 +0200 Subject: [PATCH 78/87] support file transfer and begin support for non root logins Signed-off-by: Nico Schottelius --- bin/cdist | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index b9c071b5..f34812d9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -97,7 +97,7 @@ def banner(): class Cdist: """Cdist main class to hold arbitrary data""" - def __init__(self, target_host, initial_manifest=False): + def __init__(self, target_host, initial_manifest=False, remote_user="root"): self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] @@ -120,6 +120,8 @@ class Cdist: # List of type explorers transferred self.type_explorers_transferred = {} + self.remote_user = remote_user + # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest @@ -192,7 +194,17 @@ class Cdist: def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, "root@" + self.target_host + ":" + destination]) + self.run_or_fail(["scp", "-qr", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def transfer_file(self, source, destination): + """Transfer file""" + self.run_or_fail(["scp", "-q", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" @@ -311,6 +323,16 @@ class Cdist: self.transfer_dir(self.object_parameter_dir(cdist_object), self.remote_object_parameter_dir(cdist_object)) + def transfer_object_code(self, cdist_object): + FIXME + """Transfer the object code to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) + def transfer_global_explorers(self): """Transfer the global explorers""" self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) From c147c74e1ed465e15e916b0af73cdb30d69a17c2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 13:11:45 +0200 Subject: [PATCH 79/87] begin to redo object_run() Signed-off-by: Nico Schottelius --- bin/cdist | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index f34812d9..eb1e9112 100755 --- a/bin/cdist +++ b/bin/cdist @@ -330,7 +330,7 @@ class Cdist: self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) # Synchronise parameter dir afterwards - self.transfer_dir(self.object_parameter_dir(cdist_object), + self.transfer_file(self.object_code_path(cdist_object), self.remote_object_parameter_dir(cdist_object)) def transfer_global_explorers(self): @@ -512,6 +512,9 @@ class Cdist: # self.shell_run_or_debug_fail(bin, [bin], env=env) if mode == "code": + local_dir = object_dir(cdist_object) + remote_dir = remote_object_dir(cdist_object) + # FIXME: to be implemented # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" log.debug("ERROR NOT IMPLEMENTED") From c9c808a732e66210d2780fbfc533eec263bf660e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 13:55:40 +0200 Subject: [PATCH 80/87] cleanp Signed-off-by: Nico Schottelius --- bin/cdist | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/bin/cdist b/bin/cdist index eb1e9112..7c2de358 100755 --- a/bin/cdist +++ b/bin/cdist @@ -152,6 +152,7 @@ class Cdist: def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called + print(args) args[0][:0] = [ "/bin/sh", "-e" ] if "remote" in kargs: @@ -161,6 +162,8 @@ class Cdist: del kargs["remote"] + print(args) + print(*args) log.debug("Shell exec: " + " ".join(*args)) try: subprocess.check_call(*args, **kargs) @@ -472,7 +475,7 @@ class Cdist: # Remove \n from all lines requirements = map(lambda s: s.strip(), requirements) - log.debug(requirements) + log.debug("Requirements for %s: %s", cdist_object, requirements) else: requirements = [] @@ -480,18 +483,13 @@ class Cdist: def object_run(self, cdist_object, mode): """Run gencode or code for an object""" - # FIXME: Check requirements and execute those before + log.debug("Running %s from %s", mode, cdist_object) requirements = self.list_object_requirements(cdist_object) for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) self.object_run(requirement, mode=mode) - # Find and run all available gencode scripts - if mode == "gencode": - paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) - if mode == "code": - paths = self.object_code_paths(cdist_object) - # # Setup env Variable: # @@ -502,26 +500,26 @@ class Cdist: env["__object_id"] = self.get_object_id_from_object(cdist_object), env["__object_fq"] = cdist_object - for bin in paths: - log.debug("object/bin: %s", bin) - if os.path.isfile(bin): - # FIXME: to be implemented - # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - if mode == "gencode": - # __global::__object::__object_id::__self::__target_host:: - # + if mode == "gencode": + paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) + for bin in paths: + if os.path.isfile(bin): + print(bin) self.shell_run_or_debug_fail(bin, [bin], env=env) - if mode == "code": - local_dir = object_dir(cdist_object) - remote_dir = remote_object_dir(cdist_object) - # FIXME: to be implemented - # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - log.debug("ERROR NOT IMPLEMENTED") - sys.exit(3) - # FIXME: transfer code and add remote path! - self.run_or_fail([bin], remote=True) + if mode == "code": +# paths = self.object_code_paths(cdist_object) + local_dir = self.object_dir(cdist_object) + remote_dir = self.remote_object_dir(cdist_object) + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + self.run_or_fail([bin], remote=False) + + if os.path.isfile(os.path.join(local_dir, "code-remote")): + remote_code = os.path.join(remote_dir, "code-remote") + self.run_or_fail([remote_code], remote=True) + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to host " + self.target_host) @@ -542,9 +540,11 @@ class Cdist: self.run_type_manifest(cdist_object) objects = self.list_objects() - + + log.debug("Actual run objects") # Now do the final steps over the existing objects for cdist_object in objects: + log.debug("Run object: %s", cdist_object) self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") From 5702706adf5badf120fc60d742164034ce934a76 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 14:11:54 +0200 Subject: [PATCH 81/87] fix not so obvious tuple/str error (trailing comma from c&p) Signed-off-by: Nico Schottelius --- bin/cdist | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7c2de358..7e6f6ddf 100755 --- a/bin/cdist +++ b/bin/cdist @@ -152,7 +152,6 @@ class Cdist: def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called - print(args) args[0][:0] = [ "/bin/sh", "-e" ] if "remote" in kargs: @@ -162,9 +161,8 @@ class Cdist: del kargs["remote"] - print(args) - print(*args) - log.debug("Shell exec: " + " ".join(*args)) + log.debug("Shell exec cmd: %s", args) + log.debug("Shell exec env: %s", kargs['env']) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: @@ -496,8 +494,8 @@ class Cdist: env = os.environ.copy() env['__target_host'] = self.target_host env['__global'] = self.out_dir - env["__object"] = self.object_dir(cdist_object), - env["__object_id"] = self.get_object_id_from_object(cdist_object), + env["__object"] = self.object_dir(cdist_object) + env["__object_id"] = self.get_object_id_from_object(cdist_object) env["__object_fq"] = cdist_object if mode == "gencode": From c78ce344e73578eeba144b7a6eff6419df2e4e0e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 14:30:05 +0200 Subject: [PATCH 82/87] remove debug by default, switch to infolevel and verify -d works Signed-off-by: Nico Schottelius --- bin/cdist | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7e6f6ddf..157b8d13 100755 --- a/bin/cdist +++ b/bin/cdist @@ -81,7 +81,7 @@ VERSION = "2.0.0" # -logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s') +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() # List types @@ -406,6 +406,9 @@ class Cdist: remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) output_fd.close() @@ -431,18 +434,19 @@ class Cdist: type = self.get_type_from_object(cdist_object) manifest = self.type_manifest_path(type) + log.debug("%s: Running %s", cdist_object, manifest) # FIXME: add more sensible checks for manifest if os.path.exists(manifest): - env = { "__object" : self.object_dir(cdist_object), + env = { "__object" : self.object_dir(cdist_object), "__object_id": self.get_object_id_from_object(cdist_object), "__object_fq": cdist_object, - "__type": type + "__type": type } self.run_manifest(manifest, extra_env=env) def run_manifest(self, manifest, extra_env=None): """Run a manifest""" - log.info("Running manifest %s, env=%s", manifest, extra_env) + log.debug("Running manifest %s, env=%s", manifest, extra_env) env = os.environ.copy() env['PATH'] = self.bin_dir + ":" + env['PATH'] @@ -502,7 +506,6 @@ class Cdist: paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) for bin in paths: if os.path.isfile(bin): - print(bin) self.shell_run_or_debug_fail(bin, [bin], env=env) if mode == "code": @@ -546,6 +549,8 @@ class Cdist: self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") + log.info("Finished run of %s", self.target_host) + if __name__ == "__main__": parser = argparse.ArgumentParser(description='cdist ' + VERSION) parser.add_argument('host', nargs='*', help='one or more hosts to operate on') From 6bd823995fff95414060ef4eca841f74733ecff4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 17:20:07 +0200 Subject: [PATCH 83/87] restore mistakenly deleted file Signed-off-by: Nico Schottelius --- conf/type/__directory/explorer/exists | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 conf/type/__directory/explorer/exists diff --git a/conf/type/__directory/explorer/exists b/conf/type/__directory/explorer/exists new file mode 100755 index 00000000..f8b85671 --- /dev/null +++ b/conf/type/__directory/explorer/exists @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Check whether file exists or not +# + +destination="/$__object_id" + +if [ -e "$destination" ]; then + echo yes +else + echo no +fi From 30a52cab0fe45d932d822ea7a28b6dfbfe5ef98a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:17:20 +0200 Subject: [PATCH 84/87] begin to save code and make it executable Signed-off-by: Nico Schottelius --- bin/cdist | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 157b8d13..e76ecfc7 100755 --- a/bin/cdist +++ b/bin/cdist @@ -25,6 +25,7 @@ import logging import os import subprocess import shutil +import stat import sys import tempfile @@ -93,7 +94,6 @@ def banner(): print(BANNER) - class Cdist: """Cdist main class to hold arbitrary data""" @@ -235,6 +235,8 @@ class Cdist: else: list = [] + log.debug("Explorers for %s in %s: %s", type, dir, list) + return list def list_object_paths(self, starting_point = False): @@ -341,7 +343,7 @@ class Cdist: def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" if type in self.type_explorers_transferred: - log.debug("Skipping retransfer for %s", type) + log.debug("Skipping retransfer for explorers of %s", type) return else: # Do not retransfer @@ -506,7 +508,19 @@ class Cdist: paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) for bin in paths: if os.path.isfile(bin): - self.shell_run_or_debug_fail(bin, [bin], env=env) + # omit "gen" from gencode and + outfile=os.path.join(self.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + + a = outfile_fd + b = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR + print(type(a), type(b)) + + os.fchmod(outfile_fd, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + outfile_fd.close() if mode == "code": # paths = self.object_code_paths(cdist_object) From 4e4d648593305836944dfa10811054df7ae7427e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:18:30 +0200 Subject: [PATCH 85/87] fchmod() does not work on object returned by open(), so use chmod Signed-off-by: Nico Schottelius --- bin/cdist | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index e76ecfc7..09cb9d45 100755 --- a/bin/cdist +++ b/bin/cdist @@ -514,14 +514,10 @@ class Cdist: outfile_fd = open(outfile, "w") self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - - a = outfile_fd - b = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR - print(type(a), type(b)) - - os.fchmod(outfile_fd, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) outfile_fd.close() + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + if mode == "code": # paths = self.object_code_paths(cdist_object) local_dir = self.object_dir(cdist_object) From 34ae8c292689780db3c18047bcd9d9493cf4f9be Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:41:39 +0200 Subject: [PATCH 86/87] make code execution happen in the end Signed-off-by: Nico Schottelius --- bin/cdist | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 09cb9d45..1fd3e116 100755 --- a/bin/cdist +++ b/bin/cdist @@ -516,7 +516,13 @@ class Cdist: self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) outfile_fd.close() - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == 0: + os.unlink(outfile) + else: + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) if mode == "code": # paths = self.object_code_paths(cdist_object) @@ -527,9 +533,12 @@ class Cdist: if os.path.isfile(bin): self.run_or_fail([bin], remote=False) - if os.path.isfile(os.path.join(local_dir, "code-remote")): - remote_code = os.path.join(remote_dir, "code-remote") - self.run_or_fail([remote_code], remote=True) + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.transfer_file(local_remote_code, remote_remote_code) + self.run_or_fail([remote_remote_code], remote=True) def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" From e56bbfea9ac0aba51f28200848d6bfa48ed28b49 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:45:54 +0200 Subject: [PATCH 87/87] less verbose output in default case Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 1fd3e116..4267b3fa 100755 --- a/bin/cdist +++ b/bin/cdist @@ -415,7 +415,7 @@ class Cdist: output_fd.close() def init_deploy(self): - log.info("Creating clean directory structure") + log.debug("Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote # remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) @@ -542,7 +542,7 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to host " + self.target_host) + log.info("Deploying to " + self.target_host) self.init_deploy() self.run_global_explores() self.run_initial_manifest()