From 37d4ecd9073dbc4294b9b30f136cd127bec6d0f8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 23 Jun 2011 13:38:36 +0200 Subject: [PATCH 001/166] init.sample fix Franky Van Liedekerke: > - the example init.sample file mentioned "deinstalled" for package > state, but I believe it should be "removed" Signed-off-by: Nico Schottelius --- conf/manifest/init.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/manifest/init.sample b/conf/manifest/init.sample index b683e02f..fca959e2 100755 --- a/conf/manifest/init.sample +++ b/conf/manifest/init.sample @@ -50,7 +50,7 @@ case "$__target_host" in cdist-debian) __package_apt atop --state installed - __package apache2 --state deinstalled + __package apache2 --state removed ;; cdist-redhat) From 41ee5125962352e439a4d5273e4fe9088e4fd1e7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 25 Jun 2011 23:12:38 +0200 Subject: [PATCH 002/166] update doc of __process: cannot start sshd, shouldn't stop sshd Signed-off-by: Nico Schottelius --- conf/type/__process/man.text | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/type/__process/man.text b/conf/type/__process/man.text index 3f7004c7..fd3bcf49 100644 --- a/conf/type/__process/man.text +++ b/conf/type/__process/man.text @@ -35,16 +35,16 @@ EXAMPLES -------- -------------------------------------------------------------------------------- -# Start /usr/sbin/sshd if not running -__process /usr/sbin/sshd --state running +# Start if not running +__process /usr/sbin/syslog-ng --state running -# Start /usr/sbin/sshd if not running with a different binary -__process /usr/sbin/sshd --state running --start "/etc/rc.d/sshd start" +# Start if not running with a different binary +__process /usr/sbin/nginx --state running --start "/etc/rc.d/nginx start" -# Stop the process using kill (the type default) +# Stop the process using kill (the type default) - DO NOT USE THIS __process /usr/sbin/sshd --state stopped -# Stop the process using /etc/rc.d/sshd stop +# Stop the process using /etc/rc.d/sshd stop - THIS ONE NOT AS WELL __process /usr/sbin/sshd --state stopped --stop "/etc/rc.d/sshd stop" # Ensure cups is running, which runs with -C ...: From c85aec7b1355857b30752c7b7a86332daf5de299 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Jul 2011 17:22:59 +0200 Subject: [PATCH 003/166] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/TAKEME | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index a3d74661..abe98cb4 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -48,6 +48,7 @@ CORE pink # empty file wood # empty file +- allow cdist to run without $PATH setup: ./bin/cdist-deploy-to TYPES ------ From 8da1c44081eacde29463709653a3d09e7d61a0a0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Jul 2011 17:28:47 +0200 Subject: [PATCH 004/166] import speech from cosin Signed-off-by: Nico Schottelius --- doc/speeches/2011-05-20_cosin.tex | 242 ++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 doc/speeches/2011-05-20_cosin.tex diff --git a/doc/speeches/2011-05-20_cosin.tex b/doc/speeches/2011-05-20_cosin.tex new file mode 100644 index 00000000..f60a660a --- /dev/null +++ b/doc/speeches/2011-05-20_cosin.tex @@ -0,0 +1,242 @@ +% first presentation about cmtp +\pdfminorversion=4 +%\documentclass[ucs]{beamer} +\documentclass{beamer} +%\documentclass[utf8]{beamer} +\usepackage[utf8]{inputenc} +\usepackage{german} +\usepackage{graphicx} +\usepackage{beamerthemesplit} +\setbeamercovered{dynamic} +\usetheme{Malmoe} +\usecolortheme{crane} + +\title{cdist - nutzbare Konfigurationsverwaltung} +\subtitle{Cosin 2011} + +\author{Nico -telmich- Schottelius} + +\date{25. Juni 2011} + +\begin{document} +\frame{\titlepage} + +%\section[Outline]{} +\frame{\tableofcontents} + +\section{Einleitung} +\frame +{ + \frametitle{Was ist das Problem?} + \begin{itemize} + \item Einmal konfigurieren = toll + \item Zweimal konfigurieren = naja, ... + \item Neue Sachen machen mehr Spass als alte wiederholen + \item Viele Rechner = viel Mühe? + \end{itemize} +} + +\frame +{ + \frametitle{Das ist nicht neu...} + \begin{itemize} + \item cfengine + \item Puppet + \item bcfg2 + \item chef + \item ... + \end{itemize} +} + +\frame +{ + \frametitle{Warum cdist?} + \begin{itemize} + \item Klein + \item Unix + \item Leicht zu bedienen + \item ... zu erweitern + \item Shell + \item Weil es Spaß macht! + \end{itemize} +} + +\section{Installieren} +\frame +{ + \frametitle{Vorraussetzungen} + \begin{itemize} + \item sshd + \item root login via sshd + \item Besser: ssh-pubkey konfiguriert (PermitRootLogin without-password) + \item git + \item Asciidoc für dia manpages + \end{itemize} +} + +\frame +{ + \frametitle{Installation} + \begin{center} + git clone git://git.schottelius.org/cdist + \end{center} +} + +\begin{frame}[fragile] + \frametitle{Erstellen der Manpages} + + \begin{verbatim} + # Braucht asciidoc / a2x + ./build.sh man + \end{verbatim} +\end{frame} + +\section{Nutzen} +\begin{frame}[fragile] + \frametitle{Vorbereitung PATH und MANPATH} + + \begin{verbatim} + cd cdist + eval `./bin/cdist-env` + echo $PATH + echo $MANPATH + \end{verbatim} +\end{frame} + +\begin{frame}[fragile] + \frametitle{Nun los} + \begin{verbatim} + # Fangen wir bei uns an + cdist-deploy-to localhost + \end{verbatim} +\end{frame} + +\begin{frame}[fragile] + \frametitle{Der Einstiegspunkt} + \begin{small} + \begin{verbatim} + cat << eof > conf/manifest/init + __file /etc/cdist-configured + + case "$__target_host" in + localhost) + __link /tmp/cdist-testfile \ + --source /etc/cdist-configured \ + --type symbolic + __addifnosuchline /tmp/cdist-welcome \ + --line "Welcome to cdist" + ;; + esac + eof + # Muss ausführbar sein + chmod u+x conf/manifest/init + + \end{verbatim} + \end{small} +\end{frame} + +\begin{frame}[fragile] + \frametitle{Nun los} + \begin{verbatim} + # Nun läuft es! + cdist-deploy-to localhost + \end{verbatim} +\end{frame} + +\frame +{ + \frametitle{Funktionalität zusammenfassen} + \begin{itemize}[<+->] + \item "`Typen"' (types) + \item conf/type/* + \item \_\_ vor jedem Namen (Shell-Umgebung) + \item z.B. Netzseite, Mailserver, Wiki, ... + \end{itemize} +} + +\begin{frame}[fragile] + \frametitle{Ein neuer Typ} + \begin{small} + \begin{verbatim} + % mkdir conf/type/__my_mailserver + % cat << eof > conf/type/__my_mailserver/manifest + # Dieser Typ konfiguriert meinen Mailserver + require="__package/nullmailer" \ + __file /etc/nullmailer/remotes \ + --source "$__type/files/remotes" + + # Reihenfolge spielt keine Rolle + __package nullmailer --state installed + eof + \end{verbatim} + \end{small} +\end{frame} + +\begin{frame}[fragile] + \frametitle{Ein neuer Typ (2)} + \begin{small} + \begin{verbatim} + # Wichtig: Wird ausgeführt + % chmod u+x conf/type/__my_mailserver/manifest + + # Darf nur einmal verwendet werden pro Rechner + % touch conf/type/__my_mailserver/singleton + + # Nullmailer Konfiguration + % mkdir conf/type/__my_mailserver/files + % echo my.fancy.smart.host > \ + conf/type/__my_mailserver/files/remotes + \end{verbatim} + \end{small} +\end{frame} + +\begin{frame}[fragile] + \frametitle{Neuen Typ nutzen} + \begin{small} + \begin{verbatim} + % $EDITOR conf/manifest/init + + case "$__target_host" in + localhost) + ... + __my_mailserver + ... + ;; + \end{verbatim} + \end{small} +\end{frame} + +\section{Aktualisieren} +\begin{frame}[fragile] + \frametitle{Versionen} + \begin{itemize}[<+->] + \item x.y: Stabile Version + \item master: Entwicklung + \end{itemize} +\end{frame} + +\begin{frame}[fragile] + \frametitle{Stabile Version auswählen} + \begin{center} + git checkout -b 1.7 origin/1.7 + \end{center} +\end{frame} + +\begin{frame}[fragile] + \frametitle{Aktualisieren} + \begin{center} + git pull + \end{center} +\end{frame} + +\frame +{ + \frametitle{Ende} + \begin{block}{Das war's} + Viel Spaß - Mehr Infos gibt's auf http://www.nico.schottelius.org/software/cdist/\\ + und http://l.schottelius.org/pipermail/cdist + \end{block} +} + + +\end{document} From 4d48da8e96d115a27618f65f0484beeb755d7110 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Jul 2011 19:26:44 +0200 Subject: [PATCH 005/166] import __package_zypper into contrib Signed-off-by: Nico Schottelius --- contrib/__package_zypper/README | 5 ++ contrib/__package_zypper/explorer/pkg_version | 30 +++++++++++ contrib/__package_zypper/gencode-remote | 51 ++++++++++++++++++ contrib/__package_zypper/man.text | 52 +++++++++++++++++++ contrib/__package_zypper/parameter/optional | 1 + contrib/__package_zypper/parameter/required | 1 + 6 files changed, 140 insertions(+) create mode 100644 contrib/__package_zypper/README create mode 100755 contrib/__package_zypper/explorer/pkg_version create mode 100755 contrib/__package_zypper/gencode-remote create mode 100644 contrib/__package_zypper/man.text create mode 100644 contrib/__package_zypper/parameter/optional create mode 100644 contrib/__package_zypper/parameter/required diff --git a/contrib/__package_zypper/README b/contrib/__package_zypper/README new file mode 100644 index 00000000..1e073e3f --- /dev/null +++ b/contrib/__package_zypper/README @@ -0,0 +1,5 @@ +This type was not accepted, because cleanups are needed and the +manpage does not build. + +If you read this and want this code available in the cdist core, +just fix it and submit a git url :-) diff --git a/contrib/__package_zypper/explorer/pkg_version b/contrib/__package_zypper/explorer/pkg_version new file mode 100755 index 00000000..0e078f68 --- /dev/null +++ b/contrib/__package_zypper/explorer/pkg_version @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Retrieve the status of a package - parsed dpkg output +# + +if [ -f "$__object/parameter/name" ]; then + name="$(cat "$__object/parameter/name")" +else + name="$__object_id" +fi + +rpm -q --whatprovides "$name" 2>/dev/null || true diff --git a/contrib/__package_zypper/gencode-remote b/contrib/__package_zypper/gencode-remote new file mode 100755 index 00000000..2d1112d3 --- /dev/null +++ b/contrib/__package_zypper/gencode-remote @@ -0,0 +1,51 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Manage packages with yum (mostly Fedora) +# + +if [ -f "$__object/parameter/name" ]; then + name="$__object/parameter/name" +else + name="$__object_id" +fi + +state="$(cat "$__object/parameter/state")" + +opts="-n -q" + +not_installed="^no package provides" + +case "$state" in + installed) + if grep -q "$not_installed" "$__object/explorer/pkg_version"; then + echo zypper $opts install \"$name\" + fi + ;; + removed) + if ! grep -q "$not_installed" "$__object/explorer/pkg_version"; then + echo zypper $opts remove \"$name\" + fi + ;; + *) + echo "Unknown state: $state" >&2 + exit 1 + ;; +esac diff --git a/contrib/__package_zypper/man.text b/contrib/__package_zypper/man.text new file mode 100644 index 00000000..3a4f1026 --- /dev/null +++ b/contrib/__package_zypper/man.text @@ -0,0 +1,52 @@ +cdist-type__package_zypper(7) +========================== +Franky Van Liedekerke + + +NAME +---- +cdist-type__package_zypper - Manage packages with zypper + + +DESCRIPTION +----------- +zypper is usually used on the Suse distribution to manage packages. + + +REQUIRED PARAMETERS +------------------- +state:: + Either "installed" or "removed". + + +OPTIONAL PARAMETERS +------------------- +name:: + If supplied, use the name and not the object id as the package name. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Ensure zsh in installed +__package_zypper zsh --state installed + +# If you don't want to follow pythonX packages, but always use python +__package_zypper python --state installed --name python2 + +# Remove obsolete package +__package_zypper puppet --state removed +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__package(7) + + +COPYING +------- +Copyright \(C) 2011 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/contrib/__package_zypper/parameter/optional b/contrib/__package_zypper/parameter/optional new file mode 100644 index 00000000..f121bdbf --- /dev/null +++ b/contrib/__package_zypper/parameter/optional @@ -0,0 +1 @@ +name diff --git a/contrib/__package_zypper/parameter/required b/contrib/__package_zypper/parameter/required new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/contrib/__package_zypper/parameter/required @@ -0,0 +1 @@ +state From 4b288f754cdb7f9da28127dc68169f42ecc4e6fb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Jul 2011 20:01:42 +0200 Subject: [PATCH 006/166] ++quotes Signed-off-by: Nico Schottelius --- build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index b1d27817..eded032e 100755 --- a/build.sh +++ b/build.sh @@ -88,9 +88,9 @@ case "$1" in speeches) cd "$SPEECHESDIR" for speech in *tex; do - pdflatex $speech - pdflatex $speech - pdflatex $speech + pdflatex "$speech" + pdflatex "$speech" + pdflatex "$speech" done ;; From b264e4a4d3c1638bb4c32a809ecadf40552f7920 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Jul 2011 20:26:13 +0200 Subject: [PATCH 007/166] prepare 1.7.1 release Signed-off-by: Nico Schottelius --- README | 4 ++-- doc/changelog | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README b/README index 635d381d..295b9152 100644 --- a/README +++ b/README @@ -109,8 +109,8 @@ how to use cdist. There are at least the following branches available: - * master: the development branch - * 1.7: Bugfixes, cleanups, new type and explorer rename + * Development: master + * Current Stable: 1.7 (Bugfixes, cleanups, new type and explorer rename) Old versions: diff --git a/doc/changelog b/doc/changelog index a603573b..82ae4b95 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,4 +1,4 @@ -1.7.1: +1.7.1: 2011-07-26 * Documentation: Add explorers to reference * Documentation: Typo cleanup (Derek Brost) * Type __key_value: Bugfix (Steven Armstrong) From 84b8c7cdb0f32a14d3b204507da594cb0add7d8b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Jul 2011 21:25:12 +0200 Subject: [PATCH 008/166] add new cool fancy awk script to display contents of last release (awk is cool) Signed-off-by: Nico Schottelius --- doc/dev/lastchanges | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100755 doc/dev/lastchanges diff --git a/doc/dev/lastchanges b/doc/dev/lastchanges new file mode 100755 index 00000000..8679372a --- /dev/null +++ b/doc/dev/lastchanges @@ -0,0 +1,24 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Show the lastest changes from this (upcoming) release +# + +awk '/^$/ { exit } { print $0 } END { print "-----" }' < "${0%/*}/../changelog" From f44bc29789709b4d019d98c04136a5324cb2f8d0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Jul 2011 21:55:14 +0200 Subject: [PATCH 009/166] import example configuration from nicos private config Signed-off-by: Nico Schottelius --- .../conf/type/__nico_afs/files/afs/CellServDB | 2 + .../conf/type/__nico_afs/files/afs/ThisCell | 1 + .../conf/type/__nico_afs/files/krb5/krb5.conf | 28 +++++++++++ .../conf/type/__nico_afs/files/ssh/JAS.pub | 1 + .../type/__nico_afs/files/ssh/bugblue.pub | 1 + .../type/__nico_afs/files/ssh/docsteel.pub | 1 + .../type/__nico_afs/files/ssh/downhill.pub | 1 + .../type/__nico_afs/files/ssh/sur5r@samsa.pub | 1 + .../nico/conf/type/__nico_afs/manifest | 45 ++++++++++++++++++ .../nico/conf/type/__nico_afs/singleton | 0 .../conf/type/__nico_desktop/files/hostname | 1 + .../type/__nico_desktop/files/slim-preseed | 4 ++ .../nico/conf/type/__nico_desktop/manifest | 47 +++++++++++++++++++ .../type/__nico_desktop/parameter/required | 1 + .../nico/conf/type/__nico_desktop/singleton | 0 .../conf/type/__nico_mpd/files/slim-preseed | 4 ++ .../nico/conf/type/__nico_mpd/manifest | 24 ++++++++++ .../conf/type/__nico_mpd/parameter/required | 1 + .../nico/conf/type/__nico_mpd/singleton | 0 .../type/__nico_network/files/interfaces-eth0 | 6 +++ .../__nico_network/files/interfaces-wlan0 | 11 +++++ .../nico/conf/type/__nico_network/manifest | 43 +++++++++++++++++ .../type/__nico_network/parameter/required | 1 + .../nico/conf/type/__nico_network/singleton | 0 .../type/__nico_nfs_client/files/slim-preseed | 4 ++ .../nico/conf/type/__nico_nfs_client/manifest | 47 +++++++++++++++++++ .../conf/type/__nico_nfs_client/singleton | 0 .../nico/conf/type/__nico_notebook/manifest | 38 +++++++++++++++ .../nico/conf/type/__nico_notebook/singleton | 0 .../nico/conf/type/__nico_packages/manifest | 33 +++++++++++++ .../nico/conf/type/__nico_packages/singleton | 0 .../conf/type/__nico_sudo/files/sudo-nico | 13 +++++ .../nico/conf/type/__nico_sudo/manifest | 27 +++++++++++ .../conf/type/__nico_sudo/parameter/gencode | 31 ++++++++++++ .../conf/type/__nico_sudo/parameter/manifest | 31 ++++++++++++ .../conf/type/__nico_sudo/parameter/optional | 0 .../conf/type/__nico_sudo/parameter/required | 0 .../nico/conf/type/__nico_sudo/singleton | 0 .../conf/type/__nico_tee/files/99-apt-nico | 2 + .../nico/conf/type/__nico_tee/files/hostname | 1 + .../nico/conf/type/__nico_tee/manifest | 28 +++++++++++ .../nico/conf/type/__nico_tee/singleton | 0 .../__package_zypper/README | 0 .../__package_zypper/explorer/pkg_version | 0 .../__package_zypper/gencode-remote | 0 .../__package_zypper/man.text | 0 .../__package_zypper/parameter/optional | 0 .../__package_zypper/parameter/required | 0 48 files changed, 479 insertions(+) create mode 100644 contrib/examples/nico/conf/type/__nico_afs/files/afs/CellServDB create mode 100644 contrib/examples/nico/conf/type/__nico_afs/files/afs/ThisCell create mode 100644 contrib/examples/nico/conf/type/__nico_afs/files/krb5/krb5.conf create mode 100644 contrib/examples/nico/conf/type/__nico_afs/files/ssh/JAS.pub create mode 100644 contrib/examples/nico/conf/type/__nico_afs/files/ssh/bugblue.pub create mode 100644 contrib/examples/nico/conf/type/__nico_afs/files/ssh/docsteel.pub create mode 100644 contrib/examples/nico/conf/type/__nico_afs/files/ssh/downhill.pub create mode 100644 contrib/examples/nico/conf/type/__nico_afs/files/ssh/sur5r@samsa.pub create mode 100755 contrib/examples/nico/conf/type/__nico_afs/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_afs/singleton create mode 100644 contrib/examples/nico/conf/type/__nico_desktop/files/hostname create mode 100644 contrib/examples/nico/conf/type/__nico_desktop/files/slim-preseed create mode 100755 contrib/examples/nico/conf/type/__nico_desktop/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_desktop/parameter/required create mode 100644 contrib/examples/nico/conf/type/__nico_desktop/singleton create mode 100644 contrib/examples/nico/conf/type/__nico_mpd/files/slim-preseed create mode 100644 contrib/examples/nico/conf/type/__nico_mpd/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_mpd/parameter/required create mode 100644 contrib/examples/nico/conf/type/__nico_mpd/singleton create mode 100644 contrib/examples/nico/conf/type/__nico_network/files/interfaces-eth0 create mode 100644 contrib/examples/nico/conf/type/__nico_network/files/interfaces-wlan0 create mode 100755 contrib/examples/nico/conf/type/__nico_network/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_network/parameter/required create mode 100644 contrib/examples/nico/conf/type/__nico_network/singleton create mode 100644 contrib/examples/nico/conf/type/__nico_nfs_client/files/slim-preseed create mode 100755 contrib/examples/nico/conf/type/__nico_nfs_client/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_nfs_client/singleton create mode 100755 contrib/examples/nico/conf/type/__nico_notebook/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_notebook/singleton create mode 100755 contrib/examples/nico/conf/type/__nico_packages/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_packages/singleton create mode 100644 contrib/examples/nico/conf/type/__nico_sudo/files/sudo-nico create mode 100755 contrib/examples/nico/conf/type/__nico_sudo/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_sudo/parameter/gencode create mode 100644 contrib/examples/nico/conf/type/__nico_sudo/parameter/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_sudo/parameter/optional create mode 100644 contrib/examples/nico/conf/type/__nico_sudo/parameter/required create mode 100644 contrib/examples/nico/conf/type/__nico_sudo/singleton create mode 100644 contrib/examples/nico/conf/type/__nico_tee/files/99-apt-nico create mode 100644 contrib/examples/nico/conf/type/__nico_tee/files/hostname create mode 100755 contrib/examples/nico/conf/type/__nico_tee/manifest create mode 100644 contrib/examples/nico/conf/type/__nico_tee/singleton rename contrib/{ => types_pending_inclusion}/__package_zypper/README (100%) rename contrib/{ => types_pending_inclusion}/__package_zypper/explorer/pkg_version (100%) rename contrib/{ => types_pending_inclusion}/__package_zypper/gencode-remote (100%) rename contrib/{ => types_pending_inclusion}/__package_zypper/man.text (100%) rename contrib/{ => types_pending_inclusion}/__package_zypper/parameter/optional (100%) rename contrib/{ => types_pending_inclusion}/__package_zypper/parameter/required (100%) diff --git a/contrib/examples/nico/conf/type/__nico_afs/files/afs/CellServDB b/contrib/examples/nico/conf/type/__nico_afs/files/afs/CellServDB new file mode 100644 index 00000000..dbd238d7 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/files/afs/CellServDB @@ -0,0 +1,2 @@ +>eof +129.132.186.89 # sgv-afs-sur5r diff --git a/contrib/examples/nico/conf/type/__nico_afs/files/afs/ThisCell b/contrib/examples/nico/conf/type/__nico_afs/files/afs/ThisCell new file mode 100644 index 00000000..37fb719b --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/files/afs/ThisCell @@ -0,0 +1 @@ +eof diff --git a/contrib/examples/nico/conf/type/__nico_afs/files/krb5/krb5.conf b/contrib/examples/nico/conf/type/__nico_afs/files/krb5/krb5.conf new file mode 100644 index 00000000..3b8259cb --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/files/krb5/krb5.conf @@ -0,0 +1,28 @@ +[libdefaults] + default_realm = EOF + +# The following krb5.conf variables are only for MIT Kerberos. + krb4_config = /etc/krb.conf + krb4_realms = /etc/krb.realms + kdc_timesync = 1 + ccache_type = 4 + forwardable = true + proxiable = true + allow_weak_crypto = true + +[realms] + EOF = { + kdc = sgv-afs-sur5r.ethz.ch + admin_server = sgv-afs-sur5r.ethz.ch + } + + +[login] + krb4_convert = true + krb4_get_tickets = false + +[logging] + default = FILE:/var/log/krb5.log + kdc = FILE:/var/log/krb5kdc/kdc.log + admin_server = FILE:/var/log/krb5kdc/kadmin.log + diff --git a/contrib/examples/nico/conf/type/__nico_afs/files/ssh/JAS.pub b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/JAS.pub new file mode 100644 index 00000000..222410d5 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/JAS.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/Gbl64LT3VD5hsVtk1w670S2gue1OaW9XLg/Vk/iBqsYYrLGZh+TXJsAsXOSF1sZH6QSdNlpzTPsno9KFCTQTNlYe0IrWPGRrFGw1or3M6OugJrMeSiMYQ5nhH6HMjhzCFHH8Xh4Yku8fgi3ejPpySW8umx7nBL7ndiEJ9Y+lixNWMirEPLpz9YufFm9u8GX6bPrmjIBz6EhfxaqJ2N/N6gQB+4PmNopzqWHm+n4LToA9N8qwetSfhgEg7DVaD9SrJNjNTGSgii6CritT9sF8ZBq5CZG58DTyrxCndhhHte5OCGMb5ENgO4OBHA0MrD56unHrdAZCCosa9rI+pIll abr@cltHome03 diff --git a/contrib/examples/nico/conf/type/__nico_afs/files/ssh/bugblue.pub b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/bugblue.pub new file mode 100644 index 00000000..cccbe02f --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/bugblue.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA17fnWJMTAuIvWcdnsasdfEkBabeXEhYaR03Qc/KBpS+iToSeUzlc5SeXoAczqSMsC0uYpDnhll9q3aAN82Xo06RI2ytd7TeXvFcVwzvXB+sNUsvtyPZ5Uyx7d2WTI87bm169KhGTJCaww8p+qa2UhkjOOaXZwMGjkHlvZ3WSZr5mLar9O3r4PG8SIqoFF0m+tcc2fcWIK3df3jWIk8g6j/jTaoIa18qsK/rtO90Ql20FMQJOZTKGKjHIOx2FLnXY5WKrXmXyyffgFpqi1rUAkCkjCKnm65fDjecn6FplzSUuZo/IB2GnHGNQVnNkU/18/G8KQKu9clkMxuUl8DYJBQ== bugblue@jabber.eof.name diff --git a/contrib/examples/nico/conf/type/__nico_afs/files/ssh/docsteel.pub b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/docsteel.pub new file mode 100644 index 00000000..4b40d089 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/docsteel.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA1xfKoN8D3I0uGvc66E2cpnutdgALbSVIIWy0SBGV1ZLA4CehAL5BpMO0EI1TfH4LGgpjg+CnLXOSMd+bnvWjPTxGUbGcmK45UYCyn1LzSAfVKi9Mr06wbvQj0h5w1zEAwDqt63SHGjGOHO4TeCSrPxEROPMbZ1mP1ECsb4f+3WLWE5icbzOb/QMx2zNDd29rVvFJiJMOg4AiIs7pl/T7Qxg2yN6YlDIXSXLiE2i98O26kBNWRgAFcTNBqoUs5AkZ2F4LPUGbyuLpV+wtlpYcQXOUTLoRlKw+ovBQH3L6ae9n6+rFTIEEAS08C0MOzQPC3QjmfRMC8mxVkn22XnpHbQ== default diff --git a/contrib/examples/nico/conf/type/__nico_afs/files/ssh/downhill.pub b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/downhill.pub new file mode 100644 index 00000000..bde07c29 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/downhill.pub @@ -0,0 +1 @@ +ssh-dss AAAAB3NzaC1kc3MAAAIBAMiYvUWD0Yq3vm2Dp6xJCGvRGcGIyiT8+07FQJvAWUAzudyhA9r+h58gm4uCUJTV5W33Npf70Id5LSxyZJA7LcdTXTuMxhVfERURcT/GgtxCrs+aMguitNvf7QVuiKBrvuBmBPMV/6k8UEwf/7eCQRjoXE4jxEYKUT9SW8X94wuH0HUqOqBRD2F21uMrmLgDkb12RK/9yFRV3c7waOSQU/QtO+VFwPvBNDqUTqBL0LOJJif449vMxboOZ1noS94K0Lyfz9yOGEwBYck11c2UzH4KXbv8qNpYgtuCmDjZFM2J8dnhWJgkmThZtmyfnNFbHlW0HZItVvkqLZMPDlCIqR77J9OC4lawjrX2FFKhAzcrJuw7WXr2PcFKQUh/TiypcM5f6zuU3fs2+8ZYQdwvU6j/QNW4A/zqud7v/hjAMYCVe0EIWf2Qt0SS/nFLh7dZRGV21nK9Vq0zDncVPTgDl7/L62TYieO/j/1X3HjEp2JbR+mjBWsfKM7WYZDP16xiQzBuhr0vItnKMyN9V4AbDthjqesezKuXIhv0jP1z40MppM9mr40FJpgRSY3hyt3cZQKoO2vVJevnJMuufLheocAxo037f2PUpmSiJDOF1dLywmS2Gqk4GgzNN40IPrOcz0umtKjMAtXeU6MeapfmmEbwk+2zOo75gUt6SWU8UiFhAAAAFQC/Fm0V7OYGdazrUNuyn7mcPknZhQAAAgBzO496WPYnn7/H336kdMOUoue4Kgr9shpgjyyTJ8K9UsFwm+IEh9iS0QKPgGnj54AY1FpspfrbCmRI0tma+pj2QlnRRwIGcHd7eh2nCOmr+DSD/36VmoRPvGZaJDSTVotN+qgyjddNhCGx0417fqHXSKypbASphBqyvcKwkpk8S7o5nkMeOhufeCNdTHYsenKha4W+p8srGBsIZBISNeaGAsESIK4LuaShuolciTXtT7Nsqo123EXmjdHrT7DHQuwKI+jJpvHcz/UrissLs1JD+lFLOE07lkHPDc4KKK6IDUjm/DzsVTlTWdrcn8wUZ7fhUTt7e8UDNHs6bMlnhtVkXEsHS3sbYRsbF9179jufUumXKLKkjLzpTZwni8D6GxnjUn51hQ9Ifb9UOlSlkRq+cqOU+TRQd5aQSdiXy5Ymao719MHBrAhH1aLbi9pk4VO3GGFNy/w54ZY2LRIXZsGMBgFTXzHyzqv1ejeQtOuQZw9xOzt9IZU3WuMKVGGR9D61rsgxbGI1aWGaDlyhMSL9LYyqkmEqVqnAyydAVmhpSxhoXjbDrwE/IRdjJwjlK/6NxUck9g7Ekc9pHrow0OmqH9k6SCd2npXBUybTXPYqwHUjY/KccJsW0Ia2OECGN6KgNWdSfFeAGJBrv2ct78laSyNgIguM+0MNOZQSr6QfSQAAAgEAqkHqwfmEtrlc7hKtvenEf4Dkgt6H33U7MJILNOo9qrn/StDeuuO1snbO2wbd8weJop7gnp29zJGRKYcs/p2T3YjOd9R6aRGLOlT+jEZjP1RMPeuT7W4UFajP34SezYc1MAMeT9wkABEBQyj6s+4CvC2tKJWoirziAvkSPfkYdOc3u9I1LuXHu21tP+Lky3K9KylsbnHDG6vw05GH3HbeXIa6LtxAkXiPp/+r0dABO5NzglHS36uaD4mbQGh2dIzWPm4j8mLrjg911R8XnLPdTgT2EB9mvXksLjWEEq7qzTSdacFG0127O+i4Be3h1+5wG5HM8ST1n5hTOX8tTywF3DJL5HbLNHdDQo/YlT6l/wk6HbnYdcZHviVHi6va3/lFTdhoTEPz5sgDYQp5/0vobiMyIRHSZwzcYmswHumpf6Wql/phff8xigJBDAbGdFgx1Jk2OoOVGNWEchZuzlXyfgQpatnzBcR9CAAJvAfQLPqcHb8jGyBINuoNY/0OsAbsQDDzSjOLkBEBVgBTVZbykcik2n7kBQFeNj8zAUeLQCnQcEGIGY7OkPoxaAHBGKh4+e5Wymz0fryKTJ9io78qLHTmc6xs/ep4UwlO4Ee/e9p7cGb2pOl98jGFCWUklqyzeksBTOQcfKZBPV5dYap4m6nrw59XYpVbV9yPRJ/yMhE= downhill@downhill.homelinux.net diff --git a/contrib/examples/nico/conf/type/__nico_afs/files/ssh/sur5r@samsa.pub b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/sur5r@samsa.pub new file mode 100644 index 00000000..a1fcba83 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/files/ssh/sur5r@samsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAABAEApDTX05FmMS1ArryWL6MmgcQeRFRU6E4Rgg+cPTeBuHCtBnnUoCUw7lhD2OICHIzQnfyMQasJOHH/4PnsrAxyO1Lm4KtK3zsdSdA5auunxAy0n7PZwaRKDTXCgpfXvi7ZAlzA+Mq/OjqtltfqdJgToYKxWoehwmltlwibuU1fke2v85VcTbCQRAttc6+0Y+3fToyTWecZM+X0uCsz1B5s07CLrMddo3lPVAlhYStSCbHflsM6C+NQAbxlsjkVFavii7WDYMYOd5FU0jIJt2Fy6u7Lx0ubY6hgGsP+mDOCCaRGA0JdjvWctBruj227CLswtAnYIFxvPi82R/okfx2X4YPtbqUPyLaUbr0G93g+raJEr/uXbour+wRd0TewOLq4VstwOsDfj9pm0wMFyIbsY2y97k6UZc0TE5pu3USGyW9/ainy5zD4TK4Al8lMkDHil6eItlud66KDZ4p5n5gzwuBj+ZOpOcBD5ZqLqDKst3YlHx6EuA1ddObBTrfy/nGphYYhWl1rbJ9+XOhSD8f/LIr5mjLEpSta6rHS/3dpLpSRGIy8ReG0RLbfay/fS74Iw0rEGOe/XgrfNDT0VwsgJMNV81sReepk27DaFD/vES/iPAymbbId8e8IQ7kDbhV0yK0yTkCZ5capqa9HXcut0SFRVJYGxzGT+ji5o/DcyAcaQHK9IW5i89sp04aVtZO+KZZDd2GmcDy+v9+fmBsSx2AFsoOQSXX31jJVdAXNw8idTNb88/3XDZIIEl+1KJ8Pv4UFXBW72RArpOxOrsDZYQjtaLQ4ZjTP3h823ZrBh4W3osb1znS7x4MmWBLPkmLCS0zmN8nbqhKi5EsTmSheCjCzySGShkyeqSGRIRGFk5PUsgh7hYvE+f7BhWD6x8MHbuUp9y0ODQonp022Dc4WzTc6Aa023MSNRuwV373tOqPYveuoPXDTS6vzV3IjXfv9a844HIkUTZbErxkavGBN5TEMgrALJkc8LS6M6Zg8odou8N2vWoNaKPn/DC5+H+FxJ2PSrK93hfzRMgvFvPSFzzDnixUFJClSqxf1Wvx9OW2pUuePAXBlcuFhAAnWV6w7fkmII8+qGk3m438dt2Sq6owmItzqIeJ6bohMsb9ejxeDyfk9DAQwmjS7S+BY47bYgAfsesZNRnlbeffp4rP0MAx4KoUXoNzb8tl1Jljulq58C0e5+EzRILqOYLM7WytY3+giqxN4zQJsqDp3mgSUaZ5SEHnA8JVi77MvABSoVclrEaujSLVEkxTBOiR252SFUWp3lWbxpGzBjd7gX4JAiytJql5xr+o4+nCy2O2laE6c2xS7en6SVEgC7jWflCsbDfvy9U+w1qOxa0j9fXE9Rw== sur5r@samsa diff --git a/contrib/examples/nico/conf/type/__nico_afs/manifest b/contrib/examples/nico/conf/type/__nico_afs/manifest new file mode 100755 index 00000000..7b4cc493 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_afs/manifest @@ -0,0 +1,45 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +__directory /root/.ssh --mode 0700 + +__file /root/.ssh/authorized_keys --mode 0600 + +cd "$__type/files/ssh" +for key in *; do + require="__directory/root/.ssh" \ + __addifnosuchline ssh-$key --file /root/.ssh/authorized_keys \ + --line "$(cat "$key")" +done + +for pkg in openafs-client openafs-krb5 krb5-user; do + __package $pkg --state installed +done + +# Kerberos Config +__file /etc/krb5.conf --source "$__type/files/krb5/krb5.conf" + +# AFS config +cd "$__type/files/afs" +for afsconf in *; do + __file /etc/openafs/$afsconf --source "$__type/files/afs/$afsconf" +done diff --git a/contrib/examples/nico/conf/type/__nico_afs/singleton b/contrib/examples/nico/conf/type/__nico_afs/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_desktop/files/hostname b/contrib/examples/nico/conf/type/__nico_desktop/files/hostname new file mode 100644 index 00000000..a4df6242 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_desktop/files/hostname @@ -0,0 +1 @@ +scheibe diff --git a/contrib/examples/nico/conf/type/__nico_desktop/files/slim-preseed b/contrib/examples/nico/conf/type/__nico_desktop/files/slim-preseed new file mode 100644 index 00000000..01448d74 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_desktop/files/slim-preseed @@ -0,0 +1,4 @@ +# Use slim, not gdm, if both are available +# Setup for slim, but value is available for gdm as well: +# gdm shared/default-x-display-manager select slim +slim shared/default-x-display-manager select slim diff --git a/contrib/examples/nico/conf/type/__nico_desktop/manifest b/contrib/examples/nico/conf/type/__nico_desktop/manifest new file mode 100755 index 00000000..7fbbe70f --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_desktop/manifest @@ -0,0 +1,47 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +# +# Ensure hostname is setup +# +__file /etc/hostname --source "$__object/parameter/hostname" + +# All Linux distros have those +for pkg in dvdbackup mplayer x11vnc xfmpc; do + __package $pkg --state installed +done + +case "$(cat "$__global/explorer/os")" in + debian|ubuntu) + require="__package/zsh" __user lyni --uid 1000 --shell /bin/zsh + + for pkg in chromium-browser xfce4 xtightvncviewer; do + __package $pkg --state installed + done + + # Make slim default desktop on Debian/Ubuntu + __debconf_set_selections slim --file "$__type/files/slim-preseed" + require="__debconf_set_selections/slim" __package_apt slim --state installed + ;; +esac + +# Including gaming fun - not within examples, too big for core inclusion :-p +# __nico_dosbox diff --git a/contrib/examples/nico/conf/type/__nico_desktop/parameter/required b/contrib/examples/nico/conf/type/__nico_desktop/parameter/required new file mode 100644 index 00000000..ecd88aee --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_desktop/parameter/required @@ -0,0 +1 @@ +hostname diff --git a/contrib/examples/nico/conf/type/__nico_desktop/singleton b/contrib/examples/nico/conf/type/__nico_desktop/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_mpd/files/slim-preseed b/contrib/examples/nico/conf/type/__nico_mpd/files/slim-preseed new file mode 100644 index 00000000..01448d74 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_mpd/files/slim-preseed @@ -0,0 +1,4 @@ +# Use slim, not gdm, if both are available +# Setup for slim, but value is available for gdm as well: +# gdm shared/default-x-display-manager select slim +slim shared/default-x-display-manager select slim diff --git a/contrib/examples/nico/conf/type/__nico_mpd/manifest b/contrib/examples/nico/conf/type/__nico_mpd/manifest new file mode 100644 index 00000000..d339bdf3 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_mpd/manifest @@ -0,0 +1,24 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +__package mpd --state installed + +require="__package/mpd" __file /etc/mpd.conf --source "$__type/files/mpd.conf" diff --git a/contrib/examples/nico/conf/type/__nico_mpd/parameter/required b/contrib/examples/nico/conf/type/__nico_mpd/parameter/required new file mode 100644 index 00000000..ecd88aee --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_mpd/parameter/required @@ -0,0 +1 @@ +hostname diff --git a/contrib/examples/nico/conf/type/__nico_mpd/singleton b/contrib/examples/nico/conf/type/__nico_mpd/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_network/files/interfaces-eth0 b/contrib/examples/nico/conf/type/__nico_network/files/interfaces-eth0 new file mode 100644 index 00000000..2a92eade --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_network/files/interfaces-eth0 @@ -0,0 +1,6 @@ +# generated by cdist +auto lo eth0 + +iface lo inet loopback + +iface eth0 inet dhcp diff --git a/contrib/examples/nico/conf/type/__nico_network/files/interfaces-wlan0 b/contrib/examples/nico/conf/type/__nico_network/files/interfaces-wlan0 new file mode 100644 index 00000000..49508eb2 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_network/files/interfaces-wlan0 @@ -0,0 +1,11 @@ +# This file describes the network interfaces available on your system +# and how to activate them. For more information, see interfaces(5). + +# The loopback network interface +auto lo +iface lo inet loopback + +auto wlan0 +iface wlan0 inet dhcp + wpa-ssid undef + wpa-psk rotrussland diff --git a/contrib/examples/nico/conf/type/__nico_network/manifest b/contrib/examples/nico/conf/type/__nico_network/manifest new file mode 100755 index 00000000..012c47f7 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_network/manifest @@ -0,0 +1,43 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +case "$(cat "$__global/explorer/os")" in + debian|ubuntu) + + interface="$(cat $__object/parameter/interface)" + + # + # Only Debian and alike supported currently + # + destination=/etc/network/interfaces + case "$interface" in + eth0|wlan0) + source="$__type/files/interfaces-${interface}" + ;; + *) + echo "Unknown interface: $interface" >&2 + exit 1 + ;; + esac + + __file "$destination" --source "$source" + ;; +esac diff --git a/contrib/examples/nico/conf/type/__nico_network/parameter/required b/contrib/examples/nico/conf/type/__nico_network/parameter/required new file mode 100644 index 00000000..b529896a --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_network/parameter/required @@ -0,0 +1 @@ +interface diff --git a/contrib/examples/nico/conf/type/__nico_network/singleton b/contrib/examples/nico/conf/type/__nico_network/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_nfs_client/files/slim-preseed b/contrib/examples/nico/conf/type/__nico_nfs_client/files/slim-preseed new file mode 100644 index 00000000..01448d74 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_nfs_client/files/slim-preseed @@ -0,0 +1,4 @@ +# Use slim, not gdm, if both are available +# Setup for slim, but value is available for gdm as well: +# gdm shared/default-x-display-manager select slim +slim shared/default-x-display-manager select slim diff --git a/contrib/examples/nico/conf/type/__nico_nfs_client/manifest b/contrib/examples/nico/conf/type/__nico_nfs_client/manifest new file mode 100755 index 00000000..ed1a872b --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_nfs_client/manifest @@ -0,0 +1,47 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +__directory /home/services + +for nfsshare in audio video; do + dir="/home/services/$nfsshare" + __addifnosuchline nfs-$nfsshare --file /etc/fstab \ + --line "192.168.42.1:$dir $dir nfs defaults 0 0" + require="__directory/home/services" __directory $dir +done + +__directory /home/services/eingehend +for nfsshare in bibliothek buch spiegel; do + dir="/home/services/eingehend/$nfsshare" + __addifnosuchline nfs-$nfsshare --file /etc/fstab \ + --line "192.168.42.1:$dir $dir nfs defaults,noauto 0 0" + require="__directory/home/services" __directory $dir +done + +case "$(cat "$__global/explorer/os")" in + debian|ubuntu) + __package nfs-common --state installed + ;; + + fedora|archlinux) + __package nfs-utils --state installed + ;; +esac diff --git a/contrib/examples/nico/conf/type/__nico_nfs_client/singleton b/contrib/examples/nico/conf/type/__nico_nfs_client/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_notebook/manifest b/contrib/examples/nico/conf/type/__nico_notebook/manifest new file mode 100755 index 00000000..7b010230 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_notebook/manifest @@ -0,0 +1,38 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +require="__package/zsh" __user nico --uid 1000 --shell /bin/zsh + +# +# Backup HD +# +for hd in usbhd eth-usbhd; do + dir="/home/services/$hd" + __addifnosuchline hd-$hd --file /etc/fstab \ + --line "/dev/mapper/$hd $dir auto defaults,noauto 0 0" + __directory $dir +done + +# +# Standard everywhere packages +# +__nico_packages +__motd diff --git a/contrib/examples/nico/conf/type/__nico_notebook/singleton b/contrib/examples/nico/conf/type/__nico_notebook/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_packages/manifest b/contrib/examples/nico/conf/type/__nico_packages/manifest new file mode 100755 index 00000000..a1e663f7 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_packages/manifest @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +for pkg in atop screen vim wget zsh; do + __package "$pkg" --state installed +done + +case "$(cat "$__global/explorer/os")" in + fedora) + __package nc --state installed + ;; + *) + __package netcat --state installed + ;; +esac diff --git a/contrib/examples/nico/conf/type/__nico_packages/singleton b/contrib/examples/nico/conf/type/__nico_packages/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_sudo/files/sudo-nico b/contrib/examples/nico/conf/type/__nico_sudo/files/sudo-nico new file mode 100644 index 00000000..d904c319 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_sudo/files/sudo-nico @@ -0,0 +1,13 @@ +# +# Cdist managed file +# + +# Personal one: nico, shared one: lyni +User_Alias NICO = nico, lyni + +Defaults timestamp_timeout=5 +Defaults !tty_tickets + +# Give out permissions +NICO ALL=(ALL) ALL +NICO ALL=(ALL) NOPASSWD: /usr/sbin/pm-suspend diff --git a/contrib/examples/nico/conf/type/__nico_sudo/manifest b/contrib/examples/nico/conf/type/__nico_sudo/manifest new file mode 100755 index 00000000..59315313 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_sudo/manifest @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +destination=/etc/sudoers.d/nico +source="$__type/files/sudo-nico" + +require="__package/sudo" __file "$destination" --source "$source" --mode 0440 + +__package sudo --state installed diff --git a/contrib/examples/nico/conf/type/__nico_sudo/parameter/gencode b/contrib/examples/nico/conf/type/__nico_sudo/parameter/gencode new file mode 100644 index 00000000..74792abf --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_sudo/parameter/gencode @@ -0,0 +1,31 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# USEFUL DESCRIPTION +# + +# +# This file should generate code on stdout, which will be collected by cdist +# and run on the target. +# +# To tell cdist to make use of it, you need to make it executable (chmod +x) +# +# + diff --git a/contrib/examples/nico/conf/type/__nico_sudo/parameter/manifest b/contrib/examples/nico/conf/type/__nico_sudo/parameter/manifest new file mode 100644 index 00000000..c696eda6 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_sudo/parameter/manifest @@ -0,0 +1,31 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# USEFUL DESCRIPTION +# + +# +# This is the manifest, which can be used to create other objects like this: +# __file /path/to/destination --source /from/where/ +# +# To tell cdist to make use of it, you need to make it executable (chmod +x) +# +# + diff --git a/contrib/examples/nico/conf/type/__nico_sudo/parameter/optional b/contrib/examples/nico/conf/type/__nico_sudo/parameter/optional new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_sudo/parameter/required b/contrib/examples/nico/conf/type/__nico_sudo/parameter/required new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_sudo/singleton b/contrib/examples/nico/conf/type/__nico_sudo/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/examples/nico/conf/type/__nico_tee/files/99-apt-nico b/contrib/examples/nico/conf/type/__nico_tee/files/99-apt-nico new file mode 100644 index 00000000..8d6d7c82 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_tee/files/99-apt-nico @@ -0,0 +1,2 @@ +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; diff --git a/contrib/examples/nico/conf/type/__nico_tee/files/hostname b/contrib/examples/nico/conf/type/__nico_tee/files/hostname new file mode 100644 index 00000000..a4df6242 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_tee/files/hostname @@ -0,0 +1 @@ +scheibe diff --git a/contrib/examples/nico/conf/type/__nico_tee/manifest b/contrib/examples/nico/conf/type/__nico_tee/manifest new file mode 100755 index 00000000..4c614027 --- /dev/null +++ b/contrib/examples/nico/conf/type/__nico_tee/manifest @@ -0,0 +1,28 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + + +__package unattended-upgrades --state installed + +__file /etc/apt/apt.conf.d/99-apt-nico \ + --source "$__type/files/99-apt-nico" \ + --mode 0644 diff --git a/contrib/examples/nico/conf/type/__nico_tee/singleton b/contrib/examples/nico/conf/type/__nico_tee/singleton new file mode 100644 index 00000000..e69de29b diff --git a/contrib/__package_zypper/README b/contrib/types_pending_inclusion/__package_zypper/README similarity index 100% rename from contrib/__package_zypper/README rename to contrib/types_pending_inclusion/__package_zypper/README diff --git a/contrib/__package_zypper/explorer/pkg_version b/contrib/types_pending_inclusion/__package_zypper/explorer/pkg_version similarity index 100% rename from contrib/__package_zypper/explorer/pkg_version rename to contrib/types_pending_inclusion/__package_zypper/explorer/pkg_version diff --git a/contrib/__package_zypper/gencode-remote b/contrib/types_pending_inclusion/__package_zypper/gencode-remote similarity index 100% rename from contrib/__package_zypper/gencode-remote rename to contrib/types_pending_inclusion/__package_zypper/gencode-remote diff --git a/contrib/__package_zypper/man.text b/contrib/types_pending_inclusion/__package_zypper/man.text similarity index 100% rename from contrib/__package_zypper/man.text rename to contrib/types_pending_inclusion/__package_zypper/man.text diff --git a/contrib/__package_zypper/parameter/optional b/contrib/types_pending_inclusion/__package_zypper/parameter/optional similarity index 100% rename from contrib/__package_zypper/parameter/optional rename to contrib/types_pending_inclusion/__package_zypper/parameter/optional diff --git a/contrib/__package_zypper/parameter/required b/contrib/types_pending_inclusion/__package_zypper/parameter/required similarity index 100% rename from contrib/__package_zypper/parameter/required rename to contrib/types_pending_inclusion/__package_zypper/parameter/required From df9bdef75e1554a87f3639d832e572bec4e7a21f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Jul 2011 21:56:19 +0200 Subject: [PATCH 010/166] shortcuts for public Signed-off-by: Nico Schottelius --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index eded032e..a6897d49 100755 --- a/build.sh +++ b/build.sh @@ -108,7 +108,7 @@ case "$1" in cd ${WEBDIR} && make pub ;; - pub) + p|pu|pub) git push --mirror git push --mirror github ;; From dc820cf31e35c7790d28b28c5047d4780769f1af Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 28 Jul 2011 10:54:39 +0200 Subject: [PATCH 011/166] also filter stderr and add hostname as prefix Signed-off-by: Nico Schottelius --- bin/cdist-mass-deploy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist-mass-deploy b/bin/cdist-mass-deploy index 5ebfa1c2..c2b6d6a5 100755 --- a/bin/cdist-mass-deploy +++ b/bin/cdist-mass-deploy @@ -42,13 +42,13 @@ fi i=0 while [ $# -gt 0 ]; do if [ "$parallel" ]; then - cdist-deploy-to "$1" | filter "$1" & + cdist-deploy-to "$1" 2>&1 | filter "$1" & # Record pid and host for use later i=$((i+1)) eval pid_$i=$! eval host_$i=\$1 else - cdist-deploy-to "$1" | filter "$1" + cdist-deploy-to "$1" 2>&1 | filter "$1" fi shift done From 22025217d92d23d6084c11262820948c18b4f187 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 9 Aug 2011 09:01:51 +0200 Subject: [PATCH 012/166] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 5ba529ad..28762ef4 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1 +1,26 @@ Catch broken instances in cdist-mass-deploy -p and report broken deployements at the end! + + +-------------------------------------------------------------------------------- +Bug with requirements when indirect requires is scheduled late: + +__package/collectd: Generating local code +__package/collectd: Generating remote code +__package/collectd: Transferring object +__package/collectd: Checking code-local +__package/collectd: Checking code-remote +__file/etc/collectd/collectd.conf: Generating local code +__file/etc/collectd/collectd.conf: Generating remote code +__file/etc/collectd/collectd.conf: Transferring object +__file/etc/collectd/collectd.conf: Checking code-local +__file/etc/collectd/collectd.conf: Executing code-local +scp: /etc/collectd/collectd.conf: No such file or directory +__file/etc/collectd/collectd.conf: Error: /tmp/cdist.PYKFWmj9QknE/out/object/__file/etc/collectd/collectd.conf/.cdist/code-local exited non-zero +__file/etc/collectd/collectd.conf: Warning: Faulty code: +# +# The following code was generated by /home/users/nico/oeffentlich/rechner/projekte/cdist-nutzung/conf/type/__file/gencode-local +# + +scp /home/users/nico/oeffentlich/rechner/projekte/cdist-nutzung/conf/type/__ethz_collectd/files/collectd.conf.client root@shrek08:/etc/collectd/collectd.conf +__file/etc/collectd/collectd.conf: Error: Aborting due to non-zero exit code. + From bbe0d4a5db592d8fe2568314244d59250aed8f69 Mon Sep 17 00:00:00 2001 From: nuex Date: Wed, 17 Aug 2011 09:21:00 -0400 Subject: [PATCH 013/166] __package_rubygem --- .../__package_rubygem/explorer/pkg_status | 30 +++++++++++ conf/type/__package_rubygem/gencode-remote | 51 +++++++++++++++++++ conf/type/__package_rubygem/man.text | 49 ++++++++++++++++++ .../type/__package_rubygem/parameter/optional | 2 + .../type/__package_rubygem/parameter/required | 1 + 5 files changed, 133 insertions(+) create mode 100755 conf/type/__package_rubygem/explorer/pkg_status create mode 100755 conf/type/__package_rubygem/gencode-remote create mode 100644 conf/type/__package_rubygem/man.text create mode 100644 conf/type/__package_rubygem/parameter/optional create mode 100644 conf/type/__package_rubygem/parameter/required diff --git a/conf/type/__package_rubygem/explorer/pkg_status b/conf/type/__package_rubygem/explorer/pkg_status new file mode 100755 index 00000000..97620892 --- /dev/null +++ b/conf/type/__package_rubygem/explorer/pkg_status @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2011 Chase Allen James (nx-cdist@nu-ex.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Retrieve the status of a rubygem +# + +if [ -f "$__object/parameter/name" ]; then + name="$(cat "$__object/parameter/name")" +else + name="$__object_id" +fi + +# Except gem failing, if package is not known / installed +gem list -i "$name" 2>/dev/null || exit 0 diff --git a/conf/type/__package_rubygem/gencode-remote b/conf/type/__package_rubygem/gencode-remote new file mode 100755 index 00000000..dfc08781 --- /dev/null +++ b/conf/type/__package_rubygem/gencode-remote @@ -0,0 +1,51 @@ +#!/bin/sh +# +# 2011 Chase Allen James +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Manage Rubygem packages +# + + +if [ -f "$__object/parameter/name" ]; then + name="$(cat "$__object/parameter/name")" +else + name="$__object_id" +fi + +state="$(cat "$__object/parameter/state")" +is_installed="$(grep "true" "$__object/explorer/pkg_status" || true)" + +case "$state" in + installed) + # Install only if non-existent + if [ -z "$is_installed" ]; then + echo gem install \"$name\" --no-ri --no-rdoc + fi + ;; + removed) + # Remove only if existent + if [ -n "$is_installed" ]; then + echo gem uninstall \"$name\" + fi + ;; + *) + echo "Unknown state: $state" >&2 + exit 1 + ;; +esac diff --git a/conf/type/__package_rubygem/man.text b/conf/type/__package_rubygem/man.text new file mode 100644 index 00000000..00adc8c1 --- /dev/null +++ b/conf/type/__package_rubygem/man.text @@ -0,0 +1,49 @@ +cdist-type__package_rubygem(7) +============================= +Chase Allen James > + + +NAME +---- +cdist-type__package_rubygem - Manage rubygem packages + + +DESCRIPTION +----------- +Rubygems is the default package management system for the Ruby programming language. + + +REQUIRED PARAMETERS +------------------- +state:: + Either "installed" or "removed". + + +OPTIONAL PARAMETERS +------------------- +name:: + If supplied, use the name and not the object id as the package name. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Ensure sinatra is installed +__package_rubygem sinatra --state installed + +# Remove package +__package_rubygem rails --state removed +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__package(7) + + +COPYING +------- +Copyright \(C) 2011 Chase Allen James. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/conf/type/__package_rubygem/parameter/optional b/conf/type/__package_rubygem/parameter/optional new file mode 100644 index 00000000..a52167d3 --- /dev/null +++ b/conf/type/__package_rubygem/parameter/optional @@ -0,0 +1,2 @@ +name +version diff --git a/conf/type/__package_rubygem/parameter/required b/conf/type/__package_rubygem/parameter/required new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/conf/type/__package_rubygem/parameter/required @@ -0,0 +1 @@ +state From b7deafc26d477e6318b27885081a7cba2637842e Mon Sep 17 00:00:00 2001 From: nuex Date: Wed, 17 Aug 2011 09:27:36 -0400 Subject: [PATCH 014/166] fix author email in gencode-remote --- conf/type/__package_rubygem/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__package_rubygem/gencode-remote b/conf/type/__package_rubygem/gencode-remote index dfc08781..daaba524 100755 --- a/conf/type/__package_rubygem/gencode-remote +++ b/conf/type/__package_rubygem/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Chase Allen James +# 2011 Chase Allen James # # This file is part of cdist. # From b82f2b0f23ba942f9607eacc4a08ca7feaf26354 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 20 Aug 2011 23:00:31 +0200 Subject: [PATCH 015/166] fixup manpage for cdist-type__package_rubygem Signed-off-by: Nico Schottelius --- conf/type/__package_rubygem/man.text | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/type/__package_rubygem/man.text b/conf/type/__package_rubygem/man.text index 00adc8c1..c6248ee3 100644 --- a/conf/type/__package_rubygem/man.text +++ b/conf/type/__package_rubygem/man.text @@ -1,6 +1,6 @@ cdist-type__package_rubygem(7) -============================= -Chase Allen James > +============================== +Chase Allen James NAME From 298784b12bef0600dd4df77a769b6039ea29bedd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 20 Aug 2011 23:02:53 +0200 Subject: [PATCH 016/166] __package_rubygem: remove optional parameter version, it is not used Signed-off-by: Nico Schottelius --- conf/type/__package_rubygem/parameter/optional | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/type/__package_rubygem/parameter/optional b/conf/type/__package_rubygem/parameter/optional index a52167d3..f121bdbf 100644 --- a/conf/type/__package_rubygem/parameter/optional +++ b/conf/type/__package_rubygem/parameter/optional @@ -1,2 +1 @@ name -version From 96b164979b7c57dd92514fa5bb4d4e715c8a2f4a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 21 Aug 2011 15:14:33 +0200 Subject: [PATCH 017/166] changes for the next version Signed-off-by: Nico Schottelius --- doc/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/changelog b/doc/changelog index 82ae4b95..7e98277a 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,3 +1,6 @@ +next: + * New Type: __package_rubygem (Chase Allen James) + 1.7.1: 2011-07-26 * Documentation: Add explorers to reference * Documentation: Typo cleanup (Derek Brost) From 98fba8f6bd86214f907341fd22808fe12b921959 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 31 Aug 2011 17:02:00 +0200 Subject: [PATCH 018/166] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/TAKEME | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index abe98cb4..5ded4298 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -49,6 +49,7 @@ CORE wood # empty file - allow cdist to run without $PATH setup: ./bin/cdist-deploy-to +- use absent/present for state by default TYPES ------ From 1b84a2851146c78fe1c122afc216ec0f8d7b4a07 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 00:41:37 +0200 Subject: [PATCH 019/166] add cdist Signed-off-by: Nico Schottelius --- bin/cdist | 438 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 438 insertions(+) create mode 100644 bin/cdist diff --git a/bin/cdist b/bin/cdist new file mode 100644 index 00000000..a70a11f8 --- /dev/null +++ b/bin/cdist @@ -0,0 +1,438 @@ +#!python3 +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +__cdist_version="1.7.0" + +# Fail if something bogus is going on +set -u + +################################################################################ +# cconf standard vars prefixed with cdist + +__cdist_pwd="$(pwd -P)" +__cdist_mydir="${0%/*}"; +__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)" +__cdist_myname=${0##*/}; +__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" + +################################################################################ +# Names / Constants +# +# Most values can be overriden from outside, so you can +# customise paths as you like (for distributors, geeks and hackers) +# + +: ${__cdist_name_bin:=bin} +: ${__cdist_name_cache:=cache} +: ${__cdist_name_code:=code} +: ${__cdist_name_conf_dir:=conf} +: ${__cdist_name_dot_cdist:=.cdist} +: ${__cdist_name_explorer:=explorer} +: ${__cdist_name_gencode:=gencode} +: ${__cdist_name_gencode_local:=local} +: ${__cdist_name_gencode_remote:=remote} +: ${__cdist_name_global:=global} +: ${__cdist_name_host:=host} +: ${__cdist_name_init:=init} +: ${__cdist_name_manifest:=manifest} +: ${__cdist_name_object:=object} +: ${__cdist_name_object_finished:=done} +: ${__cdist_name_object_prepared:=prepared} +: ${__cdist_name_object_id:=object_id} +: ${__cdist_name_object_source:=source} +: ${__cdist_name_objects_created:=.objects_created} +: ${__cdist_name_out_dir:=out} +: ${__cdist_name_parameter:=parameter} +: ${__cdist_name_parameter_required:=required} +: ${__cdist_name_parameter_optional:=optional} +: ${__cdist_name_require:=require} +: ${__cdist_name_self:=self} +: ${__cdist_name_singleton:=singleton} +: ${__cdist_name_target_host:=target_host} +: ${__cdist_name_target_user:=target_user} +: ${__cdist_name_type:=type} +: ${__cdist_name_type_bin:=type_bin} +: ${__cdist_name_type_explorer:=type_explorer} +: ${__cdist_name_type_explorer_pushed:=.explorer_pushed} + +# Used for IDs: Allow everything not starting with - and . +: ${__cdist_sane_regexp:=[^-\.].*} + +# Default remote user +: ${__cdist_remote_user:=root} + + +################################################################################ +# Exported variable names (usable for non core +# +: ${__cdist_name_var_explorer:=__$__cdist_name_explorer} +: ${__cdist_name_var_type_explorer:=__$__cdist_name_type_explorer} +: ${__cdist_name_var_global:=__$__cdist_name_global} +: ${__cdist_name_var_manifest:=__$__cdist_name_manifest} +: ${__cdist_name_var_target_host:=__$__cdist_name_target_host} +: ${__cdist_name_var_target_user:=__$__cdist_name_target_user} +: ${__cdist_name_var_object:=__$__cdist_name_object} +: ${__cdist_name_var_object_id:=__$__cdist_name_object_id} +: ${__cdist_name_var_self:=__$__cdist_name_self} +: ${__cdist_name_var_type:=__$__cdist_name_type} + + +################################################################################ +# Tempfiles +# +: ${__cdist_tmp_base_dir=/tmp} +__cdist_tmp_dir=$(mktemp -d "$__cdist_tmp_base_dir/cdist.XXXXXXXXXXXX") +__cdist_tmp_file=$(mktemp "$__cdist_tmp_dir/cdist.XXXXXXXXXXXX") + +################################################################################ +# Local Base +# +: ${__cdist_local_base_dir:=$__cdist_tmp_dir} + +# Cache may *NOT* be below __cdist_local_base_dir! +: ${__cdist_local_base_cache_dir:=$__cdist_abs_mydir/../$__cdist_name_cache} + +: ${__cdist_conf_dir:="$(cd "$__cdist_abs_mydir/../conf" && pwd -P)"} + +: ${__cdist_explorer_dir:=$__cdist_conf_dir/$__cdist_name_explorer} +: ${__cdist_manifest_dir:=$__cdist_conf_dir/$__cdist_name_manifest} +: ${__cdist_manifest_init:=$__cdist_manifest_dir/$__cdist_name_init} +: ${__cdist_type_dir:=$__cdist_conf_dir/$__cdist_name_type} + +################################################################################ +# Local output +# +: ${__cdist_out_dir:=$__cdist_local_base_dir/$__cdist_name_out_dir} +: ${__cdist_out_explorer_dir:=$__cdist_out_dir/$__cdist_name_explorer} +: ${__cdist_out_object_dir:=$__cdist_out_dir/$__cdist_name_object} +: ${__cdist_out_type_dir:=$__cdist_out_dir/$__cdist_name_type} +: ${__cdist_out_type_bin_dir:=$__cdist_out_dir/$__cdist_name_type_bin} + +: ${__cdist_objects_created:=$__cdist_out_object_dir/$__cdist_name_objects_created} + +################################################################################ +# Remote base +# +: ${__cdist_remote_base_dir:=/var/lib/cdist} +: ${__cdist_remote_bin_dir:=$__cdist_remote_base_dir/$__cdist_name_bin} +: ${__cdist_remote_conf_dir:=$__cdist_remote_base_dir/$__cdist_name_conf_dir} + +: ${__cdist_remote_explorer_dir:=$__cdist_remote_conf_dir/$__cdist_name_explorer} +: ${__cdist_remote_type_dir:=$__cdist_remote_conf_dir/$__cdist_name_type} + +################################################################################ +# Remote output +# +: ${__cdist_remote_out_dir:=$__cdist_remote_base_dir/$__cdist_name_out_dir} +: ${__cdist_remote_out_explorer_dir:=$__cdist_remote_out_dir/$__cdist_name_explorer} +: ${__cdist_remote_out_object_dir:=$__cdist_remote_out_dir/$__cdist_name_object} + + +################################################################################ +# Internal functions +# +__cdist_echo() +{ + __cdist_echo_type="$1"; shift + + set +u + if [ "$__cdist_object_self" ]; then + __cdist_echo_prefix="${__cdist_object_self}:" + else + __cdist_echo_prefix="core: " + fi + set -u + + case "$__cdist_echo_type" in + debug) + set +u + if [ "$__cdist_debug" ]; then + echo $__cdist_echo_prefix "Debug: $@" + fi + set -u + ;; + info) + echo $__cdist_echo_prefix "$@" + ;; + warn) + echo $__cdist_echo_prefix "Warning: $@" + ;; + error) + echo $__cdist_echo_prefix "Error: $@" >&2 + ;; + *) + echo "CORE BUG, who created the broken commit in $0?" >&2 + exit 23 + ;; + esac +} + +__cdist_exec_fail_on_error() +{ + set +e + sh -e "$@" + if [ "$?" -ne 0 ]; then + __cdist_echo error "$1 exited non-zero" + __cdist_echo warn "Faulty code:" + cat "$1" + __cdist_exit_err "Aborting due to non-zero exit code." + fi +} + +__cdist_exit_err() +{ + __cdist_echo error "$@" + exit 1 +} + +__cdist_usage() +{ + __cdist_exit_err "$__cdist_myname: $@" +} + +__cdist_init_deploy() +{ + __cdist_echo info "Creating clean directory structure " + + # Ensure there is no old stuff, neither local nor remote + rm -rf "$__cdist_local_base_dir" + ssh "${__cdist_remote_user}@$1" "rm -rf ${__cdist_remote_base_dir}" + + # Init base + mkdir -p "$__cdist_local_base_dir" + ssh "${__cdist_remote_user}@$1" "mkdir -p ${__cdist_remote_base_dir}" + + # Link configuration source directory - consistent with remote + ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" +} + +################################################################################ +# Cache +# +__cdist_cache_dir() +{ + cd "${__cdist_local_base_cache_dir}" && pwd -P +} + +__cdist_host_cache_dir() +{ + echo "$(__cdist_cache_dir)/$1" +} + +################################################################################ +# Object +# + +__cdist_object_code() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_code}-$2" +} + +__cdist_object_prepared() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_object_prepared}" +} + +__cdist_object_finished() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_object_finished}" +} + +__cdist_object_dir() +{ + echo "$(__cdist_object_base_dir "$1")/${__cdist_name_dot_cdist}" +} + +__cdist_object_base_dir() +{ + echo "${__cdist_out_object_dir}/$1" +} + + +__cdist_object_id_from_object() +{ + echo "${1#*/}" +} + +# Find objects, remove ./ and /MARKER +__cdist_object_list() +{ + local basedir="$1"; shift + + # Use subshell to prevent changing cwd in program + ( + cd "${basedir}" + + find . -name "$__cdist_name_dot_cdist" | \ + sed -e 's;^./;;' -e "s;/${__cdist_name_dot_cdist}\$;;" + ) +} + +__cdist_object_parameter_dir() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_parameter}" +} + +__cdist_object_require() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_require}" +} + +__cdist_object_source_name() +{ + echo "$1/${__cdist_name_object_source}" +} + +__cdist_object_source() +{ + cat "$(__cdist_object_source_name "$1")" +} + +__cdist_object_source_add() +{ + echo "$__cdist_manifest" >> "$(__cdist_object_source_name "$1")" +} + +__cdist_object_type_explorer_dir() +{ + echo "$(__cdist_object_dir "$1")/${__cdist_name_explorer}" +} + +################################################################################ +# Remote +# + +__cdist_remote_object_base_dir() +{ + echo "${__cdist_remote_out_object_dir}/$1" +} + +__cdist_remote_object_dir() +{ + echo "$(__cdist_remote_object_base_dir "$1")/${__cdist_name_dot_cdist}" +} + +__cdist_remote_object_parameter_dir() +{ + echo "$(__cdist_remote_object_dir "$1")/${__cdist_name_parameter}" +} + +__cdist_remote_object_type_explorer_dir() +{ + echo "$(__cdist_remote_object_dir "$1")/${__cdist_name_explorer}" +} + + +__cdist_remote_type_explorer_dir() +{ + echo "${__cdist_remote_type_dir}/$1/${__cdist_name_explorer}" +} + + +################################################################################ +# Traps +# +__cdist_tmp_removal() +{ + rm -rf "${__cdist_tmp_dir}" +} + +# Does not work in children, will be called again in every script! +# Use only in interactive "front end" scripts +__cdist_kill_on_interrupt() +{ + __cdist_tmp_removal + kill 0 + exit 1 +} + +# Remove tempfiles at normal exit +trap __cdist_tmp_removal EXIT + + +################################################################################ +# Type +# +__cdist_type_dir() +{ + echo "${__cdist_type_dir}/$1" +} + +__cdist_type_explorer_dir() +{ + echo "${__cdist_type_dir}/$1/${__cdist_name_explorer}" +} + +__cdist_type_from_object() +{ + echo "${1%%/*}" +} + +__cdist_type_has_explorer() +{ + # We only create output, if there's at least one explorer + # and can thus be used as a boolean ;-) + if [ -d "$(__cdist_type_explorer_dir "$1")" ]; then + ls -1 "$(__cdist_type_explorer_dir "$1")" + fi +} + +__cdist_type_explorer_pushed() +{ + [ -f "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" ] \ + && grep -q "$1" "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" +} + +__cdist_type_explorer_pushed_add() +{ + [ -d "$__cdist_out_type_dir" ] || mkdir "$__cdist_out_type_dir" + echo "$1" >> "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" +} + +__cdist_type_gencode() +{ + echo "${__cdist_type_dir}/$1/${__cdist_name_gencode}-$2" +} + +__cdist_type_manifest() +{ + echo "${__cdist_type_dir}/$1/${__cdist_name_manifest}" +} + +__cdist_type_parameter_dir() +{ + echo "$(__cdist_type_dir "$1")/${__cdist_name_parameter}" +} + +__cdist_type_parameter_optional() +{ + echo "$(__cdist_type_parameter_dir "$1")/$__cdist_name_parameter_optional" +} + +__cdist_type_parameter_required() +{ + echo "$(__cdist_type_parameter_dir "$1")/$__cdist_name_parameter_required" +} + +__cdist_type_singleton() +{ + echo "${__cdist_type_dir}/$1/${__cdist_name_singleton}" +} From d359f69f4bd907e88103aa41217724707a9c0325 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 00:49:29 +0200 Subject: [PATCH 020/166] +minimal __cdist_echo Signed-off-by: Nico Schottelius --- core/__cdist_cache | 34 ++++++++++++ core/__cdist_dir | 46 ++++++++++++++++ core/__cdist_dir_listing | 30 +++++++++++ core/__cdist_echo | 56 +++++++++++++++++++ core/__cdist_exit_err | 28 ++++++++++ core/__cdist_explorer_run | 79 +++++++++++++++++++++++++++ core/__cdist_explorer_run_global | 32 +++++++++++ core/__cdist_init_deploy | 40 ++++++++++++++ core/__cdist_is_executable | 44 +++++++++++++++ core/__cdist_kill_on_interrupt | 31 +++++++++++ core/__cdist_manifest_run | 53 ++++++++++++++++++ core/__cdist_manifest_run_init | 32 +++++++++++ core/__cdist_object_all | 60 +++++++++++++++++++++ core/__cdist_object_code_run | 54 +++++++++++++++++++ core/__cdist_object_explorer_run | 89 +++++++++++++++++++++++++++++++ core/__cdist_object_gencode | 66 +++++++++++++++++++++++ core/__cdist_object_gencode_run | 36 +++++++++++++ core/__cdist_object_list | 36 +++++++++++++ core/__cdist_object_manifest_run | 59 ++++++++++++++++++++ core/__cdist_object_prepare | 47 ++++++++++++++++ core/__cdist_object_run | 74 +++++++++++++++++++++++++ core/__cdist_run | 27 ++++++++++ core/__cdist_run_remote | 32 +++++++++++ core/__cdist_run_shell | 34 ++++++++++++ core/__cdist_tmp_removal | 27 ++++++++++ core/__cdist_type_build_emulation | 49 +++++++++++++++++ core/__cdist_usage | 27 ++++++++++ 27 files changed, 1222 insertions(+) create mode 100755 core/__cdist_cache create mode 100755 core/__cdist_dir create mode 100755 core/__cdist_dir_listing create mode 100755 core/__cdist_echo create mode 100755 core/__cdist_exit_err create mode 100755 core/__cdist_explorer_run create mode 100755 core/__cdist_explorer_run_global create mode 100755 core/__cdist_init_deploy create mode 100755 core/__cdist_is_executable create mode 100644 core/__cdist_kill_on_interrupt create mode 100755 core/__cdist_manifest_run create mode 100755 core/__cdist_manifest_run_init create mode 100755 core/__cdist_object_all create mode 100755 core/__cdist_object_code_run create mode 100755 core/__cdist_object_explorer_run create mode 100755 core/__cdist_object_gencode create mode 100755 core/__cdist_object_gencode_run create mode 100755 core/__cdist_object_list create mode 100755 core/__cdist_object_manifest_run create mode 100755 core/__cdist_object_prepare create mode 100755 core/__cdist_object_run create mode 100755 core/__cdist_run create mode 100755 core/__cdist_run_remote create mode 100755 core/__cdist_run_shell create mode 100755 core/__cdist_tmp_removal create mode 100755 core/__cdist_type_build_emulation create mode 100755 core/__cdist_usage diff --git a/core/__cdist_cache b/core/__cdist_cache new file mode 100755 index 00000000..95764d3d --- /dev/null +++ b/core/__cdist_cache @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2010 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Save the configuration tree into the local cache +# + +__cdist_cache() +{ + # Create base to move into + mkdir -p "${__cdist_local_base_cache_dir}" + + __cdist_echo info \ + "Caching to $(__cdist_host_cache_dir "$__cdist_target_host")" + rm -rf "$(__cdist_host_cache_dir "$__cdist_target_host")" + mv "$__cdist_local_base_dir" \ + "$(__cdist_host_cache_dir "$__cdist_target_host")" +} diff --git a/core/__cdist_dir b/core/__cdist_dir new file mode 100755 index 00000000..32ee0075 --- /dev/null +++ b/core/__cdist_dir @@ -0,0 +1,46 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Push a directory to a target, both sides have the same name (i.e. explorers) +# or +# Pull a directory from a target, both sides have the same name (i.e. explorers) +# + +__cdist_dir() +{ + [ $# -eq 3 ] || __cdist_usage " " + + # ${3%/*} will be the destination directory, so no subdirectories + # of the same name are created, if the directory is already existing + + if [ "$1" = "push" ]; then + # FIXME: add error handling with __cdist_run_remote_... or so + ssh "${__cdist_remote_user}@${__cdist_target_host}" \ + "mkdir -p \"$3\"" + scp -qr "$2" \ + "${__cdist_remote_user}@${__cdist_target_host}:${3%/*}" + elif [ "$1" = "pull" ]; then + mkdir -p "$3" + scp -qr "${__cdist_remote_user}@${__cdist_target_host}:$2" \ + "${3%/*}" + else + __cdist_exit_err "Unknown action $1" + fi +} diff --git a/core/__cdist_dir_listing b/core/__cdist_dir_listing new file mode 100755 index 00000000..f4aa2320 --- /dev/null +++ b/core/__cdist_dir_listing @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# List files in a directory, if it exists +# +# We only create output, if there's at least one entry +# and can thus be used as a boolean ;-) +# + +__cdist_dir_listing() +{ + [ -d "$1" ] && ls -1 "$1" +} diff --git a/core/__cdist_echo b/core/__cdist_echo new file mode 100755 index 00000000..a89d1821 --- /dev/null +++ b/core/__cdist_echo @@ -0,0 +1,56 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# echo / syslog alike function +# + +__cdist_echo() +{ + __cdist_echo_type="$1"; shift + + set +u + if [ "$__cdist_object_self" ]; then + __cdist_echo_prefix="${__cdist_object_self}:" + else + __cdist_echo_prefix="core: " + fi + set -u + + case "$__cdist_echo_type" in + debug) + if [ "$__cdist_debug" = 1 ]; then + echo $__cdist_echo_prefix "DEBUG: $@" + fi + ;; + info) + echo $__cdist_echo_prefix "$@" + ;; + warn) + echo $__cdist_echo_prefix "Warning: $@" + ;; + error) + echo $__cdist_echo_prefix "Error: $@" >&2 + ;; + *) + echo "CORE BUG, who created the broken commit in $0?" >&2 + exit 23 + ;; + esac +} diff --git a/core/__cdist_exit_err b/core/__cdist_exit_err new file mode 100755 index 00000000..303dbf20 --- /dev/null +++ b/core/__cdist_exit_err @@ -0,0 +1,28 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Print error and exit (perror() alike) +# + +__cdist_exit_err() +{ + __cdist_echo error "$@" + exit 1 +} diff --git a/core/__cdist_explorer_run b/core/__cdist_explorer_run new file mode 100755 index 00000000..9e58fa09 --- /dev/null +++ b/core/__cdist_explorer_run @@ -0,0 +1,79 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run explorers - FIXME: this function is ugly +# + +__cdist_explorer_run() +{ + [ $# -eq 5 ] || __cdist_usage " " + + # Ensure there is at least one explorer + num="$(ls -1 "$__cdist_explorer_dir" | wc -l)" + if [ "$num" -lt 1 ]; then + __cdist_exit_err "${__cdist_explorer_dir}: Contains no explorers" + fi + + # Check whether to setup variable for type explorer + case "$1" in + global) + ;; + type) + # FIXME: think about how and where this gets setup! + "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" + ;; + esac + + # Transfer explorers + __cdist_dir push "$2" "$3" + + # Create output directory + __cdist_run_remote mkdir -p "$4" + + # Execute all explorers - FIXME: isolate cd call? + cd "$2"; + # FIXME: cleanup double variable, no need when in directory + for __cdist_explorer_run_explorer in *; do + __cdist_explorer_explorer_name="${__cdist_explorer_run_explorer##*/}" + + if [ -f "$__cdist_explorer_run_explorer" ]; then + if [ ! -x "$__cdist_explorer_run_explorer" ]; then + __cdist_exit_err "Explorer \"$__cdist_explorer_run_explorer\" exists, but is not executable." + fi + + else + if [ -e "$__cdist_explorer_run_explorer" ]; then + __cdist_exit_err "Explorer \"$__cdist_explorer_run_explorer\" exists, but is not a file." + fi + fi + + # FIXME: no need for remote out dir probably? + # or should we leave it and continue using __cdist_dir pull? + __cdist_run_remote \ + "export $__cdist_name_var_explorer=\"$__cdist_remote_explorer_dir\";" \ + "export $__cdist_name_var_global=\"$__cdist_remote_out_dir\";" \ + "$3/$__cdist_explorer_run_explorer" ">" \ + "$4/$__cdist_explorer_run_explorer" || \ + __cdist_exit_err "Explorer $__cdist_explorer_run_explorer failed." + done + + # Transfer results back + __cdist_dir pull "$4" "$5" +} diff --git a/core/__cdist_explorer_run_global b/core/__cdist_explorer_run_global new file mode 100755 index 00000000..27359713 --- /dev/null +++ b/core/__cdist_explorer_run_global @@ -0,0 +1,32 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Copy & run the global explorers, i.e. not bound to types +# + +__cdist_explorer_run_global() +{ + __cdist_echo info "Running global explorers " + + # run the global explorers remotely + __cdist_explorer_run global \ + "$__cdist_explorer_dir" "$__cdist_remote_explorer_dir" \ + "$__cdist_remote_out_explorer_dir" "$__cdist_out_explorer_dir" +} diff --git a/core/__cdist_init_deploy b/core/__cdist_init_deploy new file mode 100755 index 00000000..4ac3168c --- /dev/null +++ b/core/__cdist_init_deploy @@ -0,0 +1,40 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Prepare deployment +# + +__cdist_init_deploy() +{ + __cdist_echo info "Creating clean directory structure " + + # Ensure there is no old stuff, neither local nor remote + rm -rf "$__cdist_local_base_dir" + ssh "${__cdist_remote_user}@${__cdist_target_host}" \ + "rm -rf ${__cdist_remote_base_dir}" + + # Init base + mkdir -p "$__cdist_local_base_dir" + ssh "${__cdist_remote_user}@${__cdist_target_host}" \ + "mkdir -p ${__cdist_remote_base_dir}" + + # Link configuration source directory - consistent with remote + ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" +} diff --git a/core/__cdist_is_executable b/core/__cdist_is_executable new file mode 100755 index 00000000..a7a6d174 --- /dev/null +++ b/core/__cdist_is_executable @@ -0,0 +1,44 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Test whether something is executable (that should be executable) or +# is missing +# + +__cdist_is_executable() +{ + [ $# -eq 1 ] || __cdist_exit_err "" + + if [ -e "$1" ]; then + if [ -f "$1" ]; then + if [ -x "$1" ]; then + # Exists and is a correct executable + true + else + __cdist_exit_err "$1 exists, but is not executable." + fi + else + __cdist_exit_err "$1 exists, but is not a file." + fi + else + # Does not exist + false + fi +} diff --git a/core/__cdist_kill_on_interrupt b/core/__cdist_kill_on_interrupt new file mode 100644 index 00000000..7cb711fa --- /dev/null +++ b/core/__cdist_kill_on_interrupt @@ -0,0 +1,31 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run the given command for each created object. +# + +# Does not work in children, will be called again in every script! +# Use only in interactive "front end" scripts +__cdist_kill_on_interrupt() +{ + __cdist_tmp_removal + kill 0 + exit 1 +} diff --git a/core/__cdist_manifest_run b/core/__cdist_manifest_run new file mode 100755 index 00000000..cf85d646 --- /dev/null +++ b/core/__cdist_manifest_run @@ -0,0 +1,53 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Let's build a cconfig tree from a configuration +# And save it into the cache tree +# + +__cdist_manifest_run() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_manifest="$1"; shift + + ################################################################################ + # Export information for cdist-type-emulator or manifest + # + + # Config dir should not get reset - FIXME: why did I do this? + export __cdist_conf_dir + + # Used to record the source in the object + export __cdist_manifest + + # Export information for manifests - __cdist_out_dir comes from cdist-config + export $__cdist_name_var_global="$__cdist_out_dir" + + ################################################################################ + # The actual run + # + + # Ensure binaries are existing - FIXME: move error handling into __cdist_type_build_emulation + __cdist_type_build_emulation \ + || __cdist_exit_err "Failed to build type emulation binaries" + + __cdist_run_shell "${__cdist_manifest}" +} diff --git a/core/__cdist_manifest_run_init b/core/__cdist_manifest_run_init new file mode 100755 index 00000000..e8fa63de --- /dev/null +++ b/core/__cdist_manifest_run_init @@ -0,0 +1,32 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Let's build a cconfig tree from a configuration +# And save it into the cache tree +# + +__cdist_manifest_run_init() +{ + # FIXME: probably do not export but always set explicitly? + export $__cdist_name_var_manifest="$__cdist_manifest_dir" + + __cdist_echo info "Running initial manifest for $__cdist_target_host " + __cdist_manifest_run "$__cdist_manifest_init" +} diff --git a/core/__cdist_object_all b/core/__cdist_object_all new file mode 100755 index 00000000..965d08f6 --- /dev/null +++ b/core/__cdist_object_all @@ -0,0 +1,60 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run the given command for each created object. +# + +__cdist_object_all() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_object_all_object_all_command="$1"; shift + + __cdist_object_all_object_all_objects="$__cdist_tmp_dir/objects" + + # Ensure object dir exists, so marker can be created + mkdir -p "${__cdist_out_object_dir}" + + # FIXME: : - why do we use a file? + # core/__cdist_object_manifest_run: touch "$__cdist_objects_created" + + # Loop until we do not create new objects anymore + # which is equal to all objects have been run + touch "$__cdist_objects_created" + while [ -f "$__cdist_objects_created" ]; do + # Assume we're done after this run + rm "$__cdist_objects_created" + + # Get listing of objects + __cdist_object_list "$__cdist_out_object_dir" > \ + "$__cdist_object_all_object_all_objects" + + # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP + while read __cdist_object_all_object; do + set -- "$@" "$__cdist_object_all_object" + done < "$__cdist_object_all_object_all_objects" + + while [ $# -gt 0 ]; do + __cdist_object_all_object="$1"; shift + $__cdist_object_all_object_all_command "$__cdist_object_all_object" + done + done +} diff --git a/core/__cdist_object_code_run b/core/__cdist_object_code_run new file mode 100755 index 00000000..8af67ab8 --- /dev/null +++ b/core/__cdist_object_code_run @@ -0,0 +1,54 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Exec the code for the given object locally and remote +# + +__cdist_object_code_run() +{ + [ $# -eq 1 ] || __cdist_exit_err "" + + + if [ ! -d "$(__cdist_object_dir "$1")" ]; then + __cdist_exit_err "Object undefined" + fi + + # Code local + export __cdist_out_object_dir="$__cdist_out_object_dir" + __cdist_echo debug "Trying to run local code" + if __cdist_is_executable \ + "$(__cdist_object_code "$1" "${__cdist_name_gencode_local}")"; then + __cdist_run_shell \ + "$(__cdist_object_code "$1" "${__cdist_name_gencode_local}")" + else + __cdist_echo debug "Local code: none" + fi + + # Code remote + __cdist_echo debug "Trying to run remote code" + if __cdist_is_executable \ + "$(__cdist_object_code "$1" "${__cdist_name_gencode_remote}")"; then + + __cdist_run_remote $(__cdist_remote_object_code "$1") + else + __cdist_echo debug "Remote code: none" + fi +} diff --git a/core/__cdist_object_explorer_run b/core/__cdist_object_explorer_run new file mode 100755 index 00000000..da59d6c3 --- /dev/null +++ b/core/__cdist_object_explorer_run @@ -0,0 +1,89 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run the explorers for the given object on the target host. +# + +# FIXME: many cleanups needed before going production! + +__cdist_object_explorer_run() +{ + __cdist_object_self="$1"; shift + + __cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" + __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" + + # Check if type of object has >= 1 explorer + __cdist_has_explorer="$(__cdist_dir_listing "$(__cdist_type_explorer_dir "$__cdist_type")" | wc -l)" + # Run the type explorers for the current object if any + if [ "$__cdist_has_explorer" -ge 1 ]; then + if ! __cdist_type_explorer_pushed "$__cdist_type"; then + # FIXME: variables! + src_dir="$(__cdist_type_explorer_dir "$__cdist_type")" + dst_dir="$(__cdist_remote_type_explorer_dir "$__cdist_type")" + __cdist_echo info "Transfering explorers for $__cdist_type " + __cdist_dir push "$src_dir" "$dst_dir" + __cdist_type_explorer_pushed_add "$__cdist_type" + fi + + __cdist_echo info "Running explorers" + # Copy object parameters + __cdist_dir push \ + "$(__cdist_object_parameter_dir "$__cdist_object_self")" \ + "$(__cdist_remote_object_parameter_dir "$__cdist_object_self")" + + # Execute explorers + # FIXME: STOPPED: + # - remove cdist-remote-explorer-run + # - problem: new variables / need to run explorer directly? + # -> or put logic into __cdist_explorer_run + # -> think about having _one_ wrapper script for remote to execute + # shell functions + + # Create remote output directory + __cdist_run_remote mkdir -p "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" + + cd "$(__cdist_type_explorer_dir "$__cdist_type")" + + + for __cdist_object_explorer_run_explorer in *; do + __cdist_run_remote \ + "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \ + "$__cdist_name_var_object_id=\"$__cdist_object_id\"" \ + "$__cdist_name_var_self=\"$__cdist_object_self\"" \ + "$(__cdist_remote_type_explorer_dir "$__cdist_type")/$__cdist_object_explorer_run_explorer" \ + ">" "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")/$__cdist_object_explorer_run_explorer" + done + +# __cdist_run_remote \ +# "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \ +# "$__cdist_name_var_object_id=\"$__cdist_object_id\"" \ +# "$__cdist_name_var_self=\"$__cdist_object_self\"" \ +# cdist-remote-explorer-run \ +# "$__cdist_name_var_type_explorer" \ +# "$(__cdist_remote_type_explorer_dir "$__cdist_type")" \ +# "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" + + # Copy back results + __cdist_dir pull "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" \ + "$(__cdist_object_type_explorer_dir "$__cdist_object_self")" + fi +} diff --git a/core/__cdist_object_gencode b/core/__cdist_object_gencode new file mode 100755 index 00000000..08ef8b7d --- /dev/null +++ b/core/__cdist_object_gencode @@ -0,0 +1,66 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Generate code from one object (object must be relative path!) +# WARNING: OUTPUT ON STDOUT, ERRORS NEED TO BE ON STDERR! +# + +# FIXME: check variable names: +# either prefix or use global or use functions directly +# functions looks good, they are cheap anyway! + +__cdist_object_gencode() +{ + [ $# -eq 2 ] || __cdist_usage "" "" + + __cdist_object_self="$1"; shift + __cdist_gencode_type="$1"; shift + + __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" + __cdist_type_gencode="$(__cdist_type_gencode "$__cdist_type" "$__cdist_gencode_type")" + __cdist_code_output="$(__cdist_object_code "$__cdist_object_self" "$__cdist_gencode_type")" + + # export variables for the gencode script + export __object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" + export __object="$(__cdist_object_dir "$__cdist_object_self")" + export __global="$__cdist_out_dir" + + if [ -x "$__cdist_type_gencode" ]; then + __cdist_run_shell "$__cdist_type_gencode" > "$__cdist_tmp_file" + else + if [ -e "$__cdist_type_gencode" ]; then + __cdist_exit_err "$__cdist_type_gencode exists, but is not executable" + fi + + # Ensure it's empty, if there is no gencode + : > "$__cdist_tmp_file" + fi + + # Only create code, if gencode created output + if [ "$(wc -l < "$__cdist_tmp_file")" -gt 0 ]; then + cat - "$__cdist_tmp_file" << eof > "$__cdist_code_output" +# +# The following code was generated by $__cdist_type_gencode +# + +eof + chmod u+x "${__cdist_code_output}" + fi +} diff --git a/core/__cdist_object_gencode_run b/core/__cdist_object_gencode_run new file mode 100755 index 00000000..308f5f33 --- /dev/null +++ b/core/__cdist_object_gencode_run @@ -0,0 +1,36 @@ +#!/bin/sh +# +# 2010 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# For the given object create the code to be executed on the target. +# + +__cdist_object_gencode_run() +{ + __cdist_object_gencode_run_object="$1"; shift + + __cdist_echo info "Generating local code " + __cdist_object_gencode "$__cdist_object_gencode_run_object" \ + "${__cdist_name_gencode_local}" + + __cdist_echo info "Generating remote code " + __cdist_object_gencode "$__cdist_object_gencode_run_object" \ + "${__cdist_name_gencode_remote}" +} diff --git a/core/__cdist_object_list b/core/__cdist_object_list new file mode 100755 index 00000000..f2785c30 --- /dev/null +++ b/core/__cdist_object_list @@ -0,0 +1,36 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Print error and exit (perror() alike) +# + +__cdist_object_list() +{ + # FIXME: no local in posix + local basedir="$1"; shift + + # Use subshell to prevent changing cwd in program + ( + cd "${basedir}" + + find . -name "$__cdist_name_dot_cdist" | \ + sed -e 's;^./;;' -e "s;/${__cdist_name_dot_cdist}\$;;" + ) +} diff --git a/core/__cdist_object_manifest_run b/core/__cdist_object_manifest_run new file mode 100755 index 00000000..efc85539 --- /dev/null +++ b/core/__cdist_object_manifest_run @@ -0,0 +1,59 @@ +#!/bin/sh +# +# 2010 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run the manifest for the given object. +# + + +__cdist_object_manifest_run() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_object_self="$1"; shift + + # FIXME: rename to __cdist_object_dir (everywhere!) + __cdist_cur_object_dir="$(__cdist_object_dir "$__cdist_object_self")" + __cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" + + __cdist_echo info "Checking manifest " + + __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" + __cdist_manifest="$(__cdist_type_manifest "$__cdist_type")" + + if [ -f "$__cdist_manifest" ]; then + if [ -x "$__cdist_manifest" ]; then + # Make __cdist_manifest available for cdist-type-emulator + export __cdist_manifest + + __cdist_echo info "Executing manifest " + export $__cdist_name_var_object="$__cdist_cur_object_dir" + export $__cdist_name_var_object_id="$__cdist_object_id" + export $__cdist_name_var_type="$(__cdist_type_dir "$__cdist_type")" + + __cdist_manifest_run "$__cdist_manifest" + + # Tell cdist-object-run-all that there may be new objects + touch "$__cdist_objects_created" + else + __cdist_exit_err "${__cdist_manifest} needs to be executable." + fi + fi +} diff --git a/core/__cdist_object_prepare b/core/__cdist_object_prepare new file mode 100755 index 00000000..24039be0 --- /dev/null +++ b/core/__cdist_object_prepare @@ -0,0 +1,47 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# For the given object: +# - run type explorers +# - run type manifest +# + +__cdist_object_prepare() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_object_self="$1"; shift + __cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")" + [ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined" + + # Export to non-core for use in manifest and gencode scripts + export $__cdist_name_var_self=$__cdist_object_self + + __cdist_object_prepared="$(__cdist_object_prepared "$__cdist_object_self")" + if [ ! -f "$__cdist_object_prepared" ]; then + __cdist_echo info "Preparing object" + __cdist_object_explorer_run "$__cdist_object_self" + __cdist_object_manifest_run "$__cdist_object_self" + + # Mark this object as prepared + touch "$__cdist_object_prepared" + fi +} diff --git a/core/__cdist_object_run b/core/__cdist_object_run new file mode 100755 index 00000000..d2c7df6e --- /dev/null +++ b/core/__cdist_object_run @@ -0,0 +1,74 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# For the given object: +# - run type explorers +# - run type manifest +# - generate code +# - copy object to target +# - execute code on target +# + +__cdist_object_run() +{ + [ $# -eq 1 ] || __cdist_usage "" + + __cdist_object_self="$1"; shift + __cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")" + [ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined" + + # Export to non-core for use in manifest and gencode scripts + export $__cdist_name_var_self=$__cdist_object_self + + # FIXME: BUG: should be named differently! + # FIXME: BUG: I can be called recursively! -> variables are probably already set / overwritten! + __cdist_object_finished="$(__cdist_object_finished "$__cdist_object_self")" + if [ ! -f "$__cdist_object_finished" ]; then + # Resolve dependencies, if any + __cdist_object_require="$(__cdist_object_require "$__cdist_object_self")" + if [ -f "$__cdist_object_require" ]; then + # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP + while read __cdist_requirement; do + set -- "$@" "$__cdist_requirement" + done < "$__cdist_object_require" + + while [ $# -gt 0 ]; do + __cdist_requirement="$1"; shift + __cdist_echo info "Resolving requirement $__cdist_requirement" + # FIXME: BUG: at this point, the other __cdist_object_run may have + # overwritten all our variables! + __cdist_object_run "$__cdist_requirement" + done + fi + + __cdist_echo debug "Before gencode" + __cdist_object_gencode_run "$__cdist_object_self" + __cdist_echo debug "Before push" + __cdist_dir push "$(__cdist_object_dir "$__cdist_object_self")" \ + "$(__cdist_remote_object_dir "$__cdist_object_self")" + __cdist_echo debug "Before run" + __cdist_object_code_run "$__cdist_object_self" + __cdist_echo debug "Object run done" + + # Mark this object as done + touch "$__cdist_object_finished" + fi +} diff --git a/core/__cdist_run b/core/__cdist_run new file mode 100755 index 00000000..8febe550 --- /dev/null +++ b/core/__cdist_run @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Exit if an error occurs running something +# + +__cdist_run() +{ + "$@" || __cdist_echo error "$1 exited non-zero, aborting." +} diff --git a/core/__cdist_run_remote b/core/__cdist_run_remote new file mode 100755 index 00000000..17074049 --- /dev/null +++ b/core/__cdist_run_remote @@ -0,0 +1,32 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Run a cdist binary on the remote side +# + +__cdist_run_remote() +{ + [ $# -ge 1 ] || __cdist_usage " [opts]" + + ssh "${__cdist_remote_user}@${__cdist_target_host}" \ + "export PATH=\"${__cdist_remote_bin_dir}:\$PATH\";" \ + "export __cdist_out_object_dir=\"$__cdist_remote_out_object_dir\";" \ + "$@" +} diff --git a/core/__cdist_run_shell b/core/__cdist_run_shell new file mode 100755 index 00000000..b6e0a57d --- /dev/null +++ b/core/__cdist_run_shell @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Exit if an error occurs when running a shell script +# + +__cdist_run_shell() +{ + # Prepend our path, so all cdist tools come before other tools + PATH="${__cdist_out_type_bin_dir}:$PATH" sh -e "$@" + if [ "$?" -ne 0 ]; then + __cdist_echo error "$1 exited non-zero" + __cdist_echo warn "Faulty code:" + cat "$1" + __cdist_exit_err "Aborting due to non-zero exit code." + fi +} diff --git a/core/__cdist_tmp_removal b/core/__cdist_tmp_removal new file mode 100755 index 00000000..74d74936 --- /dev/null +++ b/core/__cdist_tmp_removal @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Remove tmp dir +# + +__cdist_tmp_removal() +{ + rm -rf "${__cdist_tmp_dir}" +} diff --git a/core/__cdist_type_build_emulation b/core/__cdist_type_build_emulation new file mode 100755 index 00000000..3c7270ca --- /dev/null +++ b/core/__cdist_type_build_emulation @@ -0,0 +1,49 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Build pseudo binaries for type emulation +# + +__cdist_type_build_emulation() +{ + [ $# -eq 0 ] || __cdist_usage "No arguments" + + [ -f "${__cdist_out_type_bin_dir}/.marker" ] && return 0 + + __cdist_type_emulator="$__cdist_abs_mydir/cdist-type-emulator" + + if [ ! -d "${__cdist_type_dir}" ]; then + __cdist_exit_err "$__cdist_type_dir must exist and contain available types" + fi + + # Get Types + ( + cd "${__cdist_type_dir}" + ls -1 > "${__cdist_tmp_file}" + ) + + # Create binaries + mkdir -p "${__cdist_out_type_bin_dir}" + while read __cdist_type_build_emulation_type; do + ln -sf "${__cdist_type_emulator}" \ + "${__cdist_out_type_bin_dir}/${__cdist_type_build_emulation_type}" + done < "${__cdist_tmp_file}" + + touch "${__cdist_out_type_bin_dir}/.marker" +} diff --git a/core/__cdist_usage b/core/__cdist_usage new file mode 100755 index 00000000..9dfa30e4 --- /dev/null +++ b/core/__cdist_usage @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Print error and exit (perror() alike) +# + +__cdist_usage() +{ + __cdist_exit_err "$__cdist_myname: $@" +} From 34bb39193707b4f8a6e84db36bcc38801fa3f954 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:01:25 +0200 Subject: [PATCH 021/166] +header Signed-off-by: Nico Schottelius --- bin/cdist | 425 ++---------------------------------------------------- 1 file changed, 13 insertions(+), 412 deletions(-) mode change 100644 => 100755 bin/cdist diff --git a/bin/cdist b/bin/cdist old mode 100644 new mode 100755 index a70a11f8..31d7411b --- a/bin/cdist +++ b/bin/cdist @@ -1,4 +1,4 @@ -#!python3 +#!/usr/bin/env python3 # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) # @@ -19,420 +19,21 @@ # # -__cdist_version="1.7.0" +import sys # argv -# Fail if something bogus is going on -set -u +__cdist_version="2.0.0" -################################################################################ -# cconf standard vars prefixed with cdist +def cdist_echo(type, *args): + """Ignore type for now, support later""" + print(*args) -__cdist_pwd="$(pwd -P)" -__cdist_mydir="${0%/*}"; -__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)" -__cdist_myname=${0##*/}; -__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" +def cdist_deploy_to(hostname): + """Mimic the old deploy to: Deploy to one host""" + cdist_echo("info", "Deploying to host", hostname) -################################################################################ -# Names / Constants -# -# Most values can be overriden from outside, so you can -# customise paths as you like (for distributors, geeks and hackers) -# -: ${__cdist_name_bin:=bin} -: ${__cdist_name_cache:=cache} -: ${__cdist_name_code:=code} -: ${__cdist_name_conf_dir:=conf} -: ${__cdist_name_dot_cdist:=.cdist} -: ${__cdist_name_explorer:=explorer} -: ${__cdist_name_gencode:=gencode} -: ${__cdist_name_gencode_local:=local} -: ${__cdist_name_gencode_remote:=remote} -: ${__cdist_name_global:=global} -: ${__cdist_name_host:=host} -: ${__cdist_name_init:=init} -: ${__cdist_name_manifest:=manifest} -: ${__cdist_name_object:=object} -: ${__cdist_name_object_finished:=done} -: ${__cdist_name_object_prepared:=prepared} -: ${__cdist_name_object_id:=object_id} -: ${__cdist_name_object_source:=source} -: ${__cdist_name_objects_created:=.objects_created} -: ${__cdist_name_out_dir:=out} -: ${__cdist_name_parameter:=parameter} -: ${__cdist_name_parameter_required:=required} -: ${__cdist_name_parameter_optional:=optional} -: ${__cdist_name_require:=require} -: ${__cdist_name_self:=self} -: ${__cdist_name_singleton:=singleton} -: ${__cdist_name_target_host:=target_host} -: ${__cdist_name_target_user:=target_user} -: ${__cdist_name_type:=type} -: ${__cdist_name_type_bin:=type_bin} -: ${__cdist_name_type_explorer:=type_explorer} -: ${__cdist_name_type_explorer_pushed:=.explorer_pushed} +if __name__ == "__main__": + hostname=sys.argv[1] + cdist_echo("info", "cdist", __cdist_version, ": Configuring host", hostname) + cdist_deploy_to(hostname) -# Used for IDs: Allow everything not starting with - and . -: ${__cdist_sane_regexp:=[^-\.].*} - -# Default remote user -: ${__cdist_remote_user:=root} - - -################################################################################ -# Exported variable names (usable for non core -# -: ${__cdist_name_var_explorer:=__$__cdist_name_explorer} -: ${__cdist_name_var_type_explorer:=__$__cdist_name_type_explorer} -: ${__cdist_name_var_global:=__$__cdist_name_global} -: ${__cdist_name_var_manifest:=__$__cdist_name_manifest} -: ${__cdist_name_var_target_host:=__$__cdist_name_target_host} -: ${__cdist_name_var_target_user:=__$__cdist_name_target_user} -: ${__cdist_name_var_object:=__$__cdist_name_object} -: ${__cdist_name_var_object_id:=__$__cdist_name_object_id} -: ${__cdist_name_var_self:=__$__cdist_name_self} -: ${__cdist_name_var_type:=__$__cdist_name_type} - - -################################################################################ -# Tempfiles -# -: ${__cdist_tmp_base_dir=/tmp} -__cdist_tmp_dir=$(mktemp -d "$__cdist_tmp_base_dir/cdist.XXXXXXXXXXXX") -__cdist_tmp_file=$(mktemp "$__cdist_tmp_dir/cdist.XXXXXXXXXXXX") - -################################################################################ -# Local Base -# -: ${__cdist_local_base_dir:=$__cdist_tmp_dir} - -# Cache may *NOT* be below __cdist_local_base_dir! -: ${__cdist_local_base_cache_dir:=$__cdist_abs_mydir/../$__cdist_name_cache} - -: ${__cdist_conf_dir:="$(cd "$__cdist_abs_mydir/../conf" && pwd -P)"} - -: ${__cdist_explorer_dir:=$__cdist_conf_dir/$__cdist_name_explorer} -: ${__cdist_manifest_dir:=$__cdist_conf_dir/$__cdist_name_manifest} -: ${__cdist_manifest_init:=$__cdist_manifest_dir/$__cdist_name_init} -: ${__cdist_type_dir:=$__cdist_conf_dir/$__cdist_name_type} - -################################################################################ -# Local output -# -: ${__cdist_out_dir:=$__cdist_local_base_dir/$__cdist_name_out_dir} -: ${__cdist_out_explorer_dir:=$__cdist_out_dir/$__cdist_name_explorer} -: ${__cdist_out_object_dir:=$__cdist_out_dir/$__cdist_name_object} -: ${__cdist_out_type_dir:=$__cdist_out_dir/$__cdist_name_type} -: ${__cdist_out_type_bin_dir:=$__cdist_out_dir/$__cdist_name_type_bin} - -: ${__cdist_objects_created:=$__cdist_out_object_dir/$__cdist_name_objects_created} - -################################################################################ -# Remote base -# -: ${__cdist_remote_base_dir:=/var/lib/cdist} -: ${__cdist_remote_bin_dir:=$__cdist_remote_base_dir/$__cdist_name_bin} -: ${__cdist_remote_conf_dir:=$__cdist_remote_base_dir/$__cdist_name_conf_dir} - -: ${__cdist_remote_explorer_dir:=$__cdist_remote_conf_dir/$__cdist_name_explorer} -: ${__cdist_remote_type_dir:=$__cdist_remote_conf_dir/$__cdist_name_type} - -################################################################################ -# Remote output -# -: ${__cdist_remote_out_dir:=$__cdist_remote_base_dir/$__cdist_name_out_dir} -: ${__cdist_remote_out_explorer_dir:=$__cdist_remote_out_dir/$__cdist_name_explorer} -: ${__cdist_remote_out_object_dir:=$__cdist_remote_out_dir/$__cdist_name_object} - - -################################################################################ -# Internal functions -# -__cdist_echo() -{ - __cdist_echo_type="$1"; shift - - set +u - if [ "$__cdist_object_self" ]; then - __cdist_echo_prefix="${__cdist_object_self}:" - else - __cdist_echo_prefix="core: " - fi - set -u - - case "$__cdist_echo_type" in - debug) - set +u - if [ "$__cdist_debug" ]; then - echo $__cdist_echo_prefix "Debug: $@" - fi - set -u - ;; - info) - echo $__cdist_echo_prefix "$@" - ;; - warn) - echo $__cdist_echo_prefix "Warning: $@" - ;; - error) - echo $__cdist_echo_prefix "Error: $@" >&2 - ;; - *) - echo "CORE BUG, who created the broken commit in $0?" >&2 - exit 23 - ;; - esac -} - -__cdist_exec_fail_on_error() -{ - set +e - sh -e "$@" - if [ "$?" -ne 0 ]; then - __cdist_echo error "$1 exited non-zero" - __cdist_echo warn "Faulty code:" - cat "$1" - __cdist_exit_err "Aborting due to non-zero exit code." - fi -} - -__cdist_exit_err() -{ - __cdist_echo error "$@" - exit 1 -} - -__cdist_usage() -{ - __cdist_exit_err "$__cdist_myname: $@" -} - -__cdist_init_deploy() -{ - __cdist_echo info "Creating clean directory structure " - - # Ensure there is no old stuff, neither local nor remote - rm -rf "$__cdist_local_base_dir" - ssh "${__cdist_remote_user}@$1" "rm -rf ${__cdist_remote_base_dir}" - - # Init base - mkdir -p "$__cdist_local_base_dir" - ssh "${__cdist_remote_user}@$1" "mkdir -p ${__cdist_remote_base_dir}" - - # Link configuration source directory - consistent with remote - ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" -} - -################################################################################ -# Cache -# -__cdist_cache_dir() -{ - cd "${__cdist_local_base_cache_dir}" && pwd -P -} - -__cdist_host_cache_dir() -{ - echo "$(__cdist_cache_dir)/$1" -} - -################################################################################ -# Object -# - -__cdist_object_code() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_code}-$2" -} - -__cdist_object_prepared() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_object_prepared}" -} - -__cdist_object_finished() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_object_finished}" -} - -__cdist_object_dir() -{ - echo "$(__cdist_object_base_dir "$1")/${__cdist_name_dot_cdist}" -} - -__cdist_object_base_dir() -{ - echo "${__cdist_out_object_dir}/$1" -} - - -__cdist_object_id_from_object() -{ - echo "${1#*/}" -} - -# Find objects, remove ./ and /MARKER -__cdist_object_list() -{ - local basedir="$1"; shift - - # Use subshell to prevent changing cwd in program - ( - cd "${basedir}" - - find . -name "$__cdist_name_dot_cdist" | \ - sed -e 's;^./;;' -e "s;/${__cdist_name_dot_cdist}\$;;" - ) -} - -__cdist_object_parameter_dir() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_parameter}" -} - -__cdist_object_require() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_require}" -} - -__cdist_object_source_name() -{ - echo "$1/${__cdist_name_object_source}" -} - -__cdist_object_source() -{ - cat "$(__cdist_object_source_name "$1")" -} - -__cdist_object_source_add() -{ - echo "$__cdist_manifest" >> "$(__cdist_object_source_name "$1")" -} - -__cdist_object_type_explorer_dir() -{ - echo "$(__cdist_object_dir "$1")/${__cdist_name_explorer}" -} - -################################################################################ -# Remote -# - -__cdist_remote_object_base_dir() -{ - echo "${__cdist_remote_out_object_dir}/$1" -} - -__cdist_remote_object_dir() -{ - echo "$(__cdist_remote_object_base_dir "$1")/${__cdist_name_dot_cdist}" -} - -__cdist_remote_object_parameter_dir() -{ - echo "$(__cdist_remote_object_dir "$1")/${__cdist_name_parameter}" -} - -__cdist_remote_object_type_explorer_dir() -{ - echo "$(__cdist_remote_object_dir "$1")/${__cdist_name_explorer}" -} - - -__cdist_remote_type_explorer_dir() -{ - echo "${__cdist_remote_type_dir}/$1/${__cdist_name_explorer}" -} - - -################################################################################ -# Traps -# -__cdist_tmp_removal() -{ - rm -rf "${__cdist_tmp_dir}" -} - -# Does not work in children, will be called again in every script! -# Use only in interactive "front end" scripts -__cdist_kill_on_interrupt() -{ - __cdist_tmp_removal - kill 0 - exit 1 -} - -# Remove tempfiles at normal exit -trap __cdist_tmp_removal EXIT - - -################################################################################ -# Type -# -__cdist_type_dir() -{ - echo "${__cdist_type_dir}/$1" -} - -__cdist_type_explorer_dir() -{ - echo "${__cdist_type_dir}/$1/${__cdist_name_explorer}" -} - -__cdist_type_from_object() -{ - echo "${1%%/*}" -} - -__cdist_type_has_explorer() -{ - # We only create output, if there's at least one explorer - # and can thus be used as a boolean ;-) - if [ -d "$(__cdist_type_explorer_dir "$1")" ]; then - ls -1 "$(__cdist_type_explorer_dir "$1")" - fi -} - -__cdist_type_explorer_pushed() -{ - [ -f "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" ] \ - && grep -q "$1" "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" -} - -__cdist_type_explorer_pushed_add() -{ - [ -d "$__cdist_out_type_dir" ] || mkdir "$__cdist_out_type_dir" - echo "$1" >> "${__cdist_out_type_dir}/${__cdist_name_type_explorer_pushed}" -} - -__cdist_type_gencode() -{ - echo "${__cdist_type_dir}/$1/${__cdist_name_gencode}-$2" -} - -__cdist_type_manifest() -{ - echo "${__cdist_type_dir}/$1/${__cdist_name_manifest}" -} - -__cdist_type_parameter_dir() -{ - echo "$(__cdist_type_dir "$1")/${__cdist_name_parameter}" -} - -__cdist_type_parameter_optional() -{ - echo "$(__cdist_type_parameter_dir "$1")/$__cdist_name_parameter_optional" -} - -__cdist_type_parameter_required() -{ - echo "$(__cdist_type_parameter_dir "$1")/$__cdist_name_parameter_required" -} - -__cdist_type_singleton() -{ - echo "${__cdist_type_dir}/$1/${__cdist_name_singleton}" -} From 4f7fa6b8924ec1dc8651382a8d35fe532075f39e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:10:37 +0200 Subject: [PATCH 022/166] +/- todo Signed-off-by: Nico Schottelius --- core/__cdist_init_deploy | 40 ---------------------------------------- doc/dev/todo/niconext | 28 ++-------------------------- 2 files changed, 2 insertions(+), 66 deletions(-) delete mode 100755 core/__cdist_init_deploy diff --git a/core/__cdist_init_deploy b/core/__cdist_init_deploy deleted file mode 100755 index 4ac3168c..00000000 --- a/core/__cdist_init_deploy +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Prepare deployment -# - -__cdist_init_deploy() -{ - __cdist_echo info "Creating clean directory structure " - - # Ensure there is no old stuff, neither local nor remote - rm -rf "$__cdist_local_base_dir" - ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "rm -rf ${__cdist_remote_base_dir}" - - # Init base - mkdir -p "$__cdist_local_base_dir" - ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "mkdir -p ${__cdist_remote_base_dir}" - - # Link configuration source directory - consistent with remote - ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" -} diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 28762ef4..a59f909f 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,26 +1,2 @@ -Catch broken instances in cdist-mass-deploy -p and report broken deployements at the end! - - --------------------------------------------------------------------------------- -Bug with requirements when indirect requires is scheduled late: - -__package/collectd: Generating local code -__package/collectd: Generating remote code -__package/collectd: Transferring object -__package/collectd: Checking code-local -__package/collectd: Checking code-remote -__file/etc/collectd/collectd.conf: Generating local code -__file/etc/collectd/collectd.conf: Generating remote code -__file/etc/collectd/collectd.conf: Transferring object -__file/etc/collectd/collectd.conf: Checking code-local -__file/etc/collectd/collectd.conf: Executing code-local -scp: /etc/collectd/collectd.conf: No such file or directory -__file/etc/collectd/collectd.conf: Error: /tmp/cdist.PYKFWmj9QknE/out/object/__file/etc/collectd/collectd.conf/.cdist/code-local exited non-zero -__file/etc/collectd/collectd.conf: Warning: Faulty code: -# -# The following code was generated by /home/users/nico/oeffentlich/rechner/projekte/cdist-nutzung/conf/type/__file/gencode-local -# - -scp /home/users/nico/oeffentlich/rechner/projekte/cdist-nutzung/conf/type/__ethz_collectd/files/collectd.conf.client root@shrek08:/etc/collectd/collectd.conf -__file/etc/collectd/collectd.conf: Error: Aborting due to non-zero exit code. - +- rewrite in python? +- support non-ssh access? From 9711a5612d359919ff0a254042c4fec4ba17f1ca Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:21:15 +0200 Subject: [PATCH 023/166] partly implement run_or_fail, init_deploy Signed-off-by: Nico Schottelius --- bin/cdist | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 31d7411b..233e6fee 100755 --- a/bin/cdist +++ b/bin/cdist @@ -20,20 +20,51 @@ # import sys # argv +import subprocess # execute stuff __cdist_version="2.0.0" -def cdist_echo(type, *args): +def logger(type, *args): """Ignore type for now, support later""" print(*args) +def exit_error(*args): + logger(*args) + sys.exit(1) + + +def run_or_fail(*args): + try: + subprocess.check_call(*args) + except CalledProcessError: + exit_error("Command failed:", *args) + + def cdist_deploy_to(hostname): """Mimic the old deploy to: Deploy to one host""" - cdist_echo("info", "Deploying to host", hostname) + logger("info", "Deploying to host", hostname) + + +def init_deploy(): + logger("info", "Creating clean directory structure") + + # Ensure there is no old stuff, neither local nor remote + run_or_fail(["echo rm -rf", "$__cdist_local_base_dir"]) + +# ssh "${__cdist_remote_user}@${__cdist_target_host}" \ +# "rm -rf ${__cdist_remote_base_dir}" +# +# # Init base +# mkdir -p "$__cdist_local_base_dir" +# ssh "${__cdist_remote_user}@${__cdist_target_host}" \ +# "mkdir -p ${__cdist_remote_base_dir}" +# +# # Link configuraion source directory - consistent with remote +# ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" if __name__ == "__main__": hostname=sys.argv[1] - cdist_echo("info", "cdist", __cdist_version, ": Configuring host", hostname) + logger("info", "cdist", __cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) From a90751dcff360e6ba2ceb5902ba90f43bdc6686e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:36:22 +0200 Subject: [PATCH 024/166] begin to implmenent remote execution Signed-off-by: Nico Schottelius --- bin/cdist | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 233e6fee..d680eaa6 100755 --- a/bin/cdist +++ b/bin/cdist @@ -34,25 +34,34 @@ def exit_error(*args): def run_or_fail(*args): + newargs = ["echo"] + newargs.extend(*args) + try: - subprocess.check_call(*args) + subprocess.check_call(newargs) except CalledProcessError: - exit_error("Command failed:", *args) + exit_error("Command failed:", newargs) + +def remote_run_or_fail(hostname, *args): + newargs = ["ssh", hostname] + newargs.extend(*args) + + run_or_fail(newargs) def cdist_deploy_to(hostname): """Mimic the old deploy to: Deploy to one host""" logger("info", "Deploying to host", hostname) + init_deploy(hostname) -def init_deploy(): +def init_deploy(hostname): logger("info", "Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote - run_or_fail(["echo rm -rf", "$__cdist_local_base_dir"]) + run_or_fail(["echo", "rm -rf", "$__cdist_local_base_dir"]) -# ssh "${__cdist_remote_user}@${__cdist_target_host}" \ -# "rm -rf ${__cdist_remote_base_dir}" + remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) # # # Init base # mkdir -p "$__cdist_local_base_dir" From 89964b32dd70c587f2b94ef9a4e027680ffb1274 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:41:54 +0200 Subject: [PATCH 025/166] in theory, init_deploy is done Signed-off-by: Nico Schottelius --- bin/cdist | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bin/cdist b/bin/cdist index d680eaa6..2ff2e011 100755 --- a/bin/cdist +++ b/bin/cdist @@ -59,17 +59,16 @@ def init_deploy(hostname): logger("info", "Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote - run_or_fail(["echo", "rm -rf", "$__cdist_local_base_dir"]) + run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) -# -# # Init base -# mkdir -p "$__cdist_local_base_dir" -# ssh "${__cdist_remote_user}@${__cdist_target_host}" \ -# "mkdir -p ${__cdist_remote_base_dir}" -# -# # Link configuraion source directory - consistent with remote -# ln -sf "$__cdist_conf_dir" "$__cdist_local_base_dir/$__cdist_name_conf_dir" + + # Create base directories + run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) + remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) + + # Link configuraion source directory - consistent with remote + run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) if __name__ == "__main__": From ffd7f4b251bb4a61239b6a5afa274a6d5061e70c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 01:48:23 +0200 Subject: [PATCH 026/166] --todo Signed-off-by: Nico Schottelius --- bin/cdist-deploy-to | 7 ------- 1 file changed, 7 deletions(-) diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to index bf5614bc..84181bbd 100755 --- a/bin/cdist-deploy-to +++ b/bin/cdist-deploy-to @@ -43,13 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " # See cdist-stages(7) # -# Prepare local and remote directories -__cdist_init_deploy "$__cdist_target_host" - -# Transfer cdist executables -__cdist_echo info "Transferring cdist binaries to the target host " -cdist-dir push "$__cdist_target_host" \ - "${__cdist_abs_mydir}" "${__cdist_remote_bin_dir}" cdist-explorer-run-global "$__cdist_target_host" cdist-manifest-run-init "$__cdist_target_host" cdist-object-all "$__cdist_target_host" cdist-object-prepare From c2873a8fa0b132003d1cd14b5bc53a6537360c71 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:12:48 +0200 Subject: [PATCH 027/166] ipmlement base_directory Signed-off-by: Nico Schottelius --- bin/cdist | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bin/cdist b/bin/cdist index 2ff2e011..440ec459 100755 --- a/bin/cdist +++ b/bin/cdist @@ -21,6 +21,7 @@ import sys # argv import subprocess # execute stuff +import os __cdist_version="2.0.0" @@ -54,6 +55,20 @@ def cdist_deploy_to(hostname): logger("info", "Deploying to host", hostname) init_deploy(hostname) +def base_directory(): + """Returns the directory in which all cdist stuff is based in""" + os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) + return os.getcwd() + +def global_explorer_directory(): + """Returns path to directory containing the global explorers""" + +def list_global_explorers(): + """Return list of available explorers""" + os.listdir(path=global_explorer_directory()) + +def explore(hostname, type=''): + """Run explorers""" def init_deploy(hostname): logger("info", "Creating clean directory structure") @@ -75,4 +90,5 @@ if __name__ == "__main__": hostname=sys.argv[1] logger("info", "cdist", __cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) + print(base_directory()) From 11750515e1bc1bbc821082bac1c901c4eac66ec6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:16:13 +0200 Subject: [PATCH 028/166] finish global_explorer_directory() Signed-off-by: Nico Schottelius --- bin/cdist | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 440ec459..2246f943 100755 --- a/bin/cdist +++ b/bin/cdist @@ -60,8 +60,13 @@ def base_directory(): os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) return os.getcwd() +def conf_directory(): + """Returns path to main configuration directory""" + return os.path.join(base_directory(), "conf") + def global_explorer_directory(): """Returns path to directory containing the global explorers""" + return os.path.join(conf_directory(), "explorer") def list_global_explorers(): """Return list of available explorers""" @@ -90,5 +95,5 @@ if __name__ == "__main__": hostname=sys.argv[1] logger("info", "cdist", __cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) - print(base_directory()) + print(global_explorer_directory()) From 09c58e2327281d91354460012e94f8ab455a8ed0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:39:24 +0200 Subject: [PATCH 029/166] exit, if there are no global explorers Signed-off-by: Nico Schottelius --- bin/cdist | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 2246f943..1019726a 100755 --- a/bin/cdist +++ b/bin/cdist @@ -30,7 +30,7 @@ def logger(type, *args): print(*args) def exit_error(*args): - logger(*args) + logger("error", *args) sys.exit(1) @@ -50,13 +50,9 @@ def remote_run_or_fail(hostname, *args): run_or_fail(newargs) -def cdist_deploy_to(hostname): - """Mimic the old deploy to: Deploy to one host""" - logger("info", "Deploying to host", hostname) - init_deploy(hostname) - def base_directory(): """Returns the directory in which all cdist stuff is based in""" + print("Going to", __file__, os.path.join(os.path.dirname(__file__), os.pardir)) os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) return os.getcwd() @@ -70,10 +66,14 @@ def global_explorer_directory(): def list_global_explorers(): """Return list of available explorers""" - os.listdir(path=global_explorer_directory()) + return os.listdir(global_explorer_directory()) -def explore(hostname, type=''): - """Run explorers""" +def global_explore(hostname): + """Run global explorers""" + explorer = list_global_explorers() + if(len(explorer) == 0): + exit_error("No explorers found in", global_explorer_directory()) + def init_deploy(hostname): logger("info", "Creating clean directory structure") @@ -90,10 +90,16 @@ def init_deploy(hostname): # Link configuraion source directory - consistent with remote run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) +def cdist_deploy_to(hostname): + """Mimic the old deploy to: Deploy to one host""" + logger("info", "Deploying to host", hostname) + init_deploy(hostname) + global_explore(hostname) + if __name__ == "__main__": hostname=sys.argv[1] logger("info", "cdist", __cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) - print(global_explorer_directory()) + print(list_global_explorers()) From 6e3c228a6f2c1b4603e29bf682608015cefab266 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:46:09 +0200 Subject: [PATCH 030/166] transfer_dir() added Signed-off-by: Nico Schottelius --- bin/cdist | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 1019726a..767d2fe9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -36,6 +36,7 @@ def exit_error(*args): def run_or_fail(*args): newargs = ["echo"] + newargs = [] newargs.extend(*args) try: @@ -49,6 +50,8 @@ def remote_run_or_fail(hostname, *args): run_or_fail(newargs) +def transfer_dir(hostname, source, destination): + run_or_fail(["scp", "-r", source, hostname + ":" + destination]) def base_directory(): """Returns the directory in which all cdist stuff is based in""" @@ -64,16 +67,22 @@ def global_explorer_directory(): """Returns path to directory containing the global explorers""" return os.path.join(conf_directory(), "explorer") +def remote_global_explorer_directory(): + """Returns path to directory containing the global explorers""" + return os.path.join(conf_directory(), "explorer") + def list_global_explorers(): """Return list of available explorers""" return os.listdir(global_explorer_directory()) +def transfer_global_explorers(hostname): + transfer_dir(hostname, global_explorer_directory(), remote_global_explorer_directory()) + def global_explore(hostname): """Run global explorers""" explorer = list_global_explorers() if(len(explorer) == 0): exit_error("No explorers found in", global_explorer_directory()) - def init_deploy(hostname): logger("info", "Creating clean directory structure") From 94fc5b3c911de461e7437518433257e3073e431c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:51:45 +0200 Subject: [PATCH 031/166] add remove_remote_dir() Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/cdist b/bin/cdist index 767d2fe9..94154974 100755 --- a/bin/cdist +++ b/bin/cdist @@ -50,7 +50,11 @@ def remote_run_or_fail(hostname, *args): run_or_fail(newargs) +def remove_remote_dir(hostname, destination): + remote_run_or_fail(hostname, ["rm", "-rf", destination]) + def transfer_dir(hostname, source, destination): + remove_remote_dir(hostname, destination) run_or_fail(["scp", "-r", source, hostname + ":" + destination]) def base_directory(): From 78a5bbf6e89fad6850f2f1d435c081918ee8dd35 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:52:38 +0200 Subject: [PATCH 032/166] remove the currently broken init() Signed-off-by: Nico Schottelius --- bin/cdist | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 94154974..fb09e8b2 100755 --- a/bin/cdist +++ b/bin/cdist @@ -92,16 +92,16 @@ def init_deploy(hostname): logger("info", "Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote - run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) - - remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) - - # Create base directories - run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) - remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) - - # Link configuraion source directory - consistent with remote - run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) +# run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) +# +# remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) +# +# # Create base directories +# run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) +# remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) +# +# # Link configuraion source directory - consistent with remote +# run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) def cdist_deploy_to(hostname): """Mimic the old deploy to: Deploy to one host""" From d0eeafd228fae25e1e8662311b7573022ab06be5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:56:02 +0200 Subject: [PATCH 033/166] also add some bugs Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++++++++-- doc/dev/todo/niconext | 2 ++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index fb09e8b2..5cabf17b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -63,17 +63,24 @@ def base_directory(): os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) return os.getcwd() +def remote_base_directory(): + return "/var/lib/cdist" + def conf_directory(): """Returns path to main configuration directory""" return os.path.join(base_directory(), "conf") +def remote_conf_directory(): + """Returns path to remote main configuration directory""" + return os.path.join(remote_base_directory(), "conf") + def global_explorer_directory(): """Returns path to directory containing the global explorers""" return os.path.join(conf_directory(), "explorer") def remote_global_explorer_directory(): - """Returns path to directory containing the global explorers""" - return os.path.join(conf_directory(), "explorer") + """Returns path to the remote directory containing the global explorers""" + return os.path.join(remote_conf_directory(), "explorer") def list_global_explorers(): """Return list of available explorers""" diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index a59f909f..fce4033e 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,2 +1,4 @@ - rewrite in python? - support non-ssh access? + +- Bug: os.path.join() may be wrong for the remote side! From ef925714d5fc4aed09981d2c6de1a2088a24c881 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 02:57:47 +0200 Subject: [PATCH 034/166] transfer explorer Signed-off-by: Nico Schottelius --- bin/cdist | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 5cabf17b..7d28d112 100755 --- a/bin/cdist +++ b/bin/cdist @@ -45,7 +45,7 @@ def run_or_fail(*args): exit_error("Command failed:", newargs) def remote_run_or_fail(hostname, *args): - newargs = ["ssh", hostname] + newargs = ["ssh", "root@" + hostname] newargs.extend(*args) run_or_fail(newargs) @@ -55,7 +55,7 @@ def remove_remote_dir(hostname, destination): def transfer_dir(hostname, source, destination): remove_remote_dir(hostname, destination) - run_or_fail(["scp", "-r", source, hostname + ":" + destination]) + run_or_fail(["scp", "-r", source, "root@" + hostname + ":" + destination]) def base_directory(): """Returns the directory in which all cdist stuff is based in""" @@ -95,6 +95,8 @@ def global_explore(hostname): if(len(explorer) == 0): exit_error("No explorers found in", global_explorer_directory()) + transfer_global_explorers(hostname) + def init_deploy(hostname): logger("info", "Creating clean directory structure") From 85df71c9fa511b583076d500df25ab6cf384dc81 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:23:18 +0200 Subject: [PATCH 035/166] executing explorers works Signed-off-by: Nico Schottelius --- bin/cdist | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7d28d112..d2a0abd9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -30,14 +30,15 @@ def logger(type, *args): print(*args) def exit_error(*args): - logger("error", *args) + logger("error", args) sys.exit(1) def run_or_fail(*args): - newargs = ["echo"] + # newargs = ["echo"] newargs = [] newargs.extend(*args) + print(newargs) try: subprocess.check_call(newargs) @@ -45,9 +46,9 @@ def run_or_fail(*args): exit_error("Command failed:", newargs) def remote_run_or_fail(hostname, *args): + """Run something on the remote side and fail is something breaks""" newargs = ["ssh", "root@" + hostname] newargs.extend(*args) - run_or_fail(newargs) def remove_remote_dir(hostname, destination): @@ -82,6 +83,10 @@ def remote_global_explorer_directory(): """Returns path to the remote directory containing the global explorers""" return os.path.join(remote_conf_directory(), "explorer") +def remote_global_explorer_path(explorer): + """Returns path to the remote explorer""" + return os.path.join(remote_global_explorer_directory(), explorer) + def list_global_explorers(): """Return list of available explorers""" return os.listdir(global_explorer_directory()) @@ -91,11 +96,14 @@ def transfer_global_explorers(hostname): def global_explore(hostname): """Run global explorers""" - explorer = list_global_explorers() - if(len(explorer) == 0): + explorers = list_global_explorers() + if(len(explorers) == 0): exit_error("No explorers found in", global_explorer_directory()) transfer_global_explorers(hostname) + for explorer in explorers: + remote_run_or_fail(hostname, [remote_global_explorer_path(explorer)]) + def init_deploy(hostname): logger("info", "Creating clean directory structure") From 9036f8c4cce9edb5b353928855bdd995248cdb16 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:24:27 +0200 Subject: [PATCH 036/166] executing explorers works Signed-off-by: Nico Schottelius --- bin/cdist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/cdist b/bin/cdist index d2a0abd9..339349a3 100755 --- a/bin/cdist +++ b/bin/cdist @@ -103,6 +103,8 @@ def global_explore(hostname): transfer_global_explorers(hostname) for explorer in explorers: remote_run_or_fail(hostname, [remote_global_explorer_path(explorer)]) + + remote_run_or_fail(hostname, [remote_global_explorer_path("moo")]) def init_deploy(hostname): From ff2d5629d65a3bafe90df40718fd6ab393055527 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:28:06 +0200 Subject: [PATCH 037/166] submit string on failed error Signed-off-by: Nico Schottelius --- bin/cdist | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 339349a3..ab913541 100755 --- a/bin/cdist +++ b/bin/cdist @@ -30,7 +30,7 @@ def logger(type, *args): print(*args) def exit_error(*args): - logger("error", args) + logger("error", *args) sys.exit(1) @@ -42,8 +42,8 @@ def run_or_fail(*args): try: subprocess.check_call(newargs) - except CalledProcessError: - exit_error("Command failed:", newargs) + except subprocess.CalledProcessError: + exit_error("Command failed:", " ".join(newargs)) def remote_run_or_fail(hostname, *args): """Run something on the remote side and fail is something breaks""" @@ -104,8 +104,6 @@ def global_explore(hostname): for explorer in explorers: remote_run_or_fail(hostname, [remote_global_explorer_path(explorer)]) - remote_run_or_fail(hostname, [remote_global_explorer_path("moo")]) - def init_deploy(hostname): logger("info", "Creating clean directory structure") From 7a7413f1f48c9509bdd4e390f3d86017bf010ab7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:31:58 +0200 Subject: [PATCH 038/166] pause Signed-off-by: Nico Schottelius --- bin/cdist | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bin/cdist b/bin/cdist index ab913541..59d99040 100755 --- a/bin/cdist +++ b/bin/cdist @@ -75,6 +75,11 @@ def remote_conf_directory(): """Returns path to remote main configuration directory""" return os.path.join(remote_base_directory(), "conf") +def out_dir(): + FIXME: stopped - probably need static temp know! + """Local directory containing output""" + return os.path.join(base_directory(), "conf") + def global_explorer_directory(): """Returns path to directory containing the global explorers""" return os.path.join(conf_directory(), "explorer") From 572401e4f8010574df45423ccb1c21f056b719b9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 03:56:59 +0200 Subject: [PATCH 039/166] commit broken stuff Signed-off-by: Nico Schottelius --- bin/cdist | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 59d99040..e1d62391 100755 --- a/bin/cdist +++ b/bin/cdist @@ -22,8 +22,22 @@ import sys # argv import subprocess # execute stuff import os +import tempfile +import shutil -__cdist_version="2.0.0" + +class Cdist: + """Cdist main class to hold arbitrary data""" + version="2.0.0" + + def __init__(self): + self.tempdir = tempfile.mkdtemp() + print(self.tempdir) + + def __del__(self): + print("Zerstoeren") + print(self.tempdir) + shutil.rmtree(self.tempdir) def logger(type, *args): """Ignore type for now, support later""" @@ -76,7 +90,7 @@ def remote_conf_directory(): return os.path.join(remote_base_directory(), "conf") def out_dir(): - FIXME: stopped - probably need static temp know! + # FIXME: stopped - probably need static temp know! """Local directory containing output""" return os.path.join(base_directory(), "conf") @@ -134,7 +148,9 @@ def cdist_deploy_to(hostname): if __name__ == "__main__": hostname=sys.argv[1] - logger("info", "cdist", __cdist_version, ": Configuring host", hostname) +# logger("info", "cdist", cdist_version, ": Configuring host", hostname) cdist_deploy_to(hostname) print(list_global_explorers()) + c = Cdist() + From e8a6d89e3c253897dd46863348a44eb606ad97cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 04:22:21 +0200 Subject: [PATCH 040/166] wrap into a class Signed-off-by: Nico Schottelius --- bin/cdist | 187 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 98 insertions(+), 89 deletions(-) diff --git a/bin/cdist b/bin/cdist index e1d62391..53728648 100755 --- a/bin/cdist +++ b/bin/cdist @@ -30,127 +30,136 @@ class Cdist: """Cdist main class to hold arbitrary data""" version="2.0.0" - def __init__(self): + def __init__(self, hostname): self.tempdir = tempfile.mkdtemp() + self.hostname = hostname + print(self.hostname) print(self.tempdir) - def __del__(self): - print("Zerstoeren") + def cleanup(self): + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # print(self.tempdir) shutil.rmtree(self.tempdir) -def logger(type, *args): - """Ignore type for now, support later""" - print(*args) + def out_dir(self): + # FIXME: stopped - probably need static temp know! + """Local directory containing output""" + return os.path.join(base_directory(), "conf") -def exit_error(*args): - logger("error", *args) - sys.exit(1) - + def logger(self,type, *args): + """Ignore type for now, support later""" + print(*args) -def run_or_fail(*args): - # newargs = ["echo"] - newargs = [] - newargs.extend(*args) - print(newargs) + def exit_error(self,*args): + self.logger("error", *args) + sys.exit(1) + - try: - subprocess.check_call(newargs) - except subprocess.CalledProcessError: - exit_error("Command failed:", " ".join(newargs)) + def run_or_fail(self,*args): + # newargs = ["echo"] + newargs = [] + newargs.extend(*args) + print(newargs) -def remote_run_or_fail(hostname, *args): - """Run something on the remote side and fail is something breaks""" - newargs = ["ssh", "root@" + hostname] - newargs.extend(*args) - run_or_fail(newargs) + try: + subprocess.check_call(newargs) + except subprocess.CalledProcessError: + self.exit_error("Command failed:", " ".join(newargs)) -def remove_remote_dir(hostname, destination): - remote_run_or_fail(hostname, ["rm", "-rf", destination]) + def remote_run_or_fail(self, *args): + """Run something on the remote side and fail is something breaks""" + newargs = ["ssh", "root@" + self.hostname] + newargs.extend(*args) + print(newargs, "bbbbb", self.hostname) + self.run_or_fail(newargs) -def transfer_dir(hostname, source, destination): - remove_remote_dir(hostname, destination) - run_or_fail(["scp", "-r", source, "root@" + hostname + ":" + destination]) + def remove_remote_dir(self, destination): + self.remote_run_or_fail(["rm", "-rf", destination]) -def base_directory(): - """Returns the directory in which all cdist stuff is based in""" - print("Going to", __file__, os.path.join(os.path.dirname(__file__), os.pardir)) - os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) - return os.getcwd() + def transfer_dir(self, source, destination): + self.remove_remote_dir(destination) + self.run_or_fail(["scp", "-qr", source, "root@" + self.hostname + ":" + destination]) -def remote_base_directory(): - return "/var/lib/cdist" + def base_directory(self): + """Returns the directory in which all cdist stuff is based in""" + print("Going to", __file__, os.path.join(os.path.dirname(__file__), os.pardir)) + os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) + return os.getcwd() -def conf_directory(): - """Returns path to main configuration directory""" - return os.path.join(base_directory(), "conf") + def remote_base_directory(self): + return "/var/lib/cdist" -def remote_conf_directory(): - """Returns path to remote main configuration directory""" - return os.path.join(remote_base_directory(), "conf") + def conf_directory(self): + """Returns path to main configuration directory""" + return os.path.join(self.base_directory(), "conf") -def out_dir(): - # FIXME: stopped - probably need static temp know! - """Local directory containing output""" - return os.path.join(base_directory(), "conf") + def remote_conf_directory(self): + """Returns path to remote main configuration directory""" + return os.path.join(self.remote_base_directory(), "conf") -def global_explorer_directory(): - """Returns path to directory containing the global explorers""" - return os.path.join(conf_directory(), "explorer") + def global_explorer_directory(self): + """Returns path to directory containing the global explorers""" + return os.path.join(self.conf_directory(), "explorer") -def remote_global_explorer_directory(): - """Returns path to the remote directory containing the global explorers""" - return os.path.join(remote_conf_directory(), "explorer") + def remote_global_explorer_directory(self): + """Returns path to the remote directory containing the global explorers""" + return os.path.join(self.remote_conf_directory(), "explorer") -def remote_global_explorer_path(explorer): - """Returns path to the remote explorer""" - return os.path.join(remote_global_explorer_directory(), explorer) + def remote_global_explorer_path(self,explorer): + """Returns path to the remote explorer""" + return os.path.join(self.remote_global_explorer_directory(), explorer) -def list_global_explorers(): - """Return list of available explorers""" - return os.listdir(global_explorer_directory()) + def list_global_explorers(self): + """Return list of available explorers""" + return os.listdir(self.global_explorer_directory()) -def transfer_global_explorers(hostname): - transfer_dir(hostname, global_explorer_directory(), remote_global_explorer_directory()) + def transfer_global_explorers(self): + self.transfer_dir(self.global_explorer_directory(), self.remote_global_explorer_directory()) -def global_explore(hostname): - """Run global explorers""" - explorers = list_global_explorers() - if(len(explorers) == 0): - exit_error("No explorers found in", global_explorer_directory()) + def global_explore(self): + """Run global explorers""" + explorers = self.list_global_explorers() + if(len(explorers) == 0): + self.exit_error("No explorers found in", self.global_explorer_directory()) - transfer_global_explorers(hostname) - for explorer in explorers: - remote_run_or_fail(hostname, [remote_global_explorer_path(explorer)]) + self.transfer_global_explorers() + for explorer in explorers: + self.remote_run_or_fail([self.remote_global_explorer_path(explorer)]) -def init_deploy(hostname): - logger("info", "Creating clean directory structure") + def init_deploy(self): + self.logger("info", "Creating clean directory structure") - # Ensure there is no old stuff, neither local nor remote -# run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) -# -# remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) -# -# # Create base directories -# run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) -# remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) -# -# # Link configuraion source directory - consistent with remote -# run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) + # Ensure there is no old stuff, neither local nor remote + # run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) + # + # remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) + # + # # Create base directories + # run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) + # remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) + # + # # Link configuraion source directory - consistent with remote + # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) -def cdist_deploy_to(hostname): - """Mimic the old deploy to: Deploy to one host""" - logger("info", "Deploying to host", hostname) - init_deploy(hostname) - global_explore(hostname) + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + self.logger("info", "Deploying to host", self.hostname) + self.init_deploy() + self.global_explore() if __name__ == "__main__": hostname=sys.argv[1] # logger("info", "cdist", cdist_version, ": Configuring host", hostname) - cdist_deploy_to(hostname) - print(list_global_explorers()) - c = Cdist() + for host in sys.argv[1:]: + c = Cdist(host) + c.deploy_to() + print(c.list_global_explorers()) + c.cleanup() From 0dc6af251219e267791ec6d4f08667c38084f991 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 04:22:40 +0200 Subject: [PATCH 041/166] wrap into a class Signed-off-by: Nico Schottelius --- bin/cdist | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 53728648..6770c181 100755 --- a/bin/cdist +++ b/bin/cdist @@ -74,7 +74,6 @@ class Cdist: """Run something on the remote side and fail is something breaks""" newargs = ["ssh", "root@" + self.hostname] newargs.extend(*args) - print(newargs, "bbbbb", self.hostname) self.run_or_fail(newargs) def remove_remote_dir(self, destination): From e8360df96b6bb9ae4641046e862d1a414ebfc03c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 09:47:30 +0200 Subject: [PATCH 042/166] allow passing arguments to Popen() Signed-off-by: Nico Schottelius --- bin/cdist | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 6770c181..39a0b80b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -59,22 +59,22 @@ class Cdist: sys.exit(1) - def run_or_fail(self,*args): + def run_or_fail(self,*args, **kargs): # newargs = ["echo"] newargs = [] newargs.extend(*args) print(newargs) try: - subprocess.check_call(newargs) + subprocess.check_call(newargs, **kargs) except subprocess.CalledProcessError: self.exit_error("Command failed:", " ".join(newargs)) - def remote_run_or_fail(self, *args): + def remote_run_or_fail(self, *args, **kargs): """Run something on the remote side and fail is something breaks""" newargs = ["ssh", "root@" + self.hostname] newargs.extend(*args) - self.run_or_fail(newargs) + self.run_or_fail(newargs, **kargs) def remove_remote_dir(self, destination): self.remote_run_or_fail(["rm", "-rf", destination]) From 4c1939829e26ae8ba73f48793409daec8c5d0c3f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:08:44 +0200 Subject: [PATCH 043/166] import logging, begin to put constant stuff in module Signed-off-by: Nico Schottelius --- bin/cdist | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/bin/cdist b/bin/cdist index 39a0b80b..162bf1aa 100755 --- a/bin/cdist +++ b/bin/cdist @@ -19,22 +19,36 @@ # # -import sys # argv +import sys import subprocess # execute stuff import os import tempfile import shutil +import logging +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + +logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s') + class Cdist: """Cdist main class to hold arbitrary data""" version="2.0.0" def __init__(self, hostname): - self.tempdir = tempfile.mkdtemp() self.hostname = hostname - print(self.hostname) - print(self.tempdir) + + # Setup directory paths + self.temp_dir = tempfile.mkdtemp() + + self.out_dir = os.path.join(self.temp_dir, "out") + os.mkdir(self.out_dir) + + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + os.mkdir(self.global_explorer_out_dir) + + # Given paths from installation + self.conf_dir = os.path.join(self.base_dir, "conf") def cleanup(self): # Do not use in __del__: @@ -45,11 +59,6 @@ class Cdist: print(self.tempdir) shutil.rmtree(self.tempdir) - def out_dir(self): - # FIXME: stopped - probably need static temp know! - """Local directory containing output""" - return os.path.join(base_directory(), "conf") - def logger(self,type, *args): """Ignore type for now, support later""" print(*args) @@ -83,18 +92,9 @@ class Cdist: self.remove_remote_dir(destination) self.run_or_fail(["scp", "-qr", source, "root@" + self.hostname + ":" + destination]) - def base_directory(self): - """Returns the directory in which all cdist stuff is based in""" - print("Going to", __file__, os.path.join(os.path.dirname(__file__), os.pardir)) - os.chdir(os.path.join(os.path.dirname(__file__), os.pardir)) - return os.getcwd() - def remote_base_directory(self): return "/var/lib/cdist" - def conf_directory(self): - """Returns path to main configuration directory""" - return os.path.join(self.base_directory(), "conf") def remote_conf_directory(self): """Returns path to remote main configuration directory""" @@ -112,6 +112,10 @@ class Cdist: """Returns path to the remote explorer""" return os.path.join(self.remote_global_explorer_directory(), explorer) + def global_explorer_output_path(self, explorer): + """Returns path of the output for a global explorer""" + return os.path.join(self.global_explorer_out_dir, explorer) + def list_global_explorers(self): """Return list of available explorers""" return os.listdir(self.global_explorer_directory()) @@ -127,7 +131,10 @@ class Cdist: self.transfer_global_explorers() for explorer in explorers: - self.remote_run_or_fail([self.remote_global_explorer_path(explorer)]) + output = self.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + self.remote_run_or_fail([self.remote_global_explorer_path(explorer)], stdout=output_fd) + output_fd.close() def init_deploy(self): From 7d3c67c178a1996eb7a6633040c9a30668f56fda Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:44:04 +0200 Subject: [PATCH 044/166] make it work again :-) Signed-off-by: Nico Schottelius --- bin/cdist | 81 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/bin/cdist b/bin/cdist index 162bf1aa..c160b945 100755 --- a/bin/cdist +++ b/bin/cdist @@ -27,9 +27,37 @@ import shutil import logging +# Given paths from installation BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) +CONF_DIR = os.path.join(BASE_DIR, "conf") +GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") + + +#class Context(object): +# +# def __init__(self, target_host): +# self.target_host = target_host +# +# # class variable +# user_selber_shuld_wenn_aendert = 'bla' +# +# # read only, aber statisch +# @property +# def remote_base_directory(self): +# return "/var/lib/cdist" +# @property.setter +# +# @property +# def special_foo(self): +# return 'foo/{0}'.format(self.target_host) +# + logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s') +log = logging.getLogger() class Cdist: """Cdist main class to hold arbitrary data""" @@ -38,35 +66,27 @@ class Cdist: def __init__(self, hostname): self.hostname = hostname + # log.info("foobar") + # Setup directory paths self.temp_dir = tempfile.mkdtemp() - self.out_dir = os.path.join(self.temp_dir, "out") os.mkdir(self.out_dir) - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) - # Given paths from installation - self.conf_dir = os.path.join(self.base_dir, "conf") - def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization # "other globals referenced by the __del__() method may already have been deleted # or in the process of being torn down (e.g. the import machinery shutting down)" # - print(self.tempdir) - shutil.rmtree(self.tempdir) - - def logger(self,type, *args): - """Ignore type for now, support later""" - print(*args) + print(self.temp_dir) + shutil.rmtree(self.temp_dir) def exit_error(self,*args): - self.logger("error", *args) + log.error(args) sys.exit(1) - def run_or_fail(self,*args, **kargs): # newargs = ["echo"] @@ -92,42 +112,26 @@ class Cdist: self.remove_remote_dir(destination) self.run_or_fail(["scp", "-qr", source, "root@" + self.hostname + ":" + destination]) - def remote_base_directory(self): - return "/var/lib/cdist" - - - def remote_conf_directory(self): - """Returns path to remote main configuration directory""" - return os.path.join(self.remote_base_directory(), "conf") - - def global_explorer_directory(self): - """Returns path to directory containing the global explorers""" - return os.path.join(self.conf_directory(), "explorer") - - def remote_global_explorer_directory(self): - """Returns path to the remote directory containing the global explorers""" - return os.path.join(self.remote_conf_directory(), "explorer") - - def remote_global_explorer_path(self,explorer): - """Returns path to the remote explorer""" - return os.path.join(self.remote_global_explorer_directory(), explorer) - def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) + def remote_global_explorer_path(self, explorer): + """Returns path to the remote explorer""" + return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + def list_global_explorers(self): """Return list of available explorers""" - return os.listdir(self.global_explorer_directory()) + return os.listdir(GLOBAL_EXPLORER_DIR) def transfer_global_explorers(self): - self.transfer_dir(self.global_explorer_directory(), self.remote_global_explorer_directory()) + self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) def global_explore(self): """Run global explorers""" explorers = self.list_global_explorers() if(len(explorers) == 0): - self.exit_error("No explorers found in", self.global_explorer_directory()) + self.exit_error("No explorers found in", GLOBAL_EXPLORER_DIR) self.transfer_global_explorers() for explorer in explorers: @@ -138,7 +142,7 @@ class Cdist: def init_deploy(self): - self.logger("info", "Creating clean directory structure") + log.info("Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote # run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) @@ -154,14 +158,13 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - self.logger("info", "Deploying to host", self.hostname) + log.info("Deploying to host", self.hostname) self.init_deploy() self.global_explore() if __name__ == "__main__": hostname=sys.argv[1] -# logger("info", "cdist", cdist_version, ": Configuring host", hostname) for host in sys.argv[1:]: c = Cdist(host) From 8fa576152ee2666c24e6d53dd697a40d78940980 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:44:58 +0200 Subject: [PATCH 045/166] make it work again :-) Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index c160b945..1e2fe11e 100755 --- a/bin/cdist +++ b/bin/cdist @@ -158,7 +158,7 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to host", self.hostname) + log.info("Deploying to host" + self.hostname) self.init_deploy() self.global_explore() From 506e0e6c81d340f424cf589b16d7e8672a5d824f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:45:58 +0200 Subject: [PATCH 046/166] +better logging Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 1e2fe11e..56991d68 100755 --- a/bin/cdist +++ b/bin/cdist @@ -158,7 +158,7 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to host" + self.hostname) + log.info("Deploying to host " + self.hostname) self.init_deploy() self.global_explore() From 908d9d06f9737a37a1558313836c65b3a172fce2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:50:30 +0200 Subject: [PATCH 047/166] prepend __explorer on global explorer run Signed-off-by: Nico Schottelius --- bin/cdist | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 56991d68..d6421161 100755 --- a/bin/cdist +++ b/bin/cdist @@ -81,8 +81,8 @@ class Cdist: # "other globals referenced by the __del__() method may already have been deleted # or in the process of being torn down (e.g. the import machinery shutting down)" # - print(self.temp_dir) - shutil.rmtree(self.temp_dir) + print("I should cleanup " + self.temp_dir) + # shutil.rmtree(self.temp_dir) def exit_error(self,*args): log.error(args) @@ -137,7 +137,11 @@ class Cdist: for explorer in explorers: output = self.global_explorer_output_path(explorer) output_fd = open(output, mode='w') - self.remote_run_or_fail([self.remote_global_explorer_path(explorer)], stdout=output_fd) + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.remote_global_explorer_path(explorer)) + + self.remote_run_or_fail(cmd, stdout=output_fd) output_fd.close() @@ -145,12 +149,9 @@ class Cdist: log.info("Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote - # run_or_fail(["rm -rf", "$__cdist_local_base_dir"]) - # # remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) # # # Create base directories - # run_or_fail(["mkdir -p", "$__cdist_local_base_dir"]) # remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) # # # Link configuraion source directory - consistent with remote From ddff3e8b571fd907e423ea8888c482eb1f4c05f6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 11:55:29 +0200 Subject: [PATCH 048/166] support changing the initial manifest Signed-off-by: Nico Schottelius --- bin/cdist | 14 +++++++++++++- bin/cdist-deploy-to | 1 - 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index d6421161..766a2acf 100755 --- a/bin/cdist +++ b/bin/cdist @@ -31,6 +31,8 @@ import logging BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) CONF_DIR = os.path.join(BASE_DIR, "conf") GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") +MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") + REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") @@ -63,7 +65,7 @@ class Cdist: """Cdist main class to hold arbitrary data""" version="2.0.0" - def __init__(self, hostname): + def __init__(self, hostname, initial_manifest=False): self.hostname = hostname # log.info("foobar") @@ -75,6 +77,12 @@ class Cdist: self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) + # Mostly static, but can be overwritten on user demand + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(MANIFEST_DIR, "init") + def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization @@ -157,11 +165,15 @@ class Cdist: # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) + def initial_manifes(self): + """Run the initial manifest""" + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to host " + self.hostname) self.init_deploy() self.global_explore() + self.initial_manifest() if __name__ == "__main__": diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to index 84181bbd..79f42931 100755 --- a/bin/cdist-deploy-to +++ b/bin/cdist-deploy-to @@ -43,7 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " # See cdist-stages(7) # -cdist-explorer-run-global "$__cdist_target_host" cdist-manifest-run-init "$__cdist_target_host" cdist-object-all "$__cdist_target_host" cdist-object-prepare cdist-object-all "$__cdist_target_host" cdist-object-run From 9533e579b390c171f8147211d63866935b2d683b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:04:30 +0200 Subject: [PATCH 049/166] finish run of initial manifest, finish shell_run_or_debug_fail() Signed-off-by: Nico Schottelius --- bin/cdist | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 766a2acf..594b5eea 100755 --- a/bin/cdist +++ b/bin/cdist @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) # @@ -96,11 +97,23 @@ class Cdist: log.error(args) sys.exit(1) - def run_or_fail(self,*args, **kargs): + def shell_run_or_debug_fail(self, script, *args, **kargs): + kargs['shell'] = True + log.debug("Shell exec: " + " ".join(*args)) + + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + # FIXME: print out shell script! + script_fd = open(script) + log.error("Code that raised the error:\n" + script_fd.read()) + script_fd.close() + self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) + + def run_or_fail(self, *args, **kargs): # newargs = ["echo"] newargs = [] newargs.extend(*args) - print(newargs) try: subprocess.check_call(newargs, **kargs) @@ -165,15 +178,18 @@ class Cdist: # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) - def initial_manifes(self): + def run_initial_manifest(self): """Run the initial manifest""" + log.info("Running the initial manifest") + self.shell_run_or_debug_fail(self.initial_manifest, [self.initial_manifest]) + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to host " + self.hostname) self.init_deploy() self.global_explore() - self.initial_manifest() + self.run_initial_manifest() if __name__ == "__main__": From c5d960438cc111c5476b0261628b0ffcb1d25dbd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:05:20 +0200 Subject: [PATCH 050/166] import parser from steven Signed-off-by: Nico Schottelius --- bin/cdist | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/bin/cdist b/bin/cdist index 594b5eea..72e95e29 100755 --- a/bin/cdist +++ b/bin/cdist @@ -201,3 +201,41 @@ if __name__ == "__main__": print(c.list_global_explorers()) c.cleanup() + +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import logging +# TODO: configure logging based on config and/or user given arguments +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +log = logging.getLogger() + +import argparse +import sys + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Some helpfull blabla') + parser.add_argument('host', nargs='+', help='one or more hosts to operate on') + parser.add_argument('-i', '--initial-manifest', + help='path to a cdist manifest or - to read from stdin', + dest='manifest', required=True) + parser.add_argument('-p', '--parallel', + help='operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser.add_argument('-s', '--sequential', + help='operate on multiple hosts sequentially', + action='store_false', dest='parallel') + parser.add_argument('-d', '--debug', help='set log level to debug', + action='store_true') + + args = parser.parse_args(sys.argv[1:]) + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug('Look ma, now whe\'re showing debug messages') + + try: + print(args) + #import cdist + #sys.exit(cdist.main(args)) + except KeyboardInterrupt: + sys.exit(0) From 979174a5685ae810327f98b04a8cf522c504f15c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:11:31 +0200 Subject: [PATCH 051/166] integrate parser Signed-off-by: Nico Schottelius --- bin/cdist | 86 +++++++++++++++++++++++-------------------------------- 1 file changed, 36 insertions(+), 50 deletions(-) diff --git a/bin/cdist b/bin/cdist index 72e95e29..a9b400ca 100755 --- a/bin/cdist +++ b/bin/cdist @@ -20,13 +20,13 @@ # # -import sys -import subprocess # execute stuff -import os -import tempfile -import shutil +import argparse import logging - +import os +import subprocess +import shutil +import sys +import tempfile # Given paths from installation BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) @@ -37,6 +37,7 @@ MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") +VERSION = "2.0.0" #class Context(object): @@ -64,7 +65,6 @@ log = logging.getLogger() class Cdist: """Cdist main class to hold arbitrary data""" - version="2.0.0" def __init__(self, hostname, initial_manifest=False): self.hostname = hostname @@ -195,47 +195,33 @@ class Cdist: if __name__ == "__main__": hostname=sys.argv[1] - for host in sys.argv[1:]: - c = Cdist(host) - c.deploy_to() - print(c.list_global_explorers()) - c.cleanup() + parser = argparse.ArgumentParser(description='cdist ' + VERSION) + parser.add_argument('host', nargs='+', help='one or more hosts to operate on') + parser.add_argument('-d', '--debug', help='set log level to debug', + action='store_true') + parser.add_argument('-i', '--initial-manifest', + help='path to a cdist manifest or - to read from stdin', + dest='manifest', required=True) + parser.add_argument('-p', '--parallel', + help='operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser.add_argument('-s', '--sequential', + help='operate on multiple hosts sequentially', + action='store_false', dest='parallel') + + args = parser.parse_args(sys.argv[1:]) + if args.debug: + logging.root.setLevel(logging.DEBUG) + + try: + print(args) + import cdist + sys.exit(cdist.main(args)) - -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import logging -# TODO: configure logging based on config and/or user given arguments -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -log = logging.getLogger() - -import argparse -import sys - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Some helpfull blabla') - parser.add_argument('host', nargs='+', help='one or more hosts to operate on') - parser.add_argument('-i', '--initial-manifest', - help='path to a cdist manifest or - to read from stdin', - dest='manifest', required=True) - parser.add_argument('-p', '--parallel', - help='operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser.add_argument('-s', '--sequential', - help='operate on multiple hosts sequentially', - action='store_false', dest='parallel') - parser.add_argument('-d', '--debug', help='set log level to debug', - action='store_true') - - args = parser.parse_args(sys.argv[1:]) - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug('Look ma, now whe\'re showing debug messages') - - try: - print(args) - #import cdist - #sys.exit(cdist.main(args)) - except KeyboardInterrupt: - sys.exit(0) + for host in sys.argv[1:]: + c = Cdist(host) + c.deploy_to() + print(c.list_global_explorers()) + c.cleanup() + except KeyboardInterrupt: + sys.exit(0) From 1d367d5f581e246793e44016ec1df0836b822e6e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:17:44 +0200 Subject: [PATCH 052/166] make cdist localhost work again Signed-off-by: Nico Schottelius --- bin/cdist | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index a9b400ca..58ddcb3f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -201,7 +201,7 @@ if __name__ == "__main__": action='store_true') parser.add_argument('-i', '--initial-manifest', help='path to a cdist manifest or - to read from stdin', - dest='manifest', required=True) + dest='manifest', required=False) parser.add_argument('-p', '--parallel', help='operate on multiple hosts in parallel', action='store_true', dest='parallel') @@ -214,11 +214,9 @@ if __name__ == "__main__": logging.root.setLevel(logging.DEBUG) try: - print(args) - import cdist - sys.exit(cdist.main(args)) + log.debug(args) - for host in sys.argv[1:]: + for host in args.host: c = Cdist(host) c.deploy_to() print(c.list_global_explorers()) From c8ce7e98ec2da0bfad43272adab71a2b43df2303 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:45:54 +0200 Subject: [PATCH 053/166] use -e to shell Signed-off-by: Nico Schottelius --- bin/cdist | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 58ddcb3f..a5902472 100755 --- a/bin/cdist +++ b/bin/cdist @@ -101,6 +101,10 @@ class Cdist: kargs['shell'] = True log.debug("Shell exec: " + " ".join(*args)) + # Fail if the script fails + args[0].insert(0,"-e") + print(args) + try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: @@ -219,7 +223,6 @@ if __name__ == "__main__": for host in args.host: c = Cdist(host) c.deploy_to() - print(c.list_global_explorers()) c.cleanup() except KeyboardInterrupt: sys.exit(0) From 1a96f886567f676f0ea149d73fc7ef9546496b55 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:50:45 +0200 Subject: [PATCH 054/166] name variable target_host for consistency and easy handling Signed-off-by: Nico Schottelius --- bin/cdist | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index a5902472..323d2a96 100755 --- a/bin/cdist +++ b/bin/cdist @@ -66,8 +66,8 @@ log = logging.getLogger() class Cdist: """Cdist main class to hold arbitrary data""" - def __init__(self, hostname, initial_manifest=False): - self.hostname = hostname + def __init__(self, target_host, initial_manifest=False): + self.target_host = target_host # log.info("foobar") @@ -126,7 +126,7 @@ class Cdist: def remote_run_or_fail(self, *args, **kargs): """Run something on the remote side and fail is something breaks""" - newargs = ["ssh", "root@" + self.hostname] + newargs = ["ssh", "root@" + self.target_host] newargs.extend(*args) self.run_or_fail(newargs, **kargs) @@ -135,7 +135,7 @@ class Cdist: def transfer_dir(self, source, destination): self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, "root@" + self.hostname + ":" + destination]) + self.run_or_fail(["scp", "-qr", source, "root@" + self.target_host + ":" + destination]) def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" @@ -185,20 +185,23 @@ class Cdist: def run_initial_manifest(self): """Run the initial manifest""" log.info("Running the initial manifest") - self.shell_run_or_debug_fail(self.initial_manifest, [self.initial_manifest]) + env = os.environ.copy() + env['__target_host'] = self.target_host + + self.shell_run_or_debug_fail(self.initial_manifest, + [self.initial_manifest], + env=env) def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to host " + self.hostname) + log.info("Deploying to host " + self.target_host) self.init_deploy() self.global_explore() self.run_initial_manifest() if __name__ == "__main__": - hostname=sys.argv[1] - parser = argparse.ArgumentParser(description='cdist ' + VERSION) parser.add_argument('host', nargs='+', help='one or more hosts to operate on') parser.add_argument('-d', '--debug', help='set log level to debug', From 4fd8a16e6cd618ed61c9e312d5e3827bb4d15b2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 14:52:42 +0200 Subject: [PATCH 055/166] use *args not args :-) Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 323d2a96..7059f73b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -93,8 +93,8 @@ class Cdist: print("I should cleanup " + self.temp_dir) # shutil.rmtree(self.temp_dir) - def exit_error(self,*args): - log.error(args) + def exit_error(self, *args): + log.error(*args) sys.exit(1) def shell_run_or_debug_fail(self, script, *args, **kargs): From 2c2a234c74ce1a5641d3683f6ef947627585b007 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:13:05 +0200 Subject: [PATCH 056/166] use sh -e instead of shipped shell=true (see included comment) Signed-off-by: Nico Schottelius --- bin/cdist | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7059f73b..1216edab 100755 --- a/bin/cdist +++ b/bin/cdist @@ -98,13 +98,12 @@ class Cdist: sys.exit(1) def shell_run_or_debug_fail(self, script, *args, **kargs): - kargs['shell'] = True + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0].insert(0,"/bin/sh") + args[0].insert(1,"-e") log.debug("Shell exec: " + " ".join(*args)) - # Fail if the script fails - args[0].insert(0,"-e") - print(args) - try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: @@ -115,14 +114,12 @@ class Cdist: self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) def run_or_fail(self, *args, **kargs): - # newargs = ["echo"] - newargs = [] - newargs.extend(*args) + log.debug("Exec: " + " ".join(*args)) try: - subprocess.check_call(newargs, **kargs) + subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - self.exit_error("Command failed:", " ".join(newargs)) + self.exit_error("Command failed:", " ".join(*args)) def remote_run_or_fail(self, *args, **kargs): """Run something on the remote side and fail is something breaks""" From 97f7513f6eae5796fc9046530fe528d3e083dd6d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:14:28 +0200 Subject: [PATCH 057/166] -old fixme Signed-off-by: Nico Schottelius --- bin/cdist | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 1216edab..857e127d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -107,7 +107,6 @@ class Cdist: try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - # FIXME: print out shell script! script_fd = open(script) log.error("Code that raised the error:\n" + script_fd.read()) script_fd.close() From d6db6d0c7e22b45e970299bcfcbb641dfdcc774a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:20:25 +0200 Subject: [PATCH 058/166] begin move of cdist-type-emulator to lib/ and submit initial_manifest to cdist Signed-off-by: Nico Schottelius --- bin/cdist | 3 ++- {bin => lib}/cdist-type-emulator | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename {bin => lib}/cdist-type-emulator (100%) diff --git a/bin/cdist b/bin/cdist index 857e127d..6e13188d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -32,6 +32,7 @@ import tempfile BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) CONF_DIR = os.path.join(BASE_DIR, "conf") GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") +LIB_DIR = os.path.join(BASE_DIR, "lib") MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") REMOTE_BASE_DIR = "/var/lib/cdist" @@ -220,7 +221,7 @@ if __name__ == "__main__": log.debug(args) for host in args.host: - c = Cdist(host) + c = Cdist(host, initial_manifest=args.manifest) c.deploy_to() c.cleanup() except KeyboardInterrupt: diff --git a/bin/cdist-type-emulator b/lib/cdist-type-emulator similarity index 100% rename from bin/cdist-type-emulator rename to lib/cdist-type-emulator From 9ce26e5817907105c20ab9503933578717d14352 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:35:29 +0200 Subject: [PATCH 059/166] include cdist-type-emulator into path again Signed-off-by: Nico Schottelius --- bin/cdist | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bin/cdist b/bin/cdist index 6e13188d..0cc4ac7f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -34,6 +34,7 @@ CONF_DIR = os.path.join(BASE_DIR, "conf") GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") LIB_DIR = os.path.join(BASE_DIR, "lib") MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") +TYPE_DIR = os.path.join(CONF_DIR, "type") REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") @@ -64,6 +65,10 @@ VERSION = "2.0.0" logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s') log = logging.getLogger() +# List types +def list_types(): + return os.listdir(TYPE_DIR) + class Cdist: """Cdist main class to hold arbitrary data""" @@ -74,11 +79,18 @@ class Cdist: # Setup directory paths self.temp_dir = tempfile.mkdtemp() + self.out_dir = os.path.join(self.temp_dir, "out") os.mkdir(self.out_dir) + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) + # Setup binary directory + contents + self.bin_dir = os.path.join(self.out_dir, "bin") + os.mkdir(self.bin_dir) + self.link_type_to_emulator() + # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest @@ -149,6 +161,14 @@ class Cdist: def transfer_global_explorers(self): self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) + def link_type_to_emulator(self): + """Link type names to cdist-type-emulator""" + for type in list_types(): + source = os.path.join(LIB_DIR, "cdist-type-emulator") + destination = os.path.join(self.bin_dir, type) + log.debug("Linking %s to %s", source, destination) + os.symlink(source, destination) + def global_explore(self): """Run global explorers""" explorers = self.list_global_explorers() @@ -184,6 +204,7 @@ class Cdist: log.info("Running the initial manifest") env = os.environ.copy() env['__target_host'] = self.target_host + env['PATH'] = self.bin_dir + ":" + env['PATH'] self.shell_run_or_debug_fail(self.initial_manifest, [self.initial_manifest], From bf0f2b8f12949bfcd988f95533987f23d54d9695 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:39:21 +0200 Subject: [PATCH 060/166] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index fce4033e..972525aa 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,4 +1,5 @@ - rewrite in python? + - also do with cdist-type-emulator, which had quirks applied from outside to run - support non-ssh access? - Bug: os.path.join() may be wrong for the remote side! From dbbbf822fe55f714c115fa4e610b62361dfd614e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:52:34 +0200 Subject: [PATCH 061/166] make cdist-type-emulator work Signed-off-by: Nico Schottelius --- bin/cdist | 15 ++++++++++++--- bin/cdist-deploy-to | 1 - 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 0cc4ac7f..57df474d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -169,7 +169,7 @@ class Cdist: log.debug("Linking %s to %s", source, destination) os.symlink(source, destination) - def global_explore(self): + def run_global_explores(self): """Run global explorers""" explorers = self.list_global_explorers() if(len(explorers) == 0): @@ -186,6 +186,7 @@ class Cdist: self.remote_run_or_fail(cmd, stdout=output_fd) output_fd.close() + # def run_type_explorer(self): def init_deploy(self): log.info("Creating clean directory structure") @@ -203,9 +204,17 @@ class Cdist: """Run the initial manifest""" log.info("Running the initial manifest") env = os.environ.copy() - env['__target_host'] = self.target_host env['PATH'] = self.bin_dir + ":" + env['PATH'] + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + + # Legacy stuff to make cdist-type-emulator work + env['__cdist_conf_dir'] = CONF_DIR + env['__cdist_core_dir'] = os.path.join(BASE_DIR, "core") + env['__cdist_local_base_dir'] = self.temp_dir + env['__cdist_manifest'] = self.initial_manifest + self.shell_run_or_debug_fail(self.initial_manifest, [self.initial_manifest], env=env) @@ -215,7 +224,7 @@ class Cdist: """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to host " + self.target_host) self.init_deploy() - self.global_explore() + self.run_global_explores() self.run_initial_manifest() diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to index 79f42931..96650f90 100755 --- a/bin/cdist-deploy-to +++ b/bin/cdist-deploy-to @@ -43,7 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " # See cdist-stages(7) # -cdist-manifest-run-init "$__cdist_target_host" cdist-object-all "$__cdist_target_host" cdist-object-prepare cdist-object-all "$__cdist_target_host" cdist-object-run cdist-cache "$__cdist_target_host" From 0f36ddd649d5646e2955ff219524a31f1707f61e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 15:54:10 +0200 Subject: [PATCH 062/166] support object dir Signed-off-by: Nico Schottelius --- bin/cdist | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/cdist b/bin/cdist index 57df474d..d7e2f697 100755 --- a/bin/cdist +++ b/bin/cdist @@ -86,6 +86,8 @@ class Cdist: self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) + self.object_dir = os.path.join(self.out_dir, "object") + # Setup binary directory + contents self.bin_dir = os.path.join(self.out_dir, "bin") os.mkdir(self.bin_dir) From 03e2db83cd1234e55ca2b4801f31e3094e3516b8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 17:01:22 +0200 Subject: [PATCH 063/166] implement list of objects Signed-off-by: Nico Schottelius --- bin/cdist | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/bin/cdist b/bin/cdist index d7e2f697..8655dcad 100755 --- a/bin/cdist +++ b/bin/cdist @@ -160,6 +160,44 @@ class Cdist: """Return list of available explorers""" return os.listdir(GLOBAL_EXPLORER_DIR) + def list_object_paths(self, starting_point = False): + """Return list of paths of existing objects""" + object_paths = [] + + if not starting_point: + starting_point = self.object_dir + + for content in os.listdir(starting_point): + full_path = os.path.join(starting_point, content) + print(full_path) + if os.path.isdir(full_path): + log.debug("Recursing for %s", full_path) + object_paths.extend(self.list_object_paths(starting_point = full_path)) + + # Directory contains .cdist -> is an object + if content == ".cdist": + log.debug("Adding Object Path %s", starting_point) + object_paths.append(starting_point) + + return object_paths + + def list_objects(self, starting_point = False): + """Return list of existing objects""" + + if not starting_point: + starting_point = self.object_dir + + object_paths = self.list_object_paths(starting_point) + objects = [] + log.debug("Paths recieved: %s", object_paths) + log.debug("And te starting point: %s", starting_point) + + for path in object_paths: + objects.append(os.path.relpath(path, starting_point)) + + return objects + + def transfer_global_explorers(self): self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) From 4d765a702406de7b351174421e6679cb6507cdd3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 17:45:07 +0200 Subject: [PATCH 064/166] begin type explorer transfer Signed-off-by: Nico Schottelius --- bin/cdist | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 8655dcad..4e62bd64 100755 --- a/bin/cdist +++ b/bin/cdist @@ -38,7 +38,10 @@ TYPE_DIR = os.path.join(CONF_DIR, "type") REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") + +DOT_CDIST = ".cdist" VERSION = "2.0.0" @@ -93,6 +96,9 @@ class Cdist: os.mkdir(self.bin_dir) self.link_type_to_emulator() + # List of type explorers transferred + self.type_explorers_transferred = {} + # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest @@ -175,12 +181,15 @@ class Cdist: object_paths.extend(self.list_object_paths(starting_point = full_path)) # Directory contains .cdist -> is an object - if content == ".cdist": + if content == DOT_CDIST: log.debug("Adding Object Path %s", starting_point) object_paths.append(starting_point) return object_paths + def get_type_from_object(cdist_object): + return cdist_object.split(os.sep)[0] + def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -197,10 +206,20 @@ class Cdist: return objects - def transfer_global_explorers(self): self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) + def transfer_type_explorers(self, type): + """Transfer explorers of a type, but only once""" + if self.type_explorers_transferred[type] == 1 + return + else + self.type_explorers_transferred[type] = 1 + + src = os.path.join(TYPE_DIR, type) + dst = os.path.join(REMOTE_TYPE_DIR, type, "explorer") + self.transfer_dir(src, dst) + def link_type_to_emulator(self): """Link type names to cdist-type-emulator""" for type in list_types(): @@ -226,7 +245,11 @@ class Cdist: self.remote_run_or_fail(cmd, stdout=output_fd) output_fd.close() - # def run_type_explorer(self): + def run_type_explorer(self, object): + """Run type specific explorers for objects""" + # Based on bin/cdist-object-explorer-run + + # Transfering explorers for this type def init_deploy(self): log.info("Creating clean directory structure") From 97ac276433bd15722d3ada5891d47d00ddb90856 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 17:52:15 +0200 Subject: [PATCH 065/166] implement transfer of type explorers Signed-off-by: Nico Schottelius --- bin/cdist | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 4e62bd64..56327fb4 100755 --- a/bin/cdist +++ b/bin/cdist @@ -211,15 +211,20 @@ class Cdist: def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" - if self.type_explorers_transferred[type] == 1 + if type in self.type_explorers_transferred: return - else - self.type_explorers_transferred[type] = 1 src = os.path.join(TYPE_DIR, type) - dst = os.path.join(REMOTE_TYPE_DIR, type, "explorer") + base = os.path.join(REMOTE_TYPE_DIR, type) + dst = os.path.join(base, "explorer") + + # Ensure the path path exists + self.remote_run_or_fail(["mkdir", "-p", base]) self.transfer_dir(src, dst) + # Do not retransfer + self.type_explorers_transferred[type] = 1 + def link_type_to_emulator(self): """Link type names to cdist-type-emulator""" for type in list_types(): From f31f33418ff646f9107e72c33dc87c1b4d51644e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 18:03:19 +0200 Subject: [PATCH 066/166] begin to actually prepare the object/type explorers Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 56327fb4..0a7011b4 100755 --- a/bin/cdist +++ b/bin/cdist @@ -187,7 +187,7 @@ class Cdist: return object_paths - def get_type_from_object(cdist_object): + def get_type_from_object(self, cdist_object): return cdist_object.split(os.sep)[0] def list_objects(self, starting_point = False): @@ -250,11 +250,13 @@ class Cdist: self.remote_run_or_fail(cmd, stdout=output_fd) output_fd.close() - def run_type_explorer(self, object): + def run_type_explorer(self, cdist_object): """Run type specific explorers for objects""" # Based on bin/cdist-object-explorer-run # Transfering explorers for this type + type = self.get_type_from_object(cdist_object) + self.transfer_type_explorers(type) def init_deploy(self): log.info("Creating clean directory structure") @@ -294,6 +296,11 @@ class Cdist: self.init_deploy() self.run_global_explores() self.run_initial_manifest() + + objects = self.list_objects() + + for cdist_object in objects: + self.run_type_explorer(cdist_object) if __name__ == "__main__": From ab0cd9430fc270f0f99d21e16c5360173db9b914 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 22:40:12 +0200 Subject: [PATCH 067/166] add support for the banner :-) Signed-off-by: Nico Schottelius --- bin/cdist | 61 ++++++++++++++++++++++++--- conf/type/__directory/explorer/exists | 30 ------------- 2 files changed, 56 insertions(+), 35 deletions(-) delete mode 100755 conf/type/__directory/explorer/exists diff --git a/bin/cdist b/bin/cdist index 0a7011b4..ad87b38c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -28,6 +28,21 @@ import shutil import sys import tempfile +BANNER = """ + .. . .x+=:. s + dF @88> z` ^% :8 + '88bu. %8P . . -# -# -# Check whether file exists or not -# - -destination="/$__object_id" - -if [ -e "$destination" ]; then - echo yes -else - echo no -fi From 7e8362eeedd24f7df7c983d4e698dfdccb162f76 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 22:44:25 +0200 Subject: [PATCH 068/166] allow zero hosts to be specified, fixup exit_error Signed-off-by: Nico Schottelius --- bin/cdist | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index ad87b38c..8f556b1e 100755 --- a/bin/cdist +++ b/bin/cdist @@ -160,7 +160,7 @@ class Cdist: try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - self.exit_error("Command failed:", " ".join(*args)) + self.exit_error("Command failed: " + " ".join(*args)) def remote_run_or_fail(self, *args, **kargs): """Run something on the remote side and fail is something breaks""" @@ -349,20 +349,20 @@ class Cdist: if __name__ == "__main__": parser = argparse.ArgumentParser(description='cdist ' + VERSION) - parser.add_argument('host', nargs='+', help='one or more hosts to operate on') + parser.add_argument('host', nargs='*', help='one or more hosts to operate on') parser.add_argument('-b', '--banner', help='Show cdist banner', action='store_true', dest='banner') - parser.add_argument('-d', '--debug', help='set log level to debug', + parser.add_argument('-d', '--debug', help='Set log level to debug', action='store_true') parser.add_argument('-i', '--initial-manifest', - help='path to a cdist manifest or - to read from stdin', + help='Path to a cdist manifest or - to read from stdin', dest='manifest', required=False) parser.add_argument('-p', '--parallel', - help='operate on multiple hosts in parallel', + help='Operate on multiple hosts in parallel', action='store_true', dest='parallel') parser.add_argument('-s', '--sequential', - help='operate on multiple hosts sequentially', + help='Operate on multiple hosts sequentially', action='store_false', dest='parallel') args = parser.parse_args(sys.argv[1:]) From cbdb5cd05d0ea4a98d54cc83d6b03f4be136cbc7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Sep 2011 23:01:11 +0200 Subject: [PATCH 069/166] fixup type explorer run Signed-off-by: Nico Schottelius --- bin/cdist | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/bin/cdist b/bin/cdist index 8f556b1e..3f57de40 100755 --- a/bin/cdist +++ b/bin/cdist @@ -189,7 +189,13 @@ class Cdist: def list_type_explorers(self, type): """Return list of available explorers for a specific type""" - return os.listdir(type_explorer_dir(type)) + dir = self.type_explorer_dir(type) + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + return list def list_object_paths(self, starting_point = False): """Return list of paths of existing objects""" @@ -234,7 +240,7 @@ class Cdist: """Return directory that holds the explorers of a type""" return os.path.join(TYPE_DIR, type, "explorer") - def remote_type_explorer_dir(type): + def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" return os.path.join(REMOTE_TYPE_DIR, type, "explorer") @@ -246,17 +252,20 @@ class Cdist: if type in self.type_explorers_transferred: log.debug("Skipping retransfer for %s", type) return + else: + # Do not retransfer + self.type_explorers_transferred[type] = 1 src = self.type_explorer_dir(type) remote_base = os.path.join(REMOTE_TYPE_DIR, type) - dst = self.type_explorer_dir(type) + dst = self.remote_type_explorer_dir(type) - # Ensure the path path exists - self.remote_run_or_fail(["mkdir", "-p", remote_base]) - self.transfer_dir(src, dst) + # Only continue, if there is at least the directory + if os.path.isdir(src): + # Ensure that the path exists + self.remote_run_or_fail(["mkdir", "-p", remote_base]) + self.transfer_dir(src, dst) - # Do not retransfer - self.type_explorers_transferred[type] = 1 def link_type_to_emulator(self): """Link type names to cdist-type-emulator""" @@ -293,12 +302,12 @@ class Cdist: cmd = [] cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + remote_type_explorer_dir(type)) + cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - explorers = list_type_explorers(type) + explorers = self.list_type_explorers(type) for explorer in explorers: remote_cmd = cmd - remote_cmd.append(os.path.join(remote_type_explorer_dir(type), explorer)) + remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) self.remote_run_or_fail(remote_cmd) From 8889e4a5b0b0e14ecfc6752edc000bcb917fb618 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 08:06:16 +0200 Subject: [PATCH 070/166] prepare saving output of type explorers Signed-off-by: Nico Schottelius --- bin/cdist | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 3f57de40..b35b8c17 100755 --- a/bin/cdist +++ b/bin/cdist @@ -309,7 +309,12 @@ class Cdist: remote_cmd = cmd remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) - self.remote_run_or_fail(remote_cmd) + output = self.type_explorer_output_path(cdist_object) + output_fd = open(output, mode='w') + + self.remote_run_or_fail(remote_cmd, stdout=output_fd) + + output_fd.close() def init_deploy(self): log.info("Creating clean directory structure") From 894c39cd210ea0d312cd07aaf3f01d6e99ca0144 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 12:18:25 +0200 Subject: [PATCH 071/166] store results of type explorer Signed-off-by: Nico Schottelius --- bin/cdist | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index b35b8c17..e2e53c9d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -179,6 +179,14 @@ class Cdist: """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) + def type_explorer_output_dir(self, cdist_object): + """Returns and creates dir of the output for a type explorer""" + dir = os.path.join(self.object_full_path(cdist_object), DOT_CDIST, "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + + return dir + def remote_global_explorer_path(self, explorer): """Returns path to the remote explorer""" return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) @@ -219,8 +227,13 @@ class Cdist: return object_paths def get_type_from_object(self, cdist_object): + """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] + def object_full_path(self, cdist_object): + """Returns the full path to the object (""" + return os.path.join(self.object_dir, cdist_object) + def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -304,12 +317,15 @@ class Cdist: cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) + # FIXME: need to transfer object before as well! + cmd.append("__object=" + self.remote_type_explorer_dir(type)) + explorers = self.list_type_explorers(type) for explorer in explorers: remote_cmd = cmd remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) - output = self.type_explorer_output_path(cdist_object) + output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') self.remote_run_or_fail(remote_cmd, stdout=output_fd) From b2d1d1bee703e177a952e80660782cf06df5e782 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:21:16 +0200 Subject: [PATCH 072/166] begin to modify tuple (bad idea) Signed-off-by: Nico Schottelius --- bin/cdist | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/bin/cdist b/bin/cdist index e2e53c9d..0772c703 100755 --- a/bin/cdist +++ b/bin/cdist @@ -157,19 +157,29 @@ class Cdist: def run_or_fail(self, *args, **kargs): log.debug("Exec: " + " ".join(*args)) + if "remote" in kargs: + log.debug("Remote found") + if kargs["remote"]: + # Replace the list found in the tuple at position 0 + cmd = ["ssh", "root@" + self.target_host] + cmd.extend(args[0]) + args[0] = cmd + print(newargs) + + del kargs["remote"] + + log.debug(newargs) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - self.exit_error("Command failed: " + " ".join(*args)) + self.exit_error("Command failed: " + " ".join(*newargs)) - def remote_run_or_fail(self, *args, **kargs): - """Run something on the remote side and fail is something breaks""" - newargs = ["ssh", "root@" + self.target_host] - newargs.extend(*args) - self.run_or_fail(newargs, **kargs) +# def remote_run_or_fail(self, *args, **kargs): +# """Run something on the remote side and fail is something breaks""" +# self.run_or_fail(newargs, **kargs) def remove_remote_dir(self, destination): - self.remote_run_or_fail(["rm", "-rf", destination]) + self.run_or_fail(["rm", "-rf", destination], remote=True) def transfer_dir(self, source, destination): self.remove_remote_dir(destination) From c1ad93bccc6cf93b3a5129914624ec3350ca9b14 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:27:54 +0200 Subject: [PATCH 073/166] extend run_or_fail to include remote exec Signed-off-by: Nico Schottelius --- bin/cdist | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 0772c703..5227dc06 100755 --- a/bin/cdist +++ b/bin/cdist @@ -155,20 +155,14 @@ class Cdist: self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) def run_or_fail(self, *args, **kargs): - log.debug("Exec: " + " ".join(*args)) - if "remote" in kargs: log.debug("Remote found") if kargs["remote"]: - # Replace the list found in the tuple at position 0 - cmd = ["ssh", "root@" + self.target_host] - cmd.extend(args[0]) - args[0] = cmd - print(newargs) + args[0][:0] = ["ssh", "root@" + self.target_host] del kargs["remote"] - log.debug(newargs) + log.debug("Exec: " + " ".join(*args)) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: From 20aafe62f578bf51512507d53d9f385e1b94dba7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:30:53 +0200 Subject: [PATCH 074/166] make use of remote=True flag Signed-off-by: Nico Schottelius --- bin/cdist | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/bin/cdist b/bin/cdist index 5227dc06..68424cfd 100755 --- a/bin/cdist +++ b/bin/cdist @@ -168,10 +168,6 @@ class Cdist: except subprocess.CalledProcessError: self.exit_error("Command failed: " + " ".join(*newargs)) -# def remote_run_or_fail(self, *args, **kargs): -# """Run something on the remote side and fail is something breaks""" -# self.run_or_fail(newargs, **kargs) - def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) @@ -280,7 +276,7 @@ class Cdist: # Only continue, if there is at least the directory if os.path.isdir(src): # Ensure that the path exists - self.remote_run_or_fail(["mkdir", "-p", remote_base]) + self.run_or_fail(["mkdir", "-p", remote_base], remote=True) self.transfer_dir(src, dst) @@ -306,7 +302,7 @@ class Cdist: cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append(self.remote_global_explorer_path(explorer)) - self.remote_run_or_fail(cmd, stdout=output_fd) + self.run_or_fail(cmd, stdout=output_fd, remote=True) output_fd.close() def run_type_explorer(self, cdist_object): @@ -332,7 +328,7 @@ class Cdist: output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') - self.remote_run_or_fail(remote_cmd, stdout=output_fd) + self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) output_fd.close() From f21ce6a0db1f778fba8c3726155fa7562c3a4635 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:51:25 +0200 Subject: [PATCH 075/166] commit broken code Signed-off-by: Nico Schottelius --- bin/cdist | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 68424cfd..a0189fbe 100755 --- a/bin/cdist +++ b/bin/cdist @@ -98,6 +98,7 @@ class Cdist: def __init__(self, target_host, initial_manifest=False): self.target_host = target_host + self.remote_prefix = ["ssh", "root@" + self.target_host] # log.info("foobar") @@ -139,26 +140,47 @@ class Cdist: log.error(*args) sys.exit(1) + def remote_cat(filename): + cmd = self.remote_prefix + cmd.append("cat") + cmd.append(filename) + try: + subprocess.call(cmd) + except subprocess.CalledProcessError: + log.error("Remote cat failed") + def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called - args[0].insert(0,"/bin/sh") - args[0].insert(1,"-e") - log.debug("Shell exec: " + " ".join(*args)) + args[0][:0] = [ "/bin/sh", "-e" ] + if "remote" in kargs: + log.debug("Remote found") + if kargs["remote"]: + args[0][:0] = self.remote_prefix + remote = true + + del kargs["remote"] + + log.debug("Shell exec: " + " ".join(*args)) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - script_fd = open(script) - log.error("Code that raised the error:\n" + script_fd.read()) - script_fd.close() + log.error("Code that raised the error:\n") + if remote: + remote_cat(script) + else: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) def run_or_fail(self, *args, **kargs): if "remote" in kargs: log.debug("Remote found") if kargs["remote"]: - args[0][:0] = ["ssh", "root@" + self.target_host] + args[0][:0] = self.remote_prefix del kargs["remote"] @@ -166,7 +188,7 @@ class Cdist: try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - self.exit_error("Command failed: " + " ".join(*newargs)) + self.exit_error("Command failed: " + " ".join(*args)) def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) @@ -322,6 +344,7 @@ class Cdist: explorers = self.list_type_explorers(type) for explorer in explorers: + # THIS IS A BUG, because remote_cmd is NOT a copy remote_cmd = cmd remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) From 2c15069d96e0e35cffc7e4523f7ddf85d7d87212 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 14:56:51 +0200 Subject: [PATCH 076/166] make remote_cmd a new list Signed-off-by: Nico Schottelius --- bin/cdist | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index a0189fbe..70df214c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -344,9 +344,7 @@ class Cdist: explorers = self.list_type_explorers(type) for explorer in explorers: - # THIS IS A BUG, because remote_cmd is NOT a copy - remote_cmd = cmd - remote_cmd.append(os.path.join(self.remote_type_explorer_dir(type), explorer)) + remote_cmd = cmd + os.path.join(self.remote_type_explorer_dir(type), explorer) output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') From 6d5a53b7ad9368c2ee3bc704a7068fad5b52ee9c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 15:00:09 +0200 Subject: [PATCH 077/166] fixup list bug by creating a new one Signed-off-by: Nico Schottelius --- bin/cdist | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 70df214c..80d9f525 100755 --- a/bin/cdist +++ b/bin/cdist @@ -155,7 +155,6 @@ class Cdist: args[0][:0] = [ "/bin/sh", "-e" ] if "remote" in kargs: - log.debug("Remote found") if kargs["remote"]: args[0][:0] = self.remote_prefix remote = true @@ -178,7 +177,6 @@ class Cdist: def run_or_fail(self, *args, **kargs): if "remote" in kargs: - log.debug("Remote found") if kargs["remote"]: args[0][:0] = self.remote_prefix @@ -344,13 +342,10 @@ class Cdist: explorers = self.list_type_explorers(type) for explorer in explorers: - remote_cmd = cmd + os.path.join(self.remote_type_explorer_dir(type), explorer) - + remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') - self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) - output_fd.close() def init_deploy(self): From 722339f3e1164b0aac1d2e54904a2f2a510d5548 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 15:16:47 +0200 Subject: [PATCH 078/166] support getting paths of objects and their parameters Signed-off-by: Nico Schottelius --- bin/cdist | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 80d9f525..302e21be 100755 --- a/bin/cdist +++ b/bin/cdist @@ -53,6 +53,7 @@ TYPE_DIR = os.path.join(CONF_DIR, "type") REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") @@ -100,8 +101,6 @@ class Cdist: self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] - # log.info("foobar") - # Setup directory paths self.temp_dir = tempfile.mkdtemp() @@ -192,6 +191,7 @@ class Cdist: self.run_or_fail(["rm", "-rf", destination], remote=True) def transfer_dir(self, source, destination): + """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) self.run_or_fail(["scp", "-qr", source, "root@" + self.target_host + ":" + destination]) @@ -201,7 +201,7 @@ class Cdist: def type_explorer_output_dir(self, cdist_object): """Returns and creates dir of the output for a type explorer""" - dir = os.path.join(self.object_full_path(cdist_object), DOT_CDIST, "explorer") + dir = os.path.join(self.object_dir(cdist_object), "explorer") if not os.path.isdir(dir): os.mkdir(dir) @@ -250,9 +250,21 @@ class Cdist: """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] - def object_full_path(self, cdist_object): - """Returns the full path to the object (""" - return os.path.join(self.object_dir, cdist_object) + def object_dir(self, cdist_object): + """Returns the full path to the object (including .cdist)""" + return os.path.join(self.object_dir, cdist_object, DOT_CDIST) + + def remote_object_dir(self, cdist_object): + """Returns the remote full path to the object (including .cdist)""" + return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + + def object_parameter_dir(self, cdist_object): + """Returns the dir to the object parameter""" + return os.path.join(object_dir(cdist_object), "parameter") + + def remote_object_parameter_dir(self, cdist_object): + """Returns the remote dir to the object parameter""" + return os.path.join(remote_object_dir(cdist_object), "parameter") def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -277,6 +289,9 @@ class Cdist: """Return remote directory that holds the explorers of a type""" return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + def transfer_object_parameter(self, cdist_object): + self.transfer_dir(object_dir(cdist_object), ) + def transfer_global_explorers(self): self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) From db2562303ade41004fe9849074410caa18840376 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 15:28:38 +0200 Subject: [PATCH 079/166] in theory transfer the object parameters Signed-off-by: Nico Schottelius --- bin/cdist | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/bin/cdist b/bin/cdist index 302e21be..b2f10ba1 100755 --- a/bin/cdist +++ b/bin/cdist @@ -110,7 +110,7 @@ class Cdist: self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") os.mkdir(self.global_explorer_out_dir) - self.object_dir = os.path.join(self.out_dir, "object") + self.object_base_dir = os.path.join(self.out_dir, "object") # Setup binary directory + contents self.bin_dir = os.path.join(self.out_dir, "bin") @@ -230,7 +230,7 @@ class Cdist: object_paths = [] if not starting_point: - starting_point = self.object_dir + starting_point = self.object_base_dir for content in os.listdir(starting_point): full_path = os.path.join(starting_point, content) @@ -252,7 +252,7 @@ class Cdist: def object_dir(self, cdist_object): """Returns the full path to the object (including .cdist)""" - return os.path.join(self.object_dir, cdist_object, DOT_CDIST) + return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) def remote_object_dir(self, cdist_object): """Returns the remote full path to the object (including .cdist)""" @@ -260,17 +260,17 @@ class Cdist: def object_parameter_dir(self, cdist_object): """Returns the dir to the object parameter""" - return os.path.join(object_dir(cdist_object), "parameter") + return os.path.join(self.object_dir(cdist_object), "parameter") def remote_object_parameter_dir(self, cdist_object): """Returns the remote dir to the object parameter""" - return os.path.join(remote_object_dir(cdist_object), "parameter") + return os.path.join(self.remote_object_dir(cdist_object), "parameter") def list_objects(self, starting_point = False): """Return list of existing objects""" if not starting_point: - starting_point = self.object_dir + starting_point = self.object_base_dir object_paths = self.list_object_paths(starting_point) objects = [] @@ -290,9 +290,12 @@ class Cdist: return os.path.join(REMOTE_TYPE_DIR, type, "explorer") def transfer_object_parameter(self, cdist_object): - self.transfer_dir(object_dir(cdist_object), ) + """Transfer the object parameter to the remote destination""" + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) def transfer_global_explorers(self): + """Transfer the global explorers""" self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) def transfer_type_explorers(self, type): @@ -352,8 +355,9 @@ class Cdist: cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - # FIXME: need to transfer object before as well! - cmd.append("__object=" + self.remote_type_explorer_dir(type)) + # Need to transfer at least the parameters for objects to be useful + self.transfer_object_parameter(cdist_object) + cmd.append("__object=" + self.remote_object_dir(cdist_object)) explorers = self.list_type_explorers(type) for explorer in explorers: From 14ac0e3ad166cc90a947d88e2b120f3746cd67b5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 20:53:09 +0200 Subject: [PATCH 080/166] introduce remote_mkdir() Signed-off-by: Nico Schottelius --- bin/cdist | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index b2f10ba1..30af2827 100755 --- a/bin/cdist +++ b/bin/cdist @@ -139,6 +139,10 @@ class Cdist: log.error(*args) sys.exit(1) + def remote_mkdir(directory): + """Create directory on remote side""" + self.run_or_fail(["mkdir", "-p", directory], remote=True) + def remote_cat(filename): cmd = self.remote_prefix cmd.append("cat") @@ -234,14 +238,11 @@ class Cdist: for content in os.listdir(starting_point): full_path = os.path.join(starting_point, content) - print(full_path) if os.path.isdir(full_path): - log.debug("Recursing for %s", full_path) object_paths.extend(self.list_object_paths(starting_point = full_path)) # Directory contains .cdist -> is an object if content == DOT_CDIST: - log.debug("Adding Object Path %s", starting_point) object_paths.append(starting_point) return object_paths @@ -314,7 +315,7 @@ class Cdist: # Only continue, if there is at least the directory if os.path.isdir(src): # Ensure that the path exists - self.run_or_fail(["mkdir", "-p", remote_base], remote=True) + self.remote_mkdir(remote_base) self.transfer_dir(src, dst) From 99268591c2e3eddbc49bb95f5b2fa31b7f8bd42f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Sep 2011 20:54:17 +0200 Subject: [PATCH 081/166] make more use of run_or_fail Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 30af2827..72b7a752 100755 --- a/bin/cdist +++ b/bin/cdist @@ -139,18 +139,13 @@ class Cdist: log.error(*args) sys.exit(1) - def remote_mkdir(directory): + def remote_mkdir(self, directory): """Create directory on remote side""" self.run_or_fail(["mkdir", "-p", directory], remote=True) def remote_cat(filename): - cmd = self.remote_prefix - cmd.append("cat") - cmd.append(filename) - try: - subprocess.call(cmd) - except subprocess.CalledProcessError: - log.error("Remote cat failed") + """Use cat on the remote side for output""" + self.run_or_fail(["cat", filename], remote=True) def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want From 232618a675fc90accea25646171a9db30b54a2b9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:01:16 +0200 Subject: [PATCH 082/166] make manifest run more generic Signed-off-by: Nico Schottelius --- bin/cdist | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 72b7a752..4f9e95de 100755 --- a/bin/cdist +++ b/bin/cdist @@ -287,6 +287,10 @@ class Cdist: def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards self.transfer_dir(self.object_parameter_dir(cdist_object), self.remote_object_parameter_dir(cdist_object)) @@ -375,8 +379,13 @@ class Cdist: # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) + # def run_object_manifest(self, cdist_object): def run_initial_manifest(self): """Run the initial manifest""" + self.run_manifest(self.initial_manifest) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" log.info("Running the initial manifest") env = os.environ.copy() env['PATH'] = self.bin_dir + ":" + env['PATH'] @@ -390,9 +399,11 @@ class Cdist: env['__cdist_local_base_dir'] = self.temp_dir env['__cdist_manifest'] = self.initial_manifest - self.shell_run_or_debug_fail(self.initial_manifest, - [self.initial_manifest], - env=env) + # Other environment stuff + if extra_env: + env.update(extra_env) + + self.shell_run_or_debug_fail(manifest, [manifest], env=env) def deploy_to(self): From 8dff399c9654b8983e2997f483253e162c839eb3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:09:57 +0200 Subject: [PATCH 083/166] reloop until no new objects are created anyomer Signed-off-by: Nico Schottelius --- bin/cdist | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 4f9e95de..505bb012 100755 --- a/bin/cdist +++ b/bin/cdist @@ -379,11 +379,14 @@ class Cdist: # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) - # def run_object_manifest(self, cdist_object): def run_initial_manifest(self): """Run the initial manifest""" self.run_manifest(self.initial_manifest) + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + self.run_manifest(self.initial_manifest) + def run_manifest(self, manifest, extra_env=None): """Run a manifest""" log.info("Running the initial manifest") @@ -413,10 +416,18 @@ class Cdist: self.run_global_explores() self.run_initial_manifest() + old_objects = [] objects = self.list_objects() - for cdist_object in objects: - self.run_type_explorer(cdist_object) + # Continue process until no new objects are created anymore + while not old_objects == objects: + old_objects = objects.copy() + for cdist_object in objects: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + + objects = self.list_objects() + if __name__ == "__main__": From 13e3e27679443d558ddaa9846603c4da390cfb5e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:29:51 +0200 Subject: [PATCH 084/166] changes for next Signed-off-by: Nico Schottelius --- doc/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changelog b/doc/changelog index 7e98277a..6d25f234 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,5 +1,6 @@ next: * New Type: __package_rubygem (Chase Allen James) + * __self replaced by __object_fq (or so) 1.7.1: 2011-07-26 * Documentation: Add explorers to reference From 7eb418c3ab6db117db2de036226d05da3033b1cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:40:45 +0200 Subject: [PATCH 085/166] correct env for the various stages Signed-off-by: Nico Schottelius --- bin/cdist | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 505bb012..af19a676 100755 --- a/bin/cdist +++ b/bin/cdist @@ -246,6 +246,10 @@ class Cdist: """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] + def get_object_id_from_object(self, cdist_object): + """Returns everything but the first part (i.e. object_id) of an object""" + return os.sep.join(cdist_object.split(os.sep)[1:]) + def object_dir(self, cdist_object): """Returns the full path to the object (including .cdist)""" return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) @@ -281,6 +285,10 @@ class Cdist: """Return directory that holds the explorers of a type""" return os.path.join(TYPE_DIR, type, "explorer") + def type_manifest_path(self, type): + """Return path to manifest of type""" + return os.path.join(TYPE_DIR, type, "manifest") + def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" return os.path.join(REMOTE_TYPE_DIR, type, "explorer") @@ -352,6 +360,7 @@ class Cdist: self.transfer_type_explorers(type) cmd = [] + # FIXME: __global::__object_id::__self:: cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) @@ -381,15 +390,27 @@ class Cdist: def run_initial_manifest(self): """Run the initial manifest""" - self.run_manifest(self.initial_manifest) + # FIXME: add support for __manifest:: + env = { "__manifest" : MANIFEST_DIR } + self.run_manifest(self.initial_manifest, extra_env=env) def run_type_manifest(self, cdist_object): """Run manifest for a specific object""" - self.run_manifest(self.initial_manifest) + type = self.get_type_from_object(cdist_object) + manifest = self.type_manifest_path(type) + + # FIXME: add more sensible checks for manifest + if os.path.exists(manifest): + env = { "__object" : self.object_dir(cdist_object), + "__object_id": self.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": type + } + self.run_manifest(self.initial_manifest, extra_env=env) def run_manifest(self, manifest, extra_env=None): """Run a manifest""" - log.info("Running the initial manifest") + log.info("Running manifest %s", manifest) env = os.environ.copy() env['PATH'] = self.bin_dir + ":" + env['PATH'] @@ -420,8 +441,10 @@ class Cdist: objects = self.list_objects() # Continue process until no new objects are created anymore - while not old_objects == objects: - old_objects = objects.copy() + while old_objects != objects: + log.debug("Prepare stage") + old_objects = list(objects) + # FIXME: do not rerun existing objects! for cdist_object in objects: self.run_type_explorer(cdist_object) self.run_type_manifest(cdist_object) From 4e2e731374a55827e4670885b6c4bf6948c3a9d5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:43:28 +0200 Subject: [PATCH 086/166] execute the correct manifest for objects Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index af19a676..ff114325 100755 --- a/bin/cdist +++ b/bin/cdist @@ -406,11 +406,11 @@ class Cdist: "__object_fq": cdist_object, "__type": type } - self.run_manifest(self.initial_manifest, extra_env=env) + self.run_manifest(manifest, extra_env=env) def run_manifest(self, manifest, extra_env=None): """Run a manifest""" - log.info("Running manifest %s", manifest) + log.info("Running manifest %s, env=%s", manifest, extra_env) env = os.environ.copy() env['PATH'] = self.bin_dir + ":" + env['PATH'] From bc6201102eb0b1d09f4328959b38f3608b7048ba Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:44:15 +0200 Subject: [PATCH 087/166] todo-- Signed-off-by: Nico Schottelius --- bin/cdist-deploy-to | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to index 96650f90..86e15800 100755 --- a/bin/cdist-deploy-to +++ b/bin/cdist-deploy-to @@ -43,7 +43,6 @@ __cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " # See cdist-stages(7) # -cdist-object-all "$__cdist_target_host" cdist-object-prepare cdist-object-all "$__cdist_target_host" cdist-object-run cdist-cache "$__cdist_target_host" From f5c2ae3049c48bb056fdc9570648f36a892045d1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:48:24 +0200 Subject: [PATCH 088/166] begin cleanup Signed-off-by: Nico Schottelius --- bin/cdist-cache | 39 -------------------------- bin/cdist-dir | 51 ---------------------------------- bin/cdist-run-remote | 33 ---------------------- bin/cdist-type-build-emulation | 43 ---------------------------- 4 files changed, 166 deletions(-) delete mode 100755 bin/cdist-cache delete mode 100755 bin/cdist-dir delete mode 100755 bin/cdist-run-remote delete mode 100755 bin/cdist-type-build-emulation diff --git a/bin/cdist-cache b/bin/cdist-cache deleted file mode 100755 index ee27ffb4..00000000 --- a/bin/cdist-cache +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Let's build a cconfig tree from a configuration -# And save it into the cache tree -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -u - -__cdist_target_host="$1"; shift - -# Create base to move into -mkdir -p "${__cdist_local_base_cache_dir}" - -# Now determine absolute path -__cdist_ddir="$(__cdist_host_cache_dir "$__cdist_target_host")" - -__cdist_echo info "Saving cache to $__cdist_ddir " -rm -rf "$__cdist_ddir" -mv "$__cdist_local_base_dir" "$__cdist_ddir" diff --git a/bin/cdist-dir b/bin/cdist-dir deleted file mode 100755 index 0d30e14a..00000000 --- a/bin/cdist-dir +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Push a directory to a target, both sides have the same name (i.e. explorers) -# or -# Pull a directory from a target, both sides have the same name (i.e. explorers) -# - - -. cdist-config -[ $# -eq 4 ] || __cdist_usage " " -set -ue - -__cdist_action="$1"; shift -__cdist_target_host="$1"; shift -__cdist_src_dir="$1"; shift -__cdist_dst_dir="$1"; shift - -# This will be the destination directory, so no subdirectories -# of the same name are created, if the directory is already existing -__cdist_top_dir="${__cdist_dst_dir%/*}" - -if [ "$__cdist_action" = "push" ]; then - ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "mkdir -p \"${__cdist_dst_dir}\"" - scp -qr "$__cdist_src_dir" \ - "${__cdist_remote_user}@${__cdist_target_host}:${__cdist_top_dir}" -elif [ "$__cdist_action" = "pull" ]; then - mkdir -p "${__cdist_dst_dir}" - scp -qr "${__cdist_remote_user}@${__cdist_target_host}:${__cdist_src_dir}" \ - "${__cdist_top_dir}" -else - __cdist_exit_err "Unknown action $__cdist_action" -fi diff --git a/bin/cdist-run-remote b/bin/cdist-run-remote deleted file mode 100755 index 4a4452a2..00000000 --- a/bin/cdist-run-remote +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run a cdist binary on the remote side -# - -. cdist-config -[ $# -ge 2 ] || __cdist_usage " [opts]" -set -ue - -__cdist_target_host="$1"; shift - -ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "export PATH=\"${__cdist_remote_bin_dir}:\$PATH\";" \ - "export __cdist_out_object_dir=\"$__cdist_remote_out_object_dir\";" \ - "$@" diff --git a/bin/cdist-type-build-emulation b/bin/cdist-type-build-emulation deleted file mode 100755 index 51c2f5b4..00000000 --- a/bin/cdist-type-build-emulation +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Build pseudo binaries for type emulation -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -eu - -__cdist_output_dir="$1"; shift - -__cdist_type_emulator="$__cdist_abs_mydir/cdist-type-emulator" - -if [ ! -d "${__cdist_type_dir}" ]; then - __cdist_exit_err "$__cdist_type_dir must exist and contain available types" -fi - -# Get Types -cd "${__cdist_type_dir}" -ls -1 > "${__cdist_tmp_file}" - -# Create binaries -mkdir -p "${__cdist_output_dir}" -while read type; do - ln -sf "${__cdist_type_emulator}" "${__cdist_output_dir}/${type}" -done < "${__cdist_tmp_file}" From 74c280911bb1e3bca234d72236df9a584feb29c9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 00:53:10 +0200 Subject: [PATCH 089/166] begin run_object_gencode() and import missing FIXME bits deploy_to() Signed-off-by: Nico Schottelius --- bin/cdist | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bin/cdist b/bin/cdist index ff114325..46422ce5 100755 --- a/bin/cdist +++ b/bin/cdist @@ -429,6 +429,23 @@ class Cdist: self.shell_run_or_debug_fail(manifest, [manifest], env=env) + def run_object_gencode(self, cdist_object): + """Run the gencode scripts for the object""" + log.info("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + + # FIXME: if -local, -remote... + + + # Other environment stuff + if extra_env: + env.update(extra_env) + + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" @@ -451,6 +468,17 @@ class Cdist: objects = self.list_objects() + # Now do the final steps over the existing objects + for cdist_object in objects: + # FIXME: Check requirements and execute those before + + # FIXME: to be implemented + # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" + # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + if __name__ == "__main__": From 99887fe20f3ca74d604172f83ec7a5c2498863e9 Mon Sep 17 00:00:00 2001 From: Andrew Schleifer Date: Fri, 9 Sep 2011 23:24:32 -0500 Subject: [PATCH 090/166] fix typo --- bin/cdist-quickstart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist-quickstart b/bin/cdist-quickstart index 4d89180c..e146f1a8 100755 --- a/bin/cdist-quickstart +++ b/bin/cdist-quickstart @@ -249,7 +249,7 @@ fi manifestinit="conf/manifest/init" cat << eof -I'll know setup $manifestinit, containing the following code: +I'll now setup $manifestinit, containing the following code: -------------------------------------------------------------------------------- # Every machine becomes a marker, so sysadmins know that automatic From 4a5100692ab2374fafcc7744ca3b21b855b2da5d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 10:12:01 +0200 Subject: [PATCH 091/166] add helper methods to get paths from code/gencode Signed-off-by: Nico Schottelius --- bin/cdist | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 46422ce5..9eaf39fe 100755 --- a/bin/cdist +++ b/bin/cdist @@ -266,6 +266,11 @@ class Cdist: """Returns the remote dir to the object parameter""" return os.path.join(self.remote_object_dir(cdist_object), "parameter") + def object_code_paths(self, cdist_object): + """Return paths to code scripts of object""" + return [os.path.join(object_dir(cdist_object), "code-local"), + os.path.join(object_dir(cdist_object), "code-remote")] + def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -285,6 +290,11 @@ class Cdist: """Return directory that holds the explorers of a type""" return os.path.join(TYPE_DIR, type, "explorer") + def type_gencode_paths(self, type): + """Return paths to gencode scripts of type""" + return [os.path.join(TYPE_DIR, type, "gencode-local"), + os.path.join(TYPE_DIR, type, "gencode-remote")] + def type_manifest_path(self, type): """Return path to manifest of type""" return os.path.join(TYPE_DIR, type, "manifest") @@ -390,7 +400,6 @@ class Cdist: def run_initial_manifest(self): """Run the initial manifest""" - # FIXME: add support for __manifest:: env = { "__manifest" : MANIFEST_DIR } self.run_manifest(self.initial_manifest, extra_env=env) @@ -446,6 +455,26 @@ class Cdist: self.shell_run_or_debug_fail(manifest, [manifest], env=env) + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + # FIXME: Check requirements and execute those before + requirements = list_object_requirements(cdist_object) + + for requirement in requirements: + object_run(requirement, mode=mode) + + # Find and run all available gencode scripts + if mode == "gencode": + base_path = object_dir + if mode == "code": + + # FIXME: to be implemented + # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" + # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" @@ -470,14 +499,9 @@ class Cdist: # Now do the final steps over the existing objects for cdist_object in objects: - # FIXME: Check requirements and execute those before + object_run(cdist_object, mode="gencode") + object_run(cdist_object, mode="code") - # FIXME: to be implemented - # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) From c516d8359eb5780a5b9b0f4d4c51208ef684f03c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 13:19:32 +0200 Subject: [PATCH 092/166] support __object_* in run_type_explorer() Signed-off-by: Nico Schottelius --- bin/cdist | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 9eaf39fe..b3397fb8 100755 --- a/bin/cdist +++ b/bin/cdist @@ -370,9 +370,11 @@ class Cdist: self.transfer_type_explorers(type) cmd = [] - # FIXME: __global::__object_id::__self:: - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) + cmd.append("__global =" + self.out_dir) + cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) # Need to transfer at least the parameters for objects to be useful self.transfer_object_parameter(cdist_object) @@ -465,15 +467,20 @@ class Cdist: # Find and run all available gencode scripts if mode == "gencode": - base_path = object_dir + paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) if mode == "code": + paths = self.object_code_paths(cdist_object) # FIXME: to be implemented # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) + for bin in paths: + if os.path.isfile(bin): + if mode == "gencode": + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + if mode == "code": + self.run_or_fail(manifest, [manifest], env=env) def deploy_to(self): From beafb55e12b7df60f021183b38d0d4f001ec082a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 14:44:21 +0200 Subject: [PATCH 093/166] __global not available for type explorer (should it be?) Signed-off-by: Nico Schottelius --- doc/man/cdist-reference.text.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index 90efc0f8..9ef7e9c2 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -164,7 +164,7 @@ __manifest:: Available for: initial manifest __global:: Directory that contains generic output like explorer. - Available for: initial manifest, type manifest, type explorer, type codegen + Available for: initial manifest, type manifest, type codegen __object:: Directory that contains the current object. Available for: type manifest, type explorer, type codegen From 636995f44ef76deaec2ef531f6d9ef86cb0f0119 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 14:45:59 +0200 Subject: [PATCH 094/166] add some meat to object_run() Signed-off-by: Nico Schottelius --- bin/cdist | 47 ++++++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/bin/cdist b/bin/cdist index b3397fb8..a13abcf4 100755 --- a/bin/cdist +++ b/bin/cdist @@ -372,13 +372,12 @@ class Cdist: cmd = [] cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - cmd.append("__global =" + self.out_dir) + cmd.append("__object=" + self.remote_object_dir(cdist_object)) cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) cmd.append("__object_fq=" + cdist_object) # Need to transfer at least the parameters for objects to be useful self.transfer_object_parameter(cdist_object) - cmd.append("__object=" + self.remote_object_dir(cdist_object)) explorers = self.list_type_explorers(type) for explorer in explorers: @@ -459,28 +458,29 @@ class Cdist: def object_run(self, cdist_object, mode): """Run gencode or code for an object""" - # FIXME: Check requirements and execute those before - requirements = list_object_requirements(cdist_object) + # FIXME: Check requirements and execute those before + requirements = self.list_object_requirements(cdist_object) - for requirement in requirements: - object_run(requirement, mode=mode) + for requirement in requirements: + self.object_run(requirement, mode=mode) - # Find and run all available gencode scripts - if mode == "gencode": - paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) - if mode == "code": - paths = self.object_code_paths(cdist_object) + # Find and run all available gencode scripts + if mode == "gencode": + paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) + if mode == "code": + paths = self.object_code_paths(cdist_object) - # FIXME: to be implemented - # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + # FIXME: to be implemented + # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" + # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - for bin in paths: - if os.path.isfile(bin): - if mode == "gencode": - self.shell_run_or_debug_fail(manifest, [manifest], env=env) - if mode == "code": - self.run_or_fail(manifest, [manifest], env=env) + for bin in paths: + FIXME + if os.path.isfile(bin): + if mode == "gencode": + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + if mode == "code": + self.run_or_fail(manifest, [manifest], env=env) def deploy_to(self): @@ -506,11 +506,8 @@ class Cdist: # Now do the final steps over the existing objects for cdist_object in objects: - object_run(cdist_object, mode="gencode") - object_run(cdist_object, mode="code") - - - + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") if __name__ == "__main__": parser = argparse.ArgumentParser(description='cdist ' + VERSION) From 5a765bfc761d9fda4f026376f470f8309af5786e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 15:09:09 +0200 Subject: [PATCH 095/166] codegen vs. gencode Signed-off-by: Nico Schottelius --- doc/man/cdist-reference.text.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index 9ef7e9c2..a823891b 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -164,19 +164,19 @@ __manifest:: Available for: initial manifest __global:: Directory that contains generic output like explorer. - Available for: initial manifest, type manifest, type codegen + Available for: initial manifest, type manifest, type gencode __object:: Directory that contains the current object. - Available for: type manifest, type explorer, type codegen + Available for: type manifest, type explorer, type gencode __object_id:: The type unique object id. - Available for: type manifest, type explorer, type codegen + Available for: type manifest, type explorer, type gencode __self:: The full qualified name of the current object. - Available for: type manifest, type explorer, type codegen + Available for: type manifest, type explorer, type gencode __target_host:: The host we are deploying to. - Available for: initial manifest, type manifest, type codegen + Available for: initial manifest, type manifest, type gencode __target_user:: User to use for authentication on remote host. Currently static in core. From ca58841696d969db8c62c6eafd0840a895cca520 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 15:36:21 +0200 Subject: [PATCH 096/166] Broken code - it's too hot DEBUG: Shell exec: /bin/sh -e /home/users/nico/p/cdist/conf/type/__directory/gencode-remote Traceback (most recent call last): File "/home/users/nico/p/cdist/bin/cdist", line 557, in c.deploy_to() File "/home/users/nico/p/cdist/bin/cdist", line 523, in deploy_to self.object_run(cdist_object, mode="gencode") File "/home/users/nico/p/cdist/bin/cdist", line 491, in object_run self.shell_run_or_debug_fail(bin, [bin], env=env) File "/home/users/nico/p/cdist/bin/cdist", line 164, in shell_run_or_debug_fail subprocess.check_call(*args, **kargs) File "/usr/lib/python3.2/subprocess.py", line 480, in check_call retcode = call(*popenargs, **kwargs) File "/usr/lib/python3.2/subprocess.py", line 467, in call return Popen(*popenargs, **kwargs).wait() File "/usr/lib/python3.2/subprocess.py", line 741, in __init__ restore_signals, start_new_session) File "/usr/lib/python3.2/subprocess.py", line 1174, in _execute_child for k, v in env.items()] File "/usr/lib/python3.2/subprocess.py", line 1174, in for k, v in env.items()] File "/usr/lib/python3.2/os.py", line 581, in fsencode raise TypeError("expect bytes or str, not %s" % type(filename).__name__) TypeError: expect bytes or str, not tuple Signed-off-by: Nico Schottelius --- bin/cdist | 58 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/bin/cdist b/bin/cdist index a13abcf4..b9c071b5 100755 --- a/bin/cdist +++ b/bin/cdist @@ -268,8 +268,8 @@ class Cdist: def object_code_paths(self, cdist_object): """Return paths to code scripts of object""" - return [os.path.join(object_dir(cdist_object), "code-local"), - os.path.join(object_dir(cdist_object), "code-remote")] + return [os.path.join(self.object_dir(cdist_object), "code-local"), + os.path.join(self.object_dir(cdist_object), "code-remote")] def list_objects(self, starting_point = False): """Return list of existing objects""" @@ -279,7 +279,6 @@ class Cdist: object_paths = self.list_object_paths(starting_point) objects = [] - log.debug("Paths recieved: %s", object_paths) for path in object_paths: objects.append(os.path.relpath(path, starting_point)) @@ -439,22 +438,23 @@ class Cdist: self.shell_run_or_debug_fail(manifest, [manifest], env=env) - def run_object_gencode(self, cdist_object): - """Run the gencode scripts for the object""" - log.info("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() + def list_object_requirements(self, cdist_object): + """Return list of requirements for specific object""" + file=os.path.join(self.object_dir(cdist_object), "require") - env['__target_host'] = self.target_host - env['__global'] = self.out_dir + if os.path.isfile(file): + file_fd = open(file, "r") + requirements = file_fd.readlines() + file_fd.close() - # FIXME: if -local, -remote... - + # Remove \n from all lines + requirements = map(lambda s: s.strip(), requirements) - # Other environment stuff - if extra_env: - env.update(extra_env) + log.debug(requirements) + else: + requirements = [] - self.shell_run_or_debug_fail(manifest, [manifest], env=env) + return requirements def object_run(self, cdist_object, mode): """Run gencode or code for an object""" @@ -470,18 +470,32 @@ class Cdist: if mode == "code": paths = self.object_code_paths(cdist_object) - # FIXME: to be implemented - # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + env["__object"] = self.object_dir(cdist_object), + env["__object_id"] = self.get_object_id_from_object(cdist_object), + env["__object_fq"] = cdist_object for bin in paths: - FIXME + log.debug("object/bin: %s", bin) if os.path.isfile(bin): + # FIXME: to be implemented + # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" if mode == "gencode": - self.shell_run_or_debug_fail(manifest, [manifest], env=env) + # __global::__object::__object_id::__self::__target_host:: + # + self.shell_run_or_debug_fail(bin, [bin], env=env) if mode == "code": - self.run_or_fail(manifest, [manifest], env=env) - + # FIXME: to be implemented + # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" + log.debug("ERROR NOT IMPLEMENTED") + sys.exit(3) + # FIXME: transfer code and add remote path! + self.run_or_fail([bin], remote=True) def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" From 151022ffd1ef322b2e7fbeb4057630ab42d08f17 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Sep 2011 23:46:17 +0200 Subject: [PATCH 097/166] support file transfer and begin support for non root logins Signed-off-by: Nico Schottelius --- bin/cdist | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index b9c071b5..f34812d9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -97,7 +97,7 @@ def banner(): class Cdist: """Cdist main class to hold arbitrary data""" - def __init__(self, target_host, initial_manifest=False): + def __init__(self, target_host, initial_manifest=False, remote_user="root"): self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] @@ -120,6 +120,8 @@ class Cdist: # List of type explorers transferred self.type_explorers_transferred = {} + self.remote_user = remote_user + # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest @@ -192,7 +194,17 @@ class Cdist: def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, "root@" + self.target_host + ":" + destination]) + self.run_or_fail(["scp", "-qr", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def transfer_file(self, source, destination): + """Transfer file""" + self.run_or_fail(["scp", "-q", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" @@ -311,6 +323,16 @@ class Cdist: self.transfer_dir(self.object_parameter_dir(cdist_object), self.remote_object_parameter_dir(cdist_object)) + def transfer_object_code(self, cdist_object): + FIXME + """Transfer the object code to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) + def transfer_global_explorers(self): """Transfer the global explorers""" self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) From c147c74e1ed465e15e916b0af73cdb30d69a17c2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 13:11:45 +0200 Subject: [PATCH 098/166] begin to redo object_run() Signed-off-by: Nico Schottelius --- bin/cdist | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index f34812d9..eb1e9112 100755 --- a/bin/cdist +++ b/bin/cdist @@ -330,7 +330,7 @@ class Cdist: self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) # Synchronise parameter dir afterwards - self.transfer_dir(self.object_parameter_dir(cdist_object), + self.transfer_file(self.object_code_path(cdist_object), self.remote_object_parameter_dir(cdist_object)) def transfer_global_explorers(self): @@ -512,6 +512,9 @@ class Cdist: # self.shell_run_or_debug_fail(bin, [bin], env=env) if mode == "code": + local_dir = object_dir(cdist_object) + remote_dir = remote_object_dir(cdist_object) + # FIXME: to be implemented # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" log.debug("ERROR NOT IMPLEMENTED") From c9c808a732e66210d2780fbfc533eec263bf660e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 13:55:40 +0200 Subject: [PATCH 099/166] cleanp Signed-off-by: Nico Schottelius --- bin/cdist | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/bin/cdist b/bin/cdist index eb1e9112..7c2de358 100755 --- a/bin/cdist +++ b/bin/cdist @@ -152,6 +152,7 @@ class Cdist: def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called + print(args) args[0][:0] = [ "/bin/sh", "-e" ] if "remote" in kargs: @@ -161,6 +162,8 @@ class Cdist: del kargs["remote"] + print(args) + print(*args) log.debug("Shell exec: " + " ".join(*args)) try: subprocess.check_call(*args, **kargs) @@ -472,7 +475,7 @@ class Cdist: # Remove \n from all lines requirements = map(lambda s: s.strip(), requirements) - log.debug(requirements) + log.debug("Requirements for %s: %s", cdist_object, requirements) else: requirements = [] @@ -480,18 +483,13 @@ class Cdist: def object_run(self, cdist_object, mode): """Run gencode or code for an object""" - # FIXME: Check requirements and execute those before + log.debug("Running %s from %s", mode, cdist_object) requirements = self.list_object_requirements(cdist_object) for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) self.object_run(requirement, mode=mode) - # Find and run all available gencode scripts - if mode == "gencode": - paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) - if mode == "code": - paths = self.object_code_paths(cdist_object) - # # Setup env Variable: # @@ -502,26 +500,26 @@ class Cdist: env["__object_id"] = self.get_object_id_from_object(cdist_object), env["__object_fq"] = cdist_object - for bin in paths: - log.debug("object/bin: %s", bin) - if os.path.isfile(bin): - # FIXME: to be implemented - # cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - if mode == "gencode": - # __global::__object::__object_id::__self::__target_host:: - # + if mode == "gencode": + paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) + for bin in paths: + if os.path.isfile(bin): + print(bin) self.shell_run_or_debug_fail(bin, [bin], env=env) - if mode == "code": - local_dir = object_dir(cdist_object) - remote_dir = remote_object_dir(cdist_object) - # FIXME: to be implemented - # cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - log.debug("ERROR NOT IMPLEMENTED") - sys.exit(3) - # FIXME: transfer code and add remote path! - self.run_or_fail([bin], remote=True) + if mode == "code": +# paths = self.object_code_paths(cdist_object) + local_dir = self.object_dir(cdist_object) + remote_dir = self.remote_object_dir(cdist_object) + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + self.run_or_fail([bin], remote=False) + + if os.path.isfile(os.path.join(local_dir, "code-remote")): + remote_code = os.path.join(remote_dir, "code-remote") + self.run_or_fail([remote_code], remote=True) + def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to host " + self.target_host) @@ -542,9 +540,11 @@ class Cdist: self.run_type_manifest(cdist_object) objects = self.list_objects() - + + log.debug("Actual run objects") # Now do the final steps over the existing objects for cdist_object in objects: + log.debug("Run object: %s", cdist_object) self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") From 5702706adf5badf120fc60d742164034ce934a76 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 14:11:54 +0200 Subject: [PATCH 100/166] fix not so obvious tuple/str error (trailing comma from c&p) Signed-off-by: Nico Schottelius --- bin/cdist | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7c2de358..7e6f6ddf 100755 --- a/bin/cdist +++ b/bin/cdist @@ -152,7 +152,6 @@ class Cdist: def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called - print(args) args[0][:0] = [ "/bin/sh", "-e" ] if "remote" in kargs: @@ -162,9 +161,8 @@ class Cdist: del kargs["remote"] - print(args) - print(*args) - log.debug("Shell exec: " + " ".join(*args)) + log.debug("Shell exec cmd: %s", args) + log.debug("Shell exec env: %s", kargs['env']) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: @@ -496,8 +494,8 @@ class Cdist: env = os.environ.copy() env['__target_host'] = self.target_host env['__global'] = self.out_dir - env["__object"] = self.object_dir(cdist_object), - env["__object_id"] = self.get_object_id_from_object(cdist_object), + env["__object"] = self.object_dir(cdist_object) + env["__object_id"] = self.get_object_id_from_object(cdist_object) env["__object_fq"] = cdist_object if mode == "gencode": From c78ce344e73578eeba144b7a6eff6419df2e4e0e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 14:30:05 +0200 Subject: [PATCH 101/166] remove debug by default, switch to infolevel and verify -d works Signed-off-by: Nico Schottelius --- bin/cdist | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7e6f6ddf..157b8d13 100755 --- a/bin/cdist +++ b/bin/cdist @@ -81,7 +81,7 @@ VERSION = "2.0.0" # -logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s') +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() # List types @@ -406,6 +406,9 @@ class Cdist: remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) output_fd.close() @@ -431,18 +434,19 @@ class Cdist: type = self.get_type_from_object(cdist_object) manifest = self.type_manifest_path(type) + log.debug("%s: Running %s", cdist_object, manifest) # FIXME: add more sensible checks for manifest if os.path.exists(manifest): - env = { "__object" : self.object_dir(cdist_object), + env = { "__object" : self.object_dir(cdist_object), "__object_id": self.get_object_id_from_object(cdist_object), "__object_fq": cdist_object, - "__type": type + "__type": type } self.run_manifest(manifest, extra_env=env) def run_manifest(self, manifest, extra_env=None): """Run a manifest""" - log.info("Running manifest %s, env=%s", manifest, extra_env) + log.debug("Running manifest %s, env=%s", manifest, extra_env) env = os.environ.copy() env['PATH'] = self.bin_dir + ":" + env['PATH'] @@ -502,7 +506,6 @@ class Cdist: paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) for bin in paths: if os.path.isfile(bin): - print(bin) self.shell_run_or_debug_fail(bin, [bin], env=env) if mode == "code": @@ -546,6 +549,8 @@ class Cdist: self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") + log.info("Finished run of %s", self.target_host) + if __name__ == "__main__": parser = argparse.ArgumentParser(description='cdist ' + VERSION) parser.add_argument('host', nargs='*', help='one or more hosts to operate on') From 6bd823995fff95414060ef4eca841f74733ecff4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 17:20:07 +0200 Subject: [PATCH 102/166] restore mistakenly deleted file Signed-off-by: Nico Schottelius --- conf/type/__directory/explorer/exists | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 conf/type/__directory/explorer/exists diff --git a/conf/type/__directory/explorer/exists b/conf/type/__directory/explorer/exists new file mode 100755 index 00000000..f8b85671 --- /dev/null +++ b/conf/type/__directory/explorer/exists @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Check whether file exists or not +# + +destination="/$__object_id" + +if [ -e "$destination" ]; then + echo yes +else + echo no +fi From 30a52cab0fe45d932d822ea7a28b6dfbfe5ef98a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:17:20 +0200 Subject: [PATCH 103/166] begin to save code and make it executable Signed-off-by: Nico Schottelius --- bin/cdist | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index 157b8d13..e76ecfc7 100755 --- a/bin/cdist +++ b/bin/cdist @@ -25,6 +25,7 @@ import logging import os import subprocess import shutil +import stat import sys import tempfile @@ -93,7 +94,6 @@ def banner(): print(BANNER) - class Cdist: """Cdist main class to hold arbitrary data""" @@ -235,6 +235,8 @@ class Cdist: else: list = [] + log.debug("Explorers for %s in %s: %s", type, dir, list) + return list def list_object_paths(self, starting_point = False): @@ -341,7 +343,7 @@ class Cdist: def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" if type in self.type_explorers_transferred: - log.debug("Skipping retransfer for %s", type) + log.debug("Skipping retransfer for explorers of %s", type) return else: # Do not retransfer @@ -506,7 +508,19 @@ class Cdist: paths = self.type_gencode_paths(self.get_type_from_object(cdist_object)) for bin in paths: if os.path.isfile(bin): - self.shell_run_or_debug_fail(bin, [bin], env=env) + # omit "gen" from gencode and + outfile=os.path.join(self.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + + a = outfile_fd + b = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR + print(type(a), type(b)) + + os.fchmod(outfile_fd, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + outfile_fd.close() if mode == "code": # paths = self.object_code_paths(cdist_object) From 4e4d648593305836944dfa10811054df7ae7427e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:18:30 +0200 Subject: [PATCH 104/166] fchmod() does not work on object returned by open(), so use chmod Signed-off-by: Nico Schottelius --- bin/cdist | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index e76ecfc7..09cb9d45 100755 --- a/bin/cdist +++ b/bin/cdist @@ -514,14 +514,10 @@ class Cdist: outfile_fd = open(outfile, "w") self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - - a = outfile_fd - b = stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR - print(type(a), type(b)) - - os.fchmod(outfile_fd, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) outfile_fd.close() + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + if mode == "code": # paths = self.object_code_paths(cdist_object) local_dir = self.object_dir(cdist_object) From 34ae8c292689780db3c18047bcd9d9493cf4f9be Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:41:39 +0200 Subject: [PATCH 105/166] make code execution happen in the end Signed-off-by: Nico Schottelius --- bin/cdist | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 09cb9d45..1fd3e116 100755 --- a/bin/cdist +++ b/bin/cdist @@ -516,7 +516,13 @@ class Cdist: self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) outfile_fd.close() - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == 0: + os.unlink(outfile) + else: + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) if mode == "code": # paths = self.object_code_paths(cdist_object) @@ -527,9 +533,12 @@ class Cdist: if os.path.isfile(bin): self.run_or_fail([bin], remote=False) - if os.path.isfile(os.path.join(local_dir, "code-remote")): - remote_code = os.path.join(remote_dir, "code-remote") - self.run_or_fail([remote_code], remote=True) + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.transfer_file(local_remote_code, remote_remote_code) + self.run_or_fail([remote_remote_code], remote=True) def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" From e56bbfea9ac0aba51f28200848d6bfa48ed28b49 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:45:54 +0200 Subject: [PATCH 106/166] less verbose output in default case Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 1fd3e116..4267b3fa 100755 --- a/bin/cdist +++ b/bin/cdist @@ -415,7 +415,7 @@ class Cdist: output_fd.close() def init_deploy(self): - log.info("Creating clean directory structure") + log.debug("Creating clean directory structure") # Ensure there is no old stuff, neither local nor remote # remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) @@ -542,7 +542,7 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to host " + self.target_host) + log.info("Deploying to " + self.target_host) self.init_deploy() self.run_global_explores() self.run_initial_manifest() From f9f874deed99b3ed7fe3488dc65d29f67e33e620 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 20:49:46 +0200 Subject: [PATCH 107/166] new features in 2.0.0 Signed-off-by: Nico Schottelius --- doc/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/changelog b/doc/changelog index 6d25f234..50409cac 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,6 +1,7 @@ -next: +2.0.0: * New Type: __package_rubygem (Chase Allen James) * __self replaced by __object_fq (or so) + * Rewrote cdist in Python 1.7.1: 2011-07-26 * Documentation: Add explorers to reference From eb2512d4a2f176742102e8fd4750ade1e02f215d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:15:58 +0200 Subject: [PATCH 108/166] add timing information and setup remote = false by default Signed-off-by: Nico Schottelius --- bin/cdist | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 4267b3fa..1e6fc45b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -21,6 +21,7 @@ # import argparse +import datetime import logging import os import subprocess @@ -154,6 +155,7 @@ class Cdist: # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] + remote = false if "remote" in kargs: if kargs["remote"]: args[0][:0] = self.remote_prefix @@ -543,6 +545,7 @@ class Cdist: def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() self.init_deploy() self.run_global_explores() self.run_initial_manifest() @@ -568,7 +571,11 @@ class Cdist: self.object_run(cdist_object, mode="gencode") self.object_run(cdist_object, mode="code") - log.info("Finished run of %s", self.target_host) + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) if __name__ == "__main__": parser = argparse.ArgumentParser(description='cdist ' + VERSION) From be2efbbb0db75fc0a437e4c81b79f355a4ce9b35 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:16:42 +0200 Subject: [PATCH 109/166] give me a big F Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 1e6fc45b..9f713ee3 100755 --- a/bin/cdist +++ b/bin/cdist @@ -155,7 +155,7 @@ class Cdist: # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] - remote = false + remote = False if "remote" in kargs: if kargs["remote"]: args[0][:0] = self.remote_prefix From b214fe54ddbd141c5d4a18d43c27ff4ea8a073b9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:22:51 +0200 Subject: [PATCH 110/166] add old benchmarks Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-04-27.benchmark | 56 ++++++++++++++++++++++++++ doc/dev/logs/2011-04-27.benchmark.dash | 9 +++++ 2 files changed, 65 insertions(+) create mode 100644 doc/dev/logs/2011-04-27.benchmark create mode 100644 doc/dev/logs/2011-04-27.benchmark.dash diff --git a/doc/dev/logs/2011-04-27.benchmark b/doc/dev/logs/2011-04-27.benchmark new file mode 100644 index 00000000..6f1f1e21 --- /dev/null +++ b/doc/dev/logs/2011-04-27.benchmark @@ -0,0 +1,56 @@ +% x200 +data = [ 1 104; 2 129; 6 249 ; 25 1267 ] + +% x201, 4.2.2.4 dns, eth +data = [ 1 143; 2 159; 3 198; 4 244; 5 299; 6 350; 7 435; 8 429 ]; + +% x201, von zuhause aus +data2 = [ 226 242 275 296 306 357 403 400 409 685 617 672 ] +plot(0:size(data)(2)-1, data) + + +hold off; +plot(data(:,1), data(:,2)) +% per host time: +data(:,2)' ./ data(:,1)' +hold on; +plot(data(:,1), data(:,2)' ./ data(:,1)') + + +Testing on + Intel(R) Core(TM)2 Duo CPU P8400 @ 2.26GHz + 4 GiB RAM + Intel Gbit Nic + +- 169 objects, all done (i.e. rerun) +- runs are cpu bound + +1 host: + +core: cdist 1.6.2: Successfully finished run +Run: 104s (1 minutes) + +2 hosts: + +[13:48] kr:cdist-nutzung% mytime cdist-mass-deploy -p ikq03.ethz.ch ikq04.ethz.ch +Run: 129s (2 minutes) + +6 hosts: (7 specified, but ikq01.ethz.ch is dead) +cdist-mass-deploy -p ikq01.ethz.ch ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch ikq05.ethz.ch ikq06.ethz.ch ikq07.ethz.ch +Run: 294s (4 minutes) + +25 hosts + +[13:58] kr:cdist-nutzung% mytime cdist-mass-deploy -p $(sed 's/^root@//' ~/.dsh/group/ikr) + +Run: 723s (12 minutes) + + +25 hosts without proxy command now: + +Run: 1267s (21 minutes) +Run: 1212s (20 minutes) + + +[14:11] kr:cdist-nutzung% mytime cdist-mass-deploy -p $(sed 's/^root@//' ~/.dsh/group/ikr) + diff --git a/doc/dev/logs/2011-04-27.benchmark.dash b/doc/dev/logs/2011-04-27.benchmark.dash new file mode 100644 index 00000000..635062e8 --- /dev/null +++ b/doc/dev/logs/2011-04-27.benchmark.dash @@ -0,0 +1,9 @@ +Run: 78s (1 minutes) +[13:25] kr:cdist-nutzung% mytime cdist-deploy-to ikq02.ethz.ch + +% mit dash, eth, x201, Mit Apr 27 13:41:49 CEST 2011 +data = [ 0 73 77 89 107 130 151 180 197 228 251 260 199 295 335 276 ] +plot(0:size(data2)(2)-1, data2) + + + From 5429022d617b9f0a989f2eda454527761b8e6165 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:23:10 +0200 Subject: [PATCH 111/166] add old debug Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-09-08 | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/dev/logs/2011-09-08 diff --git a/doc/dev/logs/2011-09-08 b/doc/dev/logs/2011-09-08 new file mode 100644 index 00000000..42413285 --- /dev/null +++ b/doc/dev/logs/2011-09-08 @@ -0,0 +1,4 @@ +Debugging cdist: + +[0:13] kr:cdist-nutzung% ./local/update-local-core && __cdist_debug=1 __cdist_local_base_dir=/tmp/cdist cdist-deploy-to ikq04.ethz.ch + From c5d88435f13e46a0b8066ce9cf904fafa7696050 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:24:23 +0200 Subject: [PATCH 112/166] add python link Signed-off-by: Nico Schottelius --- bin/cdist-code-run | 50 ----------------------- bin/cdist-deploy-to | 49 ----------------------- bin/cdist-env | 33 --------------- bin/cdist-explorer-run-global | 43 -------------------- bin/cdist-manifest-run | 56 -------------------------- bin/cdist-manifest-run-init | 34 ---------------- bin/cdist-mass-deploy | 75 ----------------------------------- bin/cdist-object-all | 56 -------------------------- bin/cdist-object-code-run | 38 ------------------ bin/cdist-object-explorer-run | 67 ------------------------------- bin/cdist-object-gencode | 62 ----------------------------- bin/cdist-object-gencode-run | 38 ------------------ bin/cdist-object-manifest-run | 58 --------------------------- bin/cdist-object-prepare | 47 ---------------------- bin/cdist-object-push | 35 ---------------- bin/cdist-object-run | 65 ------------------------------ bin/cdist-remote-explorer-run | 68 ------------------------------- bin/cdist.py | 1 + 18 files changed, 1 insertion(+), 874 deletions(-) delete mode 100755 bin/cdist-code-run delete mode 100755 bin/cdist-deploy-to delete mode 100755 bin/cdist-env delete mode 100755 bin/cdist-explorer-run-global delete mode 100755 bin/cdist-manifest-run delete mode 100755 bin/cdist-manifest-run-init delete mode 100755 bin/cdist-mass-deploy delete mode 100755 bin/cdist-object-all delete mode 100755 bin/cdist-object-code-run delete mode 100755 bin/cdist-object-explorer-run delete mode 100755 bin/cdist-object-gencode delete mode 100755 bin/cdist-object-gencode-run delete mode 100755 bin/cdist-object-manifest-run delete mode 100755 bin/cdist-object-prepare delete mode 100755 bin/cdist-object-push delete mode 100755 bin/cdist-object-run delete mode 100755 bin/cdist-remote-explorer-run create mode 120000 bin/cdist.py diff --git a/bin/cdist-code-run b/bin/cdist-code-run deleted file mode 100755 index 3d7499bf..00000000 --- a/bin/cdist-code-run +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# This binary is executed on the remote side to execute code -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -ue - -__cdist_object_self="$1"; shift -__cdist_code_type="$1"; shift - -if [ ! -d "$(__cdist_object_dir "$__cdist_object_self")" ]; then - __cdist_exit_err "Object undefined" -fi - -__cdist_code="$(__cdist_object_code "$__cdist_object_self" "${__cdist_code_type}")" - -__cdist_echo info "Checking code-${__cdist_code_type}" - -if [ -e "$__cdist_code" ]; then - if [ -f "$__cdist_code" ]; then - if [ -x "$__cdist_code" ]; then - __cdist_echo info "Executing code-${__cdist_code_type}" - __cdist_exec_fail_on_error "$__cdist_code" - else - __cdist_exit_err "$__cdist_code exists, but is not executable." - fi - else - __cdist_exit_err "$__cdist_code exists, but is not a file." - fi -fi diff --git a/bin/cdist-deploy-to b/bin/cdist-deploy-to deleted file mode 100755 index 86e15800..00000000 --- a/bin/cdist-deploy-to +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Deploy configuration to a host -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -eu - -# Kill children on interrupt - only in interactive scripts -trap __cdist_kill_on_interrupt INT TERM - -__cdist_target_host="$1" - -# Make target host available for non-core -export $__cdist_name_var_target_host="$__cdist_target_host" -export $__cdist_name_var_target_user="$__cdist_remote_user" - -# Export variables for core, which others do not reset -export __cdist_local_base_dir - -__cdist_echo info "cdist $__cdist_version: Configuring $__cdist_target_host " - -################################################################################ -# See cdist-stages(7) -# - -cdist-object-all "$__cdist_target_host" cdist-object-run -cdist-cache "$__cdist_target_host" - -__cdist_echo info "cdist $__cdist_version: Successfully finished run" diff --git a/bin/cdist-env b/bin/cdist-env deleted file mode 100755 index 2aac5e6b..00000000 --- a/bin/cdist-env +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Setup environment for use with cdist - must be standalone! -# - -export PATH="$(cd "${0%/*}" && pwd -P):$PATH" -export MANPATH="$(cd "${0%/*}/../doc/man" && pwd -P):$MANPATH" - -if [ "$(echo ${SHELL##*/} | grep 'csh$')" ]; then - echo setenv PATH $PATH \; - echo setenv MANPATH $MANPATH -else - echo export PATH=$PATH - echo export MANPATH=$MANPATH -fi diff --git a/bin/cdist-explorer-run-global b/bin/cdist-explorer-run-global deleted file mode 100755 index b0c024f2..00000000 --- a/bin/cdist-explorer-run-global +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Copy & run the global explorers, i.e. not bound to types -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -ue - -__cdist_target_host="$1"; shift - -__cdist_echo info "Running global explorers " - -# copy the explorers -cdist-dir push "$__cdist_target_host" \ - "${__cdist_explorer_dir}" "${__cdist_remote_explorer_dir}" - -# run the initial explorers remotely -cdist-run-remote "${__cdist_target_host}" cdist-remote-explorer-run \ - "$__cdist_name_var_explorer" "$__cdist_remote_explorer_dir" \ - "$__cdist_remote_out_explorer_dir" - -# retrieve the results -cdist-dir pull "$__cdist_target_host" \ - "${__cdist_remote_out_explorer_dir}" "${__cdist_out_explorer_dir}" diff --git a/bin/cdist-manifest-run b/bin/cdist-manifest-run deleted file mode 100755 index d4ea18bb..00000000 --- a/bin/cdist-manifest-run +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Let's build a cconfig tree from a configuration -# And save it into the cache tree -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -u - -__cdist_target_host="$1"; shift -__cdist_manifest="$1"; shift - -################################################################################ -# Export information for cdist-type-emulator or manifest -# - -# Config dir should not get reset - FIXME: why did I do this? -export __cdist_conf_dir - -# Used to record the source in the object -export __cdist_manifest - -# Export information for manifests - __cdist_out_dir comes from cdist-config -export __global="$__cdist_out_dir" - -################################################################################ -# The actual run -# - -# Ensure binaries exist and are up-to-date -cdist-type-build-emulation "${__cdist_out_type_bin_dir}" \ - || __cdist_exit_err "Failed to build type emulation binaries" - -# prepend our path, so all cdist tools come before other tools -export PATH="${__cdist_out_type_bin_dir}:$PATH" - -__cdist_exec_fail_on_error "${__cdist_manifest}" diff --git a/bin/cdist-manifest-run-init b/bin/cdist-manifest-run-init deleted file mode 100755 index 28acc623..00000000 --- a/bin/cdist-manifest-run-init +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Let's build a cconfig tree from a configuration -# And save it into the cache tree -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -e - -__cdist_target_host="$1"; shift - -eval export $__cdist_name_var_manifest=\"\$__cdist_manifest_dir\" - -__cdist_echo info "Running initial manifest for $__cdist_target_host " -cdist-manifest-run "$__cdist_target_host" "$__cdist_manifest_init" diff --git a/bin/cdist-mass-deploy b/bin/cdist-mass-deploy deleted file mode 100755 index c2b6d6a5..00000000 --- a/bin/cdist-mass-deploy +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Deploy configuration to many hosts -# - -. cdist-config -[ $# -ge 1 ] || __cdist_usage "[-p] [target host ]" -set -u - -# Kill children on interrupt - only in interactive scripts -trap __cdist_kill_on_interrupt INT TERM - -filter() -{ - awk -v host=$1 '{ print "[" host "] " $0 }' -} - -parallel="" -if [ "$1" = "-p" ]; then - parallel=yes - shift -fi - -i=0 -while [ $# -gt 0 ]; do - if [ "$parallel" ]; then - cdist-deploy-to "$1" 2>&1 | filter "$1" & - # Record pid and host for use later - i=$((i+1)) - eval pid_$i=$! - eval host_$i=\$1 - else - cdist-deploy-to "$1" 2>&1 | filter "$1" - fi - shift -done - -e=0 -if [ "$parallel" ]; then - __cdist_echo info "Waiting for cdist-deploy-to jobs to finish" - while [ "$i" -gt 0 ]; do - eval pid=\$pid_$i - wait "$pid" - if [ $? -ne 0 ]; then - e=$((e+1)) - eval e_host_$e=\$host_$i - fi - i=$((i-1)) - done -fi - -# Display all failed hosts after all runs are done, so the sysadmin gets them -while [ "$e" -gt 0 ]; do - eval host=\$host_$e - __cdist_echo error "Configuration of host $host failed." - e=$((e-1)) -done diff --git a/bin/cdist-object-all b/bin/cdist-object-all deleted file mode 100755 index 391c9cc7..00000000 --- a/bin/cdist-object-all +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run the given command for each created object. -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -eu - -__cdist_target_host="$1"; shift -__cdist_command="$1"; shift - -__cdist_objects="$__cdist_tmp_dir/objects" - -# Ensure object dir exists, so marker can be created -mkdir -p "${__cdist_out_object_dir}" - -# Loop until we do not create new objects anymore -# which is equal to all objects have been run -touch "$__cdist_objects_created" -while [ -f "$__cdist_objects_created" ]; do - # Assume we're done after this run - rm "$__cdist_objects_created" - - # Get listing of objects - __cdist_object_list "$__cdist_out_object_dir" > "$__cdist_objects" - - # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP - while read __cdist_object; do - set -- "$@" "$__cdist_object" - done < "$__cdist_objects" - - while [ $# -gt 0 ]; do - __cdist_object="$1"; shift - $__cdist_command "$__cdist_target_host" "$__cdist_object" - done -done diff --git a/bin/cdist-object-code-run b/bin/cdist-object-code-run deleted file mode 100755 index fa63aaba..00000000 --- a/bin/cdist-object-code-run +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Exec the code for the given object locally and remote -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -e - -__cdist_target_host="$1"; shift -__cdist_object="$1"; shift - -# Code local -export __cdist_out_object_dir="$__cdist_out_object_dir" -cdist-code-run "$__cdist_object" "${__cdist_name_gencode_local}" - -# Code remote -cdist-run-remote "$__cdist_target_host" \ - "cdist-code-run" "$__cdist_object" "${__cdist_name_gencode_remote}" diff --git a/bin/cdist-object-explorer-run b/bin/cdist-object-explorer-run deleted file mode 100755 index b65c5cc1..00000000 --- a/bin/cdist-object-explorer-run +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run the explorers for the given object on the target host. -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -eu - -__cdist_target_host="$1"; shift -__cdist_object_self="$1"; shift - -__cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" -__cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" - -# Check if type of object has >= 1 explorer -__cdist_has_explorer="$(__cdist_type_has_explorer "$__cdist_type")" -# Run the type explorers for the current object if any -if [ "$__cdist_has_explorer" ]; then - if ! __cdist_type_explorer_pushed "$__cdist_type"; then - src_dir="$(__cdist_type_explorer_dir "$__cdist_type")" - dst_dir="$(__cdist_remote_type_explorer_dir "$__cdist_type")" - __cdist_echo info "Transfering explorers for $__cdist_type " - cdist-dir push "$__cdist_target_host" "$src_dir" "$dst_dir" - __cdist_type_explorer_pushed_add "$__cdist_type" - fi - - __cdist_echo info "Running explorers" - # Copy object parameters - cdist-dir push "$__cdist_target_host" \ - "$(__cdist_object_parameter_dir "$__cdist_object_self")" \ - "$(__cdist_remote_object_parameter_dir "$__cdist_object_self")" - - # Execute explorers - cdist-run-remote "$__cdist_target_host" \ - "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \ - "$__cdist_name_var_object_id=\"$__cdist_object_id\"" \ - "$__cdist_name_var_self=\"$__cdist_object_self\"" \ - cdist-remote-explorer-run \ - "$__cdist_name_var_type_explorer" \ - "$(__cdist_remote_type_explorer_dir "$__cdist_type")" \ - "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" - - # Copy back results - cdist-dir pull "$__cdist_target_host" \ - "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" \ - "$(__cdist_object_type_explorer_dir "$__cdist_object_self")" -fi diff --git a/bin/cdist-object-gencode b/bin/cdist-object-gencode deleted file mode 100755 index e21568a3..00000000 --- a/bin/cdist-object-gencode +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Generate code from one object (object must be relative path!) -# WARNING: OUTPUT ON STDOUT, ERRORS NEED TO BE ON STDERR! -# - -. cdist-config -[ $# -eq 3 ] || __cdist_usage "" "" "" -set -eu - -__cdist_target_host="$1"; shift -__cdist_object_self="$1"; shift -__cdist_gencode_type="$1"; shift - -__cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" -__cdist_type_gencode="$(__cdist_type_gencode "$__cdist_type" "$__cdist_gencode_type")" -__cdist_code_output="$(__cdist_object_code "$__cdist_object_self" "$__cdist_gencode_type")" - -# export variables for the gencode script -export __object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" -export __object="$(__cdist_object_dir "$__cdist_object_self")" -export __global="$__cdist_out_dir" - -if [ -x "$__cdist_type_gencode" ]; then - __cdist_exec_fail_on_error "$__cdist_type_gencode" > "$__cdist_tmp_file" -else - if [ -e "$__cdist_type_gencode" ]; then - __cdist_exit_err "$__cdist_type_gencode exists, but is not executable" - fi - - # Ensure it's empty, if there is no gencode - : > "$__cdist_tmp_file" -fi - -# Only create code, if gencode created output -if [ "$(wc -l < "$__cdist_tmp_file")" -gt 0 ]; then - cat - "$__cdist_tmp_file" << eof > "$__cdist_code_output" -# -# The following code was generated by $__cdist_type_gencode -# - -eof - chmod u+x "${__cdist_code_output}" -fi diff --git a/bin/cdist-object-gencode-run b/bin/cdist-object-gencode-run deleted file mode 100755 index 254ac1e4..00000000 --- a/bin/cdist-object-gencode-run +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# For the given object create the code to be executed on the target. -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -eu - -__cdist_target_host="$1"; shift -__cdist_object_self="$1"; shift - -__cdist_echo info "Generating local code " -cdist-object-gencode "$__cdist_target_host" "$__cdist_object_self" \ - "${__cdist_name_gencode_local}" - -__cdist_echo info "Generating remote code " -cdist-object-gencode "$__cdist_target_host" "$__cdist_object_self" \ - "${__cdist_name_gencode_remote}" diff --git a/bin/cdist-object-manifest-run b/bin/cdist-object-manifest-run deleted file mode 100755 index 34d4f867..00000000 --- a/bin/cdist-object-manifest-run +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run the manifest for the given object. -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -eu - -__cdist_target_host="$1"; shift -__cdist_object_self="$1"; shift - -# FIXME: rename to __cdist_object_dir (everywhere!) -__cdist_cur_object_dir="$(__cdist_object_dir "$__cdist_object_self")" -__cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" - -__cdist_echo info "Checking manifest " - -__cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" -__cdist_manifest="$(__cdist_type_manifest "$__cdist_type")" - -if [ -f "$__cdist_manifest" ]; then - if [ -x "$__cdist_manifest" ]; then - # Make __cdist_manifest available for cdist-type-emulator - export __cdist_manifest - - __cdist_echo info "Executing manifest " - export $__cdist_name_var_object="$__cdist_cur_object_dir" - export $__cdist_name_var_object_id="$__cdist_object_id" - export $__cdist_name_var_type="$(__cdist_type_dir "$__cdist_type")" - - cdist-manifest-run "$__cdist_target_host" "$__cdist_manifest" - - # Tell cdist-object-run-all that there may be new objects - touch "$__cdist_objects_created" - else - __cdist_exit_err "${__cdist_manifest} needs to be executable." - fi -fi diff --git a/bin/cdist-object-prepare b/bin/cdist-object-prepare deleted file mode 100755 index d21d8a63..00000000 --- a/bin/cdist-object-prepare +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# For the given object: -# - run type explorers -# - run type manifest -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -eu - -__cdist_target_host="$1"; shift -__cdist_object_self="$1"; shift -__cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")" -[ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined" - -# Export to non-core for use in manifest and gencode scripts -export $__cdist_name_var_self=$__cdist_object_self - -__cdist_object_prepared="$(__cdist_object_prepared "$__cdist_object_self")" -if [ ! -f "$__cdist_object_prepared" ]; then - __cdist_echo info "Preparing object" - cdist-object-explorer-run "$__cdist_target_host" "$__cdist_object_self" - cdist-object-manifest-run "$__cdist_target_host" "$__cdist_object_self" - - # Mark this object as prepared - touch "$__cdist_object_prepared" -fi diff --git a/bin/cdist-object-push b/bin/cdist-object-push deleted file mode 100755 index 62b00cb2..00000000 --- a/bin/cdist-object-push +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Transfer the given object to the target host. -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -eu - -__cdist_target_host="$1"; shift -__cdist_object_self="$1"; shift - -__cdist_echo info "Transferring object" -cdist-dir push "$__cdist_target_host" \ - "$(__cdist_object_dir "$__cdist_object_self")" \ - "$(__cdist_remote_object_dir "$__cdist_object_self")" diff --git a/bin/cdist-object-run b/bin/cdist-object-run deleted file mode 100755 index 4f40e7c1..00000000 --- a/bin/cdist-object-run +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# For the given object: -# - run type explorers -# - run type manifest -# - generate code -# - copy object to target -# - execute code on target -# - -. cdist-config -[ $# -eq 2 ] || __cdist_usage " " -set -eu - -__cdist_target_host="$1"; shift -__cdist_object_self="$1"; shift -__cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")" -[ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined" - -# Export to non-core for use in manifest and gencode scripts -export $__cdist_name_var_self=$__cdist_object_self - -__cdist_object_finished="$(__cdist_object_finished "$__cdist_object_self")" -if [ ! -f "$__cdist_object_finished" ]; then - # Resolve dependencies, if any - __cdist_object_require="$(__cdist_object_require "$__cdist_object_self")" - if [ -f "$__cdist_object_require" ]; then - # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP - while read __cdist_requirement; do - set -- "$@" "$__cdist_requirement" - done < "$__cdist_object_require" - - while [ $# -gt 0 ]; do - __cdist_requirement="$1"; shift - __cdist_echo info "Resolving requirement $__cdist_requirement" - cdist-object-run "$__cdist_target_host" "$__cdist_requirement" - done - fi - - cdist-object-gencode-run "$__cdist_target_host" "$__cdist_object_self" - cdist-object-push "$__cdist_target_host" "$__cdist_object_self" - cdist-object-code-run "$__cdist_target_host" "$__cdist_object_self" - - # Mark this object as done - touch "$__cdist_object_finished" -fi diff --git a/bin/cdist-remote-explorer-run b/bin/cdist-remote-explorer-run deleted file mode 100755 index d95913ba..00000000 --- a/bin/cdist-remote-explorer-run +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# This binary is executed on the remote side to execute explorers -# -# It supports different variables names to be used, so __explorers -# and __type_explorers can be submitted :-) -# - -. cdist-config -[ $# -eq 3 ] || __cdist_usage " " -set -ue - -# Variable that defines the home of the explorers -__cdist_variable_name="$1"; shift - -# Find explorers here -__cdist_explorer_dir="$1"; shift - -# Write output here -__cdist_my_out_dir="$1"; shift - -# Setup environment -export $__cdist_variable_name="$__cdist_explorer_dir" -export __global="$__cdist_remote_out_dir" - -mkdir -p "$__cdist_my_out_dir" - -# Ensure there is at least one explorer -num="$(ls -1 "$__cdist_explorer_dir" | wc -l)" -if [ "$num" -lt 1 ]; then - __cdist_exit_err "${__cdist_explorer_dir}: Contains no explorers" -fi - -# Execute all explorers -for explorer in "$__cdist_explorer_dir/"*; do - explorer_name="${explorer##*/}" - - if [ -f "$explorer" ]; then - if [ ! -x "$explorer" ]; then - __cdist_exit_err "Explorer \"$explorer\" exists, but is not executable." - fi - - # Execute explorers and save results in remote destination directory - "$explorer" > "${__cdist_my_out_dir}/$explorer_name" - else - if [ -e "$explorer" ]; then - __cdist_exit_err "Explorer \"$explorer\" exists, but is not a file." - fi - fi -done diff --git a/bin/cdist.py b/bin/cdist.py new file mode 120000 index 00000000..9a039b33 --- /dev/null +++ b/bin/cdist.py @@ -0,0 +1 @@ +cdist \ No newline at end of file From 264a76ca2f0eb41c234be3882287c5c7676b20f6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:29:05 +0200 Subject: [PATCH 113/166] more stuff to ignore Signed-off-by: Nico Schottelius --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 10d98990..d606aec7 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ doc/man/man*/docbook-xsl.css # Ignore cache for version control cache/ + +# Python +bin/__pycache__/ From c0ab4cdd8de653e36ff8c77dd1eb072325435382 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:34:24 +0200 Subject: [PATCH 114/166] remove now obsolete core directory Signed-off-by: Nico Schottelius --- core/__cdist_cache | 34 ------------ core/__cdist_dir | 46 ---------------- core/__cdist_dir_listing | 30 ----------- core/__cdist_echo | 56 ------------------- core/__cdist_exit_err | 28 ---------- core/__cdist_explorer_run | 79 --------------------------- core/__cdist_explorer_run_global | 32 ----------- core/__cdist_is_executable | 44 --------------- core/__cdist_kill_on_interrupt | 31 ----------- core/__cdist_manifest_run | 53 ------------------ core/__cdist_manifest_run_init | 32 ----------- core/__cdist_object_all | 60 --------------------- core/__cdist_object_code_run | 54 ------------------- core/__cdist_object_explorer_run | 89 ------------------------------- core/__cdist_object_gencode | 66 ----------------------- core/__cdist_object_gencode_run | 36 ------------- core/__cdist_object_list | 36 ------------- core/__cdist_object_manifest_run | 59 -------------------- core/__cdist_object_prepare | 47 ---------------- core/__cdist_object_run | 74 ------------------------- core/__cdist_run | 27 ---------- core/__cdist_run_remote | 32 ----------- core/__cdist_run_shell | 34 ------------ core/__cdist_tmp_removal | 27 ---------- core/__cdist_type_build_emulation | 49 ----------------- core/__cdist_usage | 27 ---------- 26 files changed, 1182 deletions(-) delete mode 100755 core/__cdist_cache delete mode 100755 core/__cdist_dir delete mode 100755 core/__cdist_dir_listing delete mode 100755 core/__cdist_echo delete mode 100755 core/__cdist_exit_err delete mode 100755 core/__cdist_explorer_run delete mode 100755 core/__cdist_explorer_run_global delete mode 100755 core/__cdist_is_executable delete mode 100644 core/__cdist_kill_on_interrupt delete mode 100755 core/__cdist_manifest_run delete mode 100755 core/__cdist_manifest_run_init delete mode 100755 core/__cdist_object_all delete mode 100755 core/__cdist_object_code_run delete mode 100755 core/__cdist_object_explorer_run delete mode 100755 core/__cdist_object_gencode delete mode 100755 core/__cdist_object_gencode_run delete mode 100755 core/__cdist_object_list delete mode 100755 core/__cdist_object_manifest_run delete mode 100755 core/__cdist_object_prepare delete mode 100755 core/__cdist_object_run delete mode 100755 core/__cdist_run delete mode 100755 core/__cdist_run_remote delete mode 100755 core/__cdist_run_shell delete mode 100755 core/__cdist_tmp_removal delete mode 100755 core/__cdist_type_build_emulation delete mode 100755 core/__cdist_usage diff --git a/core/__cdist_cache b/core/__cdist_cache deleted file mode 100755 index 95764d3d..00000000 --- a/core/__cdist_cache +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Save the configuration tree into the local cache -# - -__cdist_cache() -{ - # Create base to move into - mkdir -p "${__cdist_local_base_cache_dir}" - - __cdist_echo info \ - "Caching to $(__cdist_host_cache_dir "$__cdist_target_host")" - rm -rf "$(__cdist_host_cache_dir "$__cdist_target_host")" - mv "$__cdist_local_base_dir" \ - "$(__cdist_host_cache_dir "$__cdist_target_host")" -} diff --git a/core/__cdist_dir b/core/__cdist_dir deleted file mode 100755 index 32ee0075..00000000 --- a/core/__cdist_dir +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Push a directory to a target, both sides have the same name (i.e. explorers) -# or -# Pull a directory from a target, both sides have the same name (i.e. explorers) -# - -__cdist_dir() -{ - [ $# -eq 3 ] || __cdist_usage " " - - # ${3%/*} will be the destination directory, so no subdirectories - # of the same name are created, if the directory is already existing - - if [ "$1" = "push" ]; then - # FIXME: add error handling with __cdist_run_remote_... or so - ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "mkdir -p \"$3\"" - scp -qr "$2" \ - "${__cdist_remote_user}@${__cdist_target_host}:${3%/*}" - elif [ "$1" = "pull" ]; then - mkdir -p "$3" - scp -qr "${__cdist_remote_user}@${__cdist_target_host}:$2" \ - "${3%/*}" - else - __cdist_exit_err "Unknown action $1" - fi -} diff --git a/core/__cdist_dir_listing b/core/__cdist_dir_listing deleted file mode 100755 index f4aa2320..00000000 --- a/core/__cdist_dir_listing +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# List files in a directory, if it exists -# -# We only create output, if there's at least one entry -# and can thus be used as a boolean ;-) -# - -__cdist_dir_listing() -{ - [ -d "$1" ] && ls -1 "$1" -} diff --git a/core/__cdist_echo b/core/__cdist_echo deleted file mode 100755 index a89d1821..00000000 --- a/core/__cdist_echo +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# echo / syslog alike function -# - -__cdist_echo() -{ - __cdist_echo_type="$1"; shift - - set +u - if [ "$__cdist_object_self" ]; then - __cdist_echo_prefix="${__cdist_object_self}:" - else - __cdist_echo_prefix="core: " - fi - set -u - - case "$__cdist_echo_type" in - debug) - if [ "$__cdist_debug" = 1 ]; then - echo $__cdist_echo_prefix "DEBUG: $@" - fi - ;; - info) - echo $__cdist_echo_prefix "$@" - ;; - warn) - echo $__cdist_echo_prefix "Warning: $@" - ;; - error) - echo $__cdist_echo_prefix "Error: $@" >&2 - ;; - *) - echo "CORE BUG, who created the broken commit in $0?" >&2 - exit 23 - ;; - esac -} diff --git a/core/__cdist_exit_err b/core/__cdist_exit_err deleted file mode 100755 index 303dbf20..00000000 --- a/core/__cdist_exit_err +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Print error and exit (perror() alike) -# - -__cdist_exit_err() -{ - __cdist_echo error "$@" - exit 1 -} diff --git a/core/__cdist_explorer_run b/core/__cdist_explorer_run deleted file mode 100755 index 9e58fa09..00000000 --- a/core/__cdist_explorer_run +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run explorers - FIXME: this function is ugly -# - -__cdist_explorer_run() -{ - [ $# -eq 5 ] || __cdist_usage " " - - # Ensure there is at least one explorer - num="$(ls -1 "$__cdist_explorer_dir" | wc -l)" - if [ "$num" -lt 1 ]; then - __cdist_exit_err "${__cdist_explorer_dir}: Contains no explorers" - fi - - # Check whether to setup variable for type explorer - case "$1" in - global) - ;; - type) - # FIXME: think about how and where this gets setup! - "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" - ;; - esac - - # Transfer explorers - __cdist_dir push "$2" "$3" - - # Create output directory - __cdist_run_remote mkdir -p "$4" - - # Execute all explorers - FIXME: isolate cd call? - cd "$2"; - # FIXME: cleanup double variable, no need when in directory - for __cdist_explorer_run_explorer in *; do - __cdist_explorer_explorer_name="${__cdist_explorer_run_explorer##*/}" - - if [ -f "$__cdist_explorer_run_explorer" ]; then - if [ ! -x "$__cdist_explorer_run_explorer" ]; then - __cdist_exit_err "Explorer \"$__cdist_explorer_run_explorer\" exists, but is not executable." - fi - - else - if [ -e "$__cdist_explorer_run_explorer" ]; then - __cdist_exit_err "Explorer \"$__cdist_explorer_run_explorer\" exists, but is not a file." - fi - fi - - # FIXME: no need for remote out dir probably? - # or should we leave it and continue using __cdist_dir pull? - __cdist_run_remote \ - "export $__cdist_name_var_explorer=\"$__cdist_remote_explorer_dir\";" \ - "export $__cdist_name_var_global=\"$__cdist_remote_out_dir\";" \ - "$3/$__cdist_explorer_run_explorer" ">" \ - "$4/$__cdist_explorer_run_explorer" || \ - __cdist_exit_err "Explorer $__cdist_explorer_run_explorer failed." - done - - # Transfer results back - __cdist_dir pull "$4" "$5" -} diff --git a/core/__cdist_explorer_run_global b/core/__cdist_explorer_run_global deleted file mode 100755 index 27359713..00000000 --- a/core/__cdist_explorer_run_global +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Copy & run the global explorers, i.e. not bound to types -# - -__cdist_explorer_run_global() -{ - __cdist_echo info "Running global explorers " - - # run the global explorers remotely - __cdist_explorer_run global \ - "$__cdist_explorer_dir" "$__cdist_remote_explorer_dir" \ - "$__cdist_remote_out_explorer_dir" "$__cdist_out_explorer_dir" -} diff --git a/core/__cdist_is_executable b/core/__cdist_is_executable deleted file mode 100755 index a7a6d174..00000000 --- a/core/__cdist_is_executable +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Test whether something is executable (that should be executable) or -# is missing -# - -__cdist_is_executable() -{ - [ $# -eq 1 ] || __cdist_exit_err "" - - if [ -e "$1" ]; then - if [ -f "$1" ]; then - if [ -x "$1" ]; then - # Exists and is a correct executable - true - else - __cdist_exit_err "$1 exists, but is not executable." - fi - else - __cdist_exit_err "$1 exists, but is not a file." - fi - else - # Does not exist - false - fi -} diff --git a/core/__cdist_kill_on_interrupt b/core/__cdist_kill_on_interrupt deleted file mode 100644 index 7cb711fa..00000000 --- a/core/__cdist_kill_on_interrupt +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run the given command for each created object. -# - -# Does not work in children, will be called again in every script! -# Use only in interactive "front end" scripts -__cdist_kill_on_interrupt() -{ - __cdist_tmp_removal - kill 0 - exit 1 -} diff --git a/core/__cdist_manifest_run b/core/__cdist_manifest_run deleted file mode 100755 index cf85d646..00000000 --- a/core/__cdist_manifest_run +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Let's build a cconfig tree from a configuration -# And save it into the cache tree -# - -__cdist_manifest_run() -{ - [ $# -eq 1 ] || __cdist_usage "" - - __cdist_manifest="$1"; shift - - ################################################################################ - # Export information for cdist-type-emulator or manifest - # - - # Config dir should not get reset - FIXME: why did I do this? - export __cdist_conf_dir - - # Used to record the source in the object - export __cdist_manifest - - # Export information for manifests - __cdist_out_dir comes from cdist-config - export $__cdist_name_var_global="$__cdist_out_dir" - - ################################################################################ - # The actual run - # - - # Ensure binaries are existing - FIXME: move error handling into __cdist_type_build_emulation - __cdist_type_build_emulation \ - || __cdist_exit_err "Failed to build type emulation binaries" - - __cdist_run_shell "${__cdist_manifest}" -} diff --git a/core/__cdist_manifest_run_init b/core/__cdist_manifest_run_init deleted file mode 100755 index e8fa63de..00000000 --- a/core/__cdist_manifest_run_init +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Let's build a cconfig tree from a configuration -# And save it into the cache tree -# - -__cdist_manifest_run_init() -{ - # FIXME: probably do not export but always set explicitly? - export $__cdist_name_var_manifest="$__cdist_manifest_dir" - - __cdist_echo info "Running initial manifest for $__cdist_target_host " - __cdist_manifest_run "$__cdist_manifest_init" -} diff --git a/core/__cdist_object_all b/core/__cdist_object_all deleted file mode 100755 index 965d08f6..00000000 --- a/core/__cdist_object_all +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run the given command for each created object. -# - -__cdist_object_all() -{ - [ $# -eq 1 ] || __cdist_usage "" - - __cdist_object_all_object_all_command="$1"; shift - - __cdist_object_all_object_all_objects="$__cdist_tmp_dir/objects" - - # Ensure object dir exists, so marker can be created - mkdir -p "${__cdist_out_object_dir}" - - # FIXME: : - why do we use a file? - # core/__cdist_object_manifest_run: touch "$__cdist_objects_created" - - # Loop until we do not create new objects anymore - # which is equal to all objects have been run - touch "$__cdist_objects_created" - while [ -f "$__cdist_objects_created" ]; do - # Assume we're done after this run - rm "$__cdist_objects_created" - - # Get listing of objects - __cdist_object_list "$__cdist_out_object_dir" > \ - "$__cdist_object_all_object_all_objects" - - # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP - while read __cdist_object_all_object; do - set -- "$@" "$__cdist_object_all_object" - done < "$__cdist_object_all_object_all_objects" - - while [ $# -gt 0 ]; do - __cdist_object_all_object="$1"; shift - $__cdist_object_all_object_all_command "$__cdist_object_all_object" - done - done -} diff --git a/core/__cdist_object_code_run b/core/__cdist_object_code_run deleted file mode 100755 index 8af67ab8..00000000 --- a/core/__cdist_object_code_run +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Exec the code for the given object locally and remote -# - -__cdist_object_code_run() -{ - [ $# -eq 1 ] || __cdist_exit_err "" - - - if [ ! -d "$(__cdist_object_dir "$1")" ]; then - __cdist_exit_err "Object undefined" - fi - - # Code local - export __cdist_out_object_dir="$__cdist_out_object_dir" - __cdist_echo debug "Trying to run local code" - if __cdist_is_executable \ - "$(__cdist_object_code "$1" "${__cdist_name_gencode_local}")"; then - __cdist_run_shell \ - "$(__cdist_object_code "$1" "${__cdist_name_gencode_local}")" - else - __cdist_echo debug "Local code: none" - fi - - # Code remote - __cdist_echo debug "Trying to run remote code" - if __cdist_is_executable \ - "$(__cdist_object_code "$1" "${__cdist_name_gencode_remote}")"; then - - __cdist_run_remote $(__cdist_remote_object_code "$1") - else - __cdist_echo debug "Remote code: none" - fi -} diff --git a/core/__cdist_object_explorer_run b/core/__cdist_object_explorer_run deleted file mode 100755 index da59d6c3..00000000 --- a/core/__cdist_object_explorer_run +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run the explorers for the given object on the target host. -# - -# FIXME: many cleanups needed before going production! - -__cdist_object_explorer_run() -{ - __cdist_object_self="$1"; shift - - __cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" - __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" - - # Check if type of object has >= 1 explorer - __cdist_has_explorer="$(__cdist_dir_listing "$(__cdist_type_explorer_dir "$__cdist_type")" | wc -l)" - # Run the type explorers for the current object if any - if [ "$__cdist_has_explorer" -ge 1 ]; then - if ! __cdist_type_explorer_pushed "$__cdist_type"; then - # FIXME: variables! - src_dir="$(__cdist_type_explorer_dir "$__cdist_type")" - dst_dir="$(__cdist_remote_type_explorer_dir "$__cdist_type")" - __cdist_echo info "Transfering explorers for $__cdist_type " - __cdist_dir push "$src_dir" "$dst_dir" - __cdist_type_explorer_pushed_add "$__cdist_type" - fi - - __cdist_echo info "Running explorers" - # Copy object parameters - __cdist_dir push \ - "$(__cdist_object_parameter_dir "$__cdist_object_self")" \ - "$(__cdist_remote_object_parameter_dir "$__cdist_object_self")" - - # Execute explorers - # FIXME: STOPPED: - # - remove cdist-remote-explorer-run - # - problem: new variables / need to run explorer directly? - # -> or put logic into __cdist_explorer_run - # -> think about having _one_ wrapper script for remote to execute - # shell functions - - # Create remote output directory - __cdist_run_remote mkdir -p "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" - - cd "$(__cdist_type_explorer_dir "$__cdist_type")" - - - for __cdist_object_explorer_run_explorer in *; do - __cdist_run_remote \ - "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \ - "$__cdist_name_var_object_id=\"$__cdist_object_id\"" \ - "$__cdist_name_var_self=\"$__cdist_object_self\"" \ - "$(__cdist_remote_type_explorer_dir "$__cdist_type")/$__cdist_object_explorer_run_explorer" \ - ">" "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")/$__cdist_object_explorer_run_explorer" - done - -# __cdist_run_remote \ -# "$__cdist_name_var_object=\"$(__cdist_remote_object_dir "$__cdist_object_self")\"" \ -# "$__cdist_name_var_object_id=\"$__cdist_object_id\"" \ -# "$__cdist_name_var_self=\"$__cdist_object_self\"" \ -# cdist-remote-explorer-run \ -# "$__cdist_name_var_type_explorer" \ -# "$(__cdist_remote_type_explorer_dir "$__cdist_type")" \ -# "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" - - # Copy back results - __cdist_dir pull "$(__cdist_remote_object_type_explorer_dir "$__cdist_object_self")" \ - "$(__cdist_object_type_explorer_dir "$__cdist_object_self")" - fi -} diff --git a/core/__cdist_object_gencode b/core/__cdist_object_gencode deleted file mode 100755 index 08ef8b7d..00000000 --- a/core/__cdist_object_gencode +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Generate code from one object (object must be relative path!) -# WARNING: OUTPUT ON STDOUT, ERRORS NEED TO BE ON STDERR! -# - -# FIXME: check variable names: -# either prefix or use global or use functions directly -# functions looks good, they are cheap anyway! - -__cdist_object_gencode() -{ - [ $# -eq 2 ] || __cdist_usage "" "" - - __cdist_object_self="$1"; shift - __cdist_gencode_type="$1"; shift - - __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" - __cdist_type_gencode="$(__cdist_type_gencode "$__cdist_type" "$__cdist_gencode_type")" - __cdist_code_output="$(__cdist_object_code "$__cdist_object_self" "$__cdist_gencode_type")" - - # export variables for the gencode script - export __object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" - export __object="$(__cdist_object_dir "$__cdist_object_self")" - export __global="$__cdist_out_dir" - - if [ -x "$__cdist_type_gencode" ]; then - __cdist_run_shell "$__cdist_type_gencode" > "$__cdist_tmp_file" - else - if [ -e "$__cdist_type_gencode" ]; then - __cdist_exit_err "$__cdist_type_gencode exists, but is not executable" - fi - - # Ensure it's empty, if there is no gencode - : > "$__cdist_tmp_file" - fi - - # Only create code, if gencode created output - if [ "$(wc -l < "$__cdist_tmp_file")" -gt 0 ]; then - cat - "$__cdist_tmp_file" << eof > "$__cdist_code_output" -# -# The following code was generated by $__cdist_type_gencode -# - -eof - chmod u+x "${__cdist_code_output}" - fi -} diff --git a/core/__cdist_object_gencode_run b/core/__cdist_object_gencode_run deleted file mode 100755 index 308f5f33..00000000 --- a/core/__cdist_object_gencode_run +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# For the given object create the code to be executed on the target. -# - -__cdist_object_gencode_run() -{ - __cdist_object_gencode_run_object="$1"; shift - - __cdist_echo info "Generating local code " - __cdist_object_gencode "$__cdist_object_gencode_run_object" \ - "${__cdist_name_gencode_local}" - - __cdist_echo info "Generating remote code " - __cdist_object_gencode "$__cdist_object_gencode_run_object" \ - "${__cdist_name_gencode_remote}" -} diff --git a/core/__cdist_object_list b/core/__cdist_object_list deleted file mode 100755 index f2785c30..00000000 --- a/core/__cdist_object_list +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Print error and exit (perror() alike) -# - -__cdist_object_list() -{ - # FIXME: no local in posix - local basedir="$1"; shift - - # Use subshell to prevent changing cwd in program - ( - cd "${basedir}" - - find . -name "$__cdist_name_dot_cdist" | \ - sed -e 's;^./;;' -e "s;/${__cdist_name_dot_cdist}\$;;" - ) -} diff --git a/core/__cdist_object_manifest_run b/core/__cdist_object_manifest_run deleted file mode 100755 index efc85539..00000000 --- a/core/__cdist_object_manifest_run +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/sh -# -# 2010 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run the manifest for the given object. -# - - -__cdist_object_manifest_run() -{ - [ $# -eq 1 ] || __cdist_usage "" - - __cdist_object_self="$1"; shift - - # FIXME: rename to __cdist_object_dir (everywhere!) - __cdist_cur_object_dir="$(__cdist_object_dir "$__cdist_object_self")" - __cdist_object_id="$(__cdist_object_id_from_object "$__cdist_object_self")" - - __cdist_echo info "Checking manifest " - - __cdist_type="$(__cdist_type_from_object "$__cdist_object_self")" - __cdist_manifest="$(__cdist_type_manifest "$__cdist_type")" - - if [ -f "$__cdist_manifest" ]; then - if [ -x "$__cdist_manifest" ]; then - # Make __cdist_manifest available for cdist-type-emulator - export __cdist_manifest - - __cdist_echo info "Executing manifest " - export $__cdist_name_var_object="$__cdist_cur_object_dir" - export $__cdist_name_var_object_id="$__cdist_object_id" - export $__cdist_name_var_type="$(__cdist_type_dir "$__cdist_type")" - - __cdist_manifest_run "$__cdist_manifest" - - # Tell cdist-object-run-all that there may be new objects - touch "$__cdist_objects_created" - else - __cdist_exit_err "${__cdist_manifest} needs to be executable." - fi - fi -} diff --git a/core/__cdist_object_prepare b/core/__cdist_object_prepare deleted file mode 100755 index 24039be0..00000000 --- a/core/__cdist_object_prepare +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# For the given object: -# - run type explorers -# - run type manifest -# - -__cdist_object_prepare() -{ - [ $# -eq 1 ] || __cdist_usage "" - - __cdist_object_self="$1"; shift - __cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")" - [ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined" - - # Export to non-core for use in manifest and gencode scripts - export $__cdist_name_var_self=$__cdist_object_self - - __cdist_object_prepared="$(__cdist_object_prepared "$__cdist_object_self")" - if [ ! -f "$__cdist_object_prepared" ]; then - __cdist_echo info "Preparing object" - __cdist_object_explorer_run "$__cdist_object_self" - __cdist_object_manifest_run "$__cdist_object_self" - - # Mark this object as prepared - touch "$__cdist_object_prepared" - fi -} diff --git a/core/__cdist_object_run b/core/__cdist_object_run deleted file mode 100755 index d2c7df6e..00000000 --- a/core/__cdist_object_run +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# For the given object: -# - run type explorers -# - run type manifest -# - generate code -# - copy object to target -# - execute code on target -# - -__cdist_object_run() -{ - [ $# -eq 1 ] || __cdist_usage "" - - __cdist_object_self="$1"; shift - __cdist_object_dir="$(__cdist_object_dir "$__cdist_object_self")" - [ -d "$__cdist_object_dir" ] || __cdist_exit_err "Object undefined" - - # Export to non-core for use in manifest and gencode scripts - export $__cdist_name_var_self=$__cdist_object_self - - # FIXME: BUG: should be named differently! - # FIXME: BUG: I can be called recursively! -> variables are probably already set / overwritten! - __cdist_object_finished="$(__cdist_object_finished "$__cdist_object_self")" - if [ ! -f "$__cdist_object_finished" ]; then - # Resolve dependencies, if any - __cdist_object_require="$(__cdist_object_require "$__cdist_object_self")" - if [ -f "$__cdist_object_require" ]; then - # NEED TO CREATE ARRAY, SSH DESTROYS WHILE READ LOOP - while read __cdist_requirement; do - set -- "$@" "$__cdist_requirement" - done < "$__cdist_object_require" - - while [ $# -gt 0 ]; do - __cdist_requirement="$1"; shift - __cdist_echo info "Resolving requirement $__cdist_requirement" - # FIXME: BUG: at this point, the other __cdist_object_run may have - # overwritten all our variables! - __cdist_object_run "$__cdist_requirement" - done - fi - - __cdist_echo debug "Before gencode" - __cdist_object_gencode_run "$__cdist_object_self" - __cdist_echo debug "Before push" - __cdist_dir push "$(__cdist_object_dir "$__cdist_object_self")" \ - "$(__cdist_remote_object_dir "$__cdist_object_self")" - __cdist_echo debug "Before run" - __cdist_object_code_run "$__cdist_object_self" - __cdist_echo debug "Object run done" - - # Mark this object as done - touch "$__cdist_object_finished" - fi -} diff --git a/core/__cdist_run b/core/__cdist_run deleted file mode 100755 index 8febe550..00000000 --- a/core/__cdist_run +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Exit if an error occurs running something -# - -__cdist_run() -{ - "$@" || __cdist_echo error "$1 exited non-zero, aborting." -} diff --git a/core/__cdist_run_remote b/core/__cdist_run_remote deleted file mode 100755 index 17074049..00000000 --- a/core/__cdist_run_remote +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Run a cdist binary on the remote side -# - -__cdist_run_remote() -{ - [ $# -ge 1 ] || __cdist_usage " [opts]" - - ssh "${__cdist_remote_user}@${__cdist_target_host}" \ - "export PATH=\"${__cdist_remote_bin_dir}:\$PATH\";" \ - "export __cdist_out_object_dir=\"$__cdist_remote_out_object_dir\";" \ - "$@" -} diff --git a/core/__cdist_run_shell b/core/__cdist_run_shell deleted file mode 100755 index b6e0a57d..00000000 --- a/core/__cdist_run_shell +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Exit if an error occurs when running a shell script -# - -__cdist_run_shell() -{ - # Prepend our path, so all cdist tools come before other tools - PATH="${__cdist_out_type_bin_dir}:$PATH" sh -e "$@" - if [ "$?" -ne 0 ]; then - __cdist_echo error "$1 exited non-zero" - __cdist_echo warn "Faulty code:" - cat "$1" - __cdist_exit_err "Aborting due to non-zero exit code." - fi -} diff --git a/core/__cdist_tmp_removal b/core/__cdist_tmp_removal deleted file mode 100755 index 74d74936..00000000 --- a/core/__cdist_tmp_removal +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Remove tmp dir -# - -__cdist_tmp_removal() -{ - rm -rf "${__cdist_tmp_dir}" -} diff --git a/core/__cdist_type_build_emulation b/core/__cdist_type_build_emulation deleted file mode 100755 index 3c7270ca..00000000 --- a/core/__cdist_type_build_emulation +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Build pseudo binaries for type emulation -# - -__cdist_type_build_emulation() -{ - [ $# -eq 0 ] || __cdist_usage "No arguments" - - [ -f "${__cdist_out_type_bin_dir}/.marker" ] && return 0 - - __cdist_type_emulator="$__cdist_abs_mydir/cdist-type-emulator" - - if [ ! -d "${__cdist_type_dir}" ]; then - __cdist_exit_err "$__cdist_type_dir must exist and contain available types" - fi - - # Get Types - ( - cd "${__cdist_type_dir}" - ls -1 > "${__cdist_tmp_file}" - ) - - # Create binaries - mkdir -p "${__cdist_out_type_bin_dir}" - while read __cdist_type_build_emulation_type; do - ln -sf "${__cdist_type_emulator}" \ - "${__cdist_out_type_bin_dir}/${__cdist_type_build_emulation_type}" - done < "${__cdist_tmp_file}" - - touch "${__cdist_out_type_bin_dir}/.marker" -} diff --git a/core/__cdist_usage b/core/__cdist_usage deleted file mode 100755 index 9dfa30e4..00000000 --- a/core/__cdist_usage +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Print error and exit (perror() alike) -# - -__cdist_usage() -{ - __cdist_exit_err "$__cdist_myname: $@" -} From 47c5ebcf3f34de77f35904c90b96c2f1f44eb733 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:37:40 +0200 Subject: [PATCH 115/166] remove parallel and sequential args, until they are working Signed-off-by: Nico Schottelius --- bin/cdist | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 9f713ee3..a04575a8 100755 --- a/bin/cdist +++ b/bin/cdist @@ -588,12 +588,12 @@ if __name__ == "__main__": parser.add_argument('-i', '--initial-manifest', help='Path to a cdist manifest or - to read from stdin', dest='manifest', required=False) - parser.add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser.add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially', - action='store_false', dest='parallel') +# parser.add_argument('-p', '--parallel', +# help='Operate on multiple hosts in parallel', +# action='store_true', dest='parallel') +# parser.add_argument('-s', '--sequential', +# help='Operate on multiple hosts sequentially', +# action='store_false', dest='parallel') args = parser.parse_args(sys.argv[1:]) if args.debug: From 326f2644c7a9d7471a73b55403c97b620f28b570 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 21:41:56 +0200 Subject: [PATCH 116/166] introduce type_dir() and use it :-) Signed-off-by: Nico Schottelius --- bin/cdist | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index a04575a8..5b2b8d88 100755 --- a/bin/cdist +++ b/bin/cdist @@ -302,9 +302,13 @@ class Cdist: return objects + def type_dir(self, type): + """Return directory the type""" + return os.path.join(TYPE_DIR, type) + def type_explorer_dir(self, type): """Return directory that holds the explorers of a type""" - return os.path.join(TYPE_DIR, type, "explorer") + return os.path.join(self.type_dir(type), "explorer") def type_gencode_paths(self, type): """Return paths to gencode scripts of type""" @@ -444,7 +448,7 @@ class Cdist: env = { "__object" : self.object_dir(cdist_object), "__object_id": self.get_object_id_from_object(cdist_object), "__object_fq": cdist_object, - "__type": type + "__type": self.type_dir(type) } self.run_manifest(manifest, extra_env=env) From 0d1f121e2153b92aa6166279d10a2765e9999c8f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 22:21:46 +0200 Subject: [PATCH 117/166] do not fail without objects Signed-off-by: Nico Schottelius --- bin/cdist | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/bin/cdist b/bin/cdist index 5b2b8d88..7a4d3186 100755 --- a/bin/cdist +++ b/bin/cdist @@ -121,6 +121,9 @@ class Cdist: # List of type explorers transferred self.type_explorers_transferred = {} + # objects + self.objects_prepared = [] + self.remote_user = remote_user # Mostly static, but can be overwritten on user demand @@ -241,13 +244,10 @@ class Cdist: return list - def list_object_paths(self, starting_point = False): + def list_object_paths(self, starting_point): """Return list of paths of existing objects""" object_paths = [] - if not starting_point: - starting_point = self.object_base_dir - for content in os.listdir(starting_point): full_path = os.path.join(starting_point, content) if os.path.isdir(full_path): @@ -288,17 +288,15 @@ class Cdist: return [os.path.join(self.object_dir(cdist_object), "code-local"), os.path.join(self.object_dir(cdist_object), "code-remote")] - def list_objects(self, starting_point = False): + def list_objects(self): """Return list of existing objects""" - if not starting_point: - starting_point = self.object_base_dir - - object_paths = self.list_object_paths(starting_point) objects = [] + if os.path.isdir(self.object_base_dir): + object_paths = self.list_object_paths(self.object_base_dir) - for path in object_paths: - objects.append(os.path.relpath(path, starting_point)) + for path in object_paths: + objects.append(os.path.relpath(path, self.object_base_dir)) return objects @@ -561,10 +559,14 @@ class Cdist: while old_objects != objects: log.debug("Prepare stage") old_objects = list(objects) - # FIXME: do not rerun existing objects! for cdist_object in objects: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) objects = self.list_objects() From db658328d219b524b493bb5810c79ca27fbaf163 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 23:11:27 +0200 Subject: [PATCH 118/166] add sh -e header to code to avoid OSError: [Errno 8] Exec format error Signed-off-by: Nico Schottelius --- bin/cdist | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 7a4d3186..bb6e38fa 100755 --- a/bin/cdist +++ b/bin/cdist @@ -59,6 +59,7 @@ REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") +CODE_HEADER = "#!/bin/sh -e" DOT_CDIST = ".cdist" VERSION = "2.0.0" @@ -517,15 +518,18 @@ class Cdist: os.path.basename(bin)[3:]) outfile_fd = open(outfile, "w") + outfile_fd.write(CODE_HEADER) + self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) outfile_fd.close() status = os.stat(outfile) # Remove output if empty, else make it executable - if status.st_size == 0: + if status.st_size == len(CODE_HEADER): os.unlink(outfile) else: + # Add header and make executable os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) if mode == "code": From 9a0b57c4b5266a39bcbbf9e44e9efa3ddcfa8ad2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 23:41:46 +0200 Subject: [PATCH 119/166] update todos Signed-off-by: Nico Schottelius --- doc/dev/todo/TAKEME | 36 +++++++----------------------------- doc/dev/todo/niconext | 5 +---- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index 5ded4298..e67469ce 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -5,26 +5,13 @@ Feel free to pick one! CORE ---- -- Inconsistent error messages if object is not existing! - -> always use "Object undefined" -- Add echo function / beautify output - __cdist_echo [level] [messages...] - level := syslog alike: - debug, notice, err - Include object_self prefixing, if given! -- Think about moving cdist-type-build-emulation out of cdist-manifest-run to - cdist-deploy-to: more dependency of cdist-manifest-run, but a lot of - less cycles consumed - -- cdist-object-gencode: remove code if output empty? - - also take care of that in cdist-code-run! -- Remove cdist-object-push, covers only one line and is used only once: - [20:22] kr:bin% grep cdist-object-push * - cdist-object-run: cdist-object-push "$__cdist_target_host" "$__cdist_object" - [20:22] kr:bin% - - probably remove or improve cdist-type-template -- add $__tmp? +- allow cdist to run without $PATH setup: ./bin/cdist-deploy-to +- support non-ssh access? + +USER INTERFACE +-------------- +- add support $__tmp? - for use in manifest, code, etc.? - for creating temporary files, etc. @@ -36,20 +23,11 @@ CORE -> for current host -> add function to cdist-config, import from cdist-cache -- check all all internal variables are prefixed with __cdist - Define / document "this is what should be on host X" and have it parsable by different (shinken) tool -> given after manifest run already! -- Allow types to have parameters without values (boolean flags). - e.g. __chair fancychair --pink --wood - would result in: - $__object/parameter/ - pink # empty file - wood # empty file - -- allow cdist to run without $PATH setup: ./bin/cdist-deploy-to -- use absent/present for state by default +- use absent/present for state by default? TYPES ------ diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 972525aa..a4c23e8b 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,5 +1,2 @@ -- rewrite in python? - - also do with cdist-type-emulator, which had quirks applied from outside to run -- support non-ssh access? - - Bug: os.path.join() may be wrong for the remote side! + -> does not matter for now! From 2490b983ba5e7b76504de5fc6bae3a5e542c8974 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 23:42:53 +0200 Subject: [PATCH 120/166] ++todo Signed-off-by: Nico Schottelius --- bin/cdist | 7 +++++-- doc/dev/todo/TAKEME | 2 -- doc/dev/todo/niconext | 4 ++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index bb6e38fa..6da7d04c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -99,7 +99,7 @@ def banner(): class Cdist: """Cdist main class to hold arbitrary data""" - def __init__(self, target_host, initial_manifest=False, remote_user="root"): + def __init__(self, target_host, initial_manifest=False, remote_user="root", home=None): self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] @@ -593,6 +593,9 @@ if __name__ == "__main__": parser.add_argument('-b', '--banner', help='Show cdist banner', action='store_true', dest='banner') + parser.add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store_true', dest='cdist_home') parser.add_argument('-d', '--debug', help='Set log level to debug', action='store_true') parser.add_argument('-i', '--initial-manifest', @@ -617,7 +620,7 @@ if __name__ == "__main__": log.debug(args) for host in args.host: - c = Cdist(host, initial_manifest=args.manifest) + c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home) c.deploy_to() c.cleanup() except KeyboardInterrupt: diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index e67469ce..5439a1b9 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -44,5 +44,3 @@ DOCUMENTATION - asciidoc interprets __, which we use for variables names -> seek through docs and replace with \_\_! - reference explorers in cdist-reference! -- compare running times: - one, 5, 10, 50, 100, 1000 hosts => how does cdist scale? diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index a4c23e8b..f8535c07 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,2 +1,6 @@ +- Remove all FIXME entries +- Write cdist-manpage +- Remove obsolete manpages +- Support different home instead of ../ - Bug: os.path.join() may be wrong for the remote side! -> does not matter for now! From 1598e18c28b30643d4191fbcbfefa913ef48bdb6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 23:54:18 +0200 Subject: [PATCH 121/166] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index f8535c07..3968b0ef 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -4,3 +4,7 @@ - Support different home instead of ../ - Bug: os.path.join() may be wrong for the remote side! -> does not matter for now! + +- Rewrite cdist-type-emulator + - Remove legacy code in cdist + - Remove cdist-config From 1cc7600c9c10125e8b415368b155c4054bf6b1cf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 11 Sep 2011 23:54:34 +0200 Subject: [PATCH 122/166] begin to make home configurable Signed-off-by: Nico Schottelius --- bin/cdist | 51 +++++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/bin/cdist b/bin/cdist index 6da7d04c..3361d5da 100755 --- a/bin/cdist +++ b/bin/cdist @@ -46,13 +46,6 @@ BANNER = """ """ # Given paths from installation -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -CONF_DIR = os.path.join(BASE_DIR, "conf") -GLOBAL_EXPLORER_DIR = os.path.join(CONF_DIR, "explorer") -LIB_DIR = os.path.join(BASE_DIR, "lib") -MANIFEST_DIR = os.path.join(CONF_DIR, "manifest") -TYPE_DIR = os.path.join(CONF_DIR, "type") - REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") @@ -87,10 +80,6 @@ VERSION = "2.0.0" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() -# List types -def list_types(): - return os.listdir(TYPE_DIR) - def banner(): """Guess what :-)""" print(BANNER) @@ -106,6 +95,17 @@ class Cdist: # Setup directory paths self.temp_dir = tempfile.mkdtemp() + + if home: + self.base_dir = home + else: + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + self.conf_dir = os.path.join(self.base_dir, "conf") + self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") + self.out_dir = os.path.join(self.temp_dir, "out") os.mkdir(self.out_dir) @@ -131,7 +131,7 @@ class Cdist: if initial_manifest: self.initial_manifest = initial_manifest else: - self.initial_manifest = os.path.join(MANIFEST_DIR, "init") + self.initial_manifest = os.path.join(self.manifest_dir, "init") def cleanup(self): # Do not use in __del__: @@ -231,7 +231,7 @@ class Cdist: def list_global_explorers(self): """Return list of available explorers""" - return os.listdir(GLOBAL_EXPLORER_DIR) + return os.listdir(self.global_explorer_dir) def list_type_explorers(self, type): """Return list of available explorers for a specific type""" @@ -245,6 +245,9 @@ class Cdist: return list + def list_types(self): + return os.listdir(self.type_base_dir) + def list_object_paths(self, starting_point): """Return list of paths of existing objects""" object_paths = [] @@ -303,7 +306,7 @@ class Cdist: def type_dir(self, type): """Return directory the type""" - return os.path.join(TYPE_DIR, type) + return os.path.join(self.type_base_dir, type) def type_explorer_dir(self, type): """Return directory that holds the explorers of a type""" @@ -311,12 +314,12 @@ class Cdist: def type_gencode_paths(self, type): """Return paths to gencode scripts of type""" - return [os.path.join(TYPE_DIR, type, "gencode-local"), - os.path.join(TYPE_DIR, type, "gencode-remote")] + return [os.path.join(self.type_base_dir, type, "gencode-local"), + os.path.join(self.type_base_dir, type, "gencode-remote")] def type_manifest_path(self, type): """Return path to manifest of type""" - return os.path.join(TYPE_DIR, type, "manifest") + return os.path.join(self.type_base_dir, type, "manifest") def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" @@ -343,7 +346,7 @@ class Cdist: def transfer_global_explorers(self): """Transfer the global explorers""" - self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR) + self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" @@ -367,8 +370,8 @@ class Cdist: def link_type_to_emulator(self): """Link type names to cdist-type-emulator""" - for type in list_types(): - source = os.path.join(LIB_DIR, "cdist-type-emulator") + for type in self.list_types(): + source = os.path.join(self.lib_dir, "cdist-type-emulator") destination = os.path.join(self.bin_dir, type) log.debug("Linking %s to %s", source, destination) os.symlink(source, destination) @@ -377,7 +380,7 @@ class Cdist: """Run global explorers""" explorers = self.list_global_explorers() if(len(explorers) == 0): - self.exit_error("No explorers found in", GLOBAL_EXPLORER_DIR) + self.exit_error("No explorers found in", self.global_explorer_dir) self.transfer_global_explorers() for explorer in explorers: @@ -433,7 +436,7 @@ class Cdist: def run_initial_manifest(self): """Run the initial manifest""" - env = { "__manifest" : MANIFEST_DIR } + env = { "__manifest" : self.manifest_dir } self.run_manifest(self.initial_manifest, extra_env=env) def run_type_manifest(self, cdist_object): @@ -461,8 +464,8 @@ class Cdist: env['__global'] = self.out_dir # Legacy stuff to make cdist-type-emulator work - env['__cdist_conf_dir'] = CONF_DIR - env['__cdist_core_dir'] = os.path.join(BASE_DIR, "core") + env['__cdist_conf_dir'] = self.conf_dir + env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") env['__cdist_local_base_dir'] = self.temp_dir env['__cdist_manifest'] = self.initial_manifest From 5260f230e503dbf72122802d700b484c44afce48 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 00:06:59 +0200 Subject: [PATCH 123/166] support -V, --version Signed-off-by: Nico Schottelius --- bin/cdist | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/cdist b/bin/cdist index 3361d5da..7a702c41 100755 --- a/bin/cdist +++ b/bin/cdist @@ -100,6 +100,7 @@ class Cdist: self.base_dir = home else: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + self.conf_dir = os.path.join(self.base_dir, "conf") self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") self.lib_dir = os.path.join(self.base_dir, "lib") @@ -611,10 +612,16 @@ if __name__ == "__main__": # help='Operate on multiple hosts sequentially', # action='store_false', dest='parallel') + parser.add_argument('-V', '--version', help='Show version', + action='version', version='%(prog)s ' + VERSION) + args = parser.parse_args(sys.argv[1:]) if args.debug: logging.root.setLevel(logging.DEBUG) + print(args) + sys.exit(1) + if args.banner: banner() sys.exit(0) From 8f34f8659bfc971620de62ba32e846c28823ce9b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 00:08:30 +0200 Subject: [PATCH 124/166] support -V, --version Signed-off-by: Nico Schottelius --- bin/cdist | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/cdist b/bin/cdist index 7a702c41..5656ce62 100755 --- a/bin/cdist +++ b/bin/cdist @@ -614,6 +614,7 @@ if __name__ == "__main__": parser.add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + VERSION) + arser.add_argument('--foo', nargs=2) args = parser.parse_args(sys.argv[1:]) if args.debug: From c5355ca10478810e246dfc8615dc777a370d99cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 00:16:18 +0200 Subject: [PATCH 125/166] fixup -c option Signed-off-by: Nico Schottelius --- bin/cdist | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/bin/cdist b/bin/cdist index 5656ce62..dca36b5f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -599,7 +599,7 @@ if __name__ == "__main__": action='store_true', dest='banner') parser.add_argument('-c', '--cdist-home', help='Change cdist home (default: .. from bin directory)', - action='store_true', dest='cdist_home') + action='store') parser.add_argument('-d', '--debug', help='Set log level to debug', action='store_true') parser.add_argument('-i', '--initial-manifest', @@ -614,22 +614,17 @@ if __name__ == "__main__": parser.add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + VERSION) - arser.add_argument('--foo', nargs=2) args = parser.parse_args(sys.argv[1:]) + log.debug(args) if args.debug: logging.root.setLevel(logging.DEBUG) - print(args) - sys.exit(1) - if args.banner: banner() sys.exit(0) try: - log.debug(args) - for host in args.host: c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home) c.deploy_to() From 1a790edbf6cbeff0a7517eaeb38312637145d164 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 00:20:06 +0200 Subject: [PATCH 126/166] remove context class until we need it Signed-off-by: Nico Schottelius --- bin/cdist | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/bin/cdist b/bin/cdist index dca36b5f..5eb678be 100755 --- a/bin/cdist +++ b/bin/cdist @@ -56,27 +56,6 @@ CODE_HEADER = "#!/bin/sh -e" DOT_CDIST = ".cdist" VERSION = "2.0.0" - -#class Context(object): -# -# def __init__(self, target_host): -# self.target_host = target_host -# -# # class variable -# user_selber_shuld_wenn_aendert = 'bla' -# -# # read only, aber statisch -# @property -# def remote_base_directory(self): -# return "/var/lib/cdist" -# @property.setter -# -# @property -# def special_foo(self): -# return 'foo/{0}'.format(self.target_host) -# - - logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() From 3cf203a668d8a2642033113d659d74f71dab9255 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 00:30:07 +0200 Subject: [PATCH 127/166] begin to remove obsolete documentation Signed-off-by: Nico Schottelius --- doc/man/man1/cdist-cache.text | 31 -------------- doc/man/man1/cdist-code-run.text | 34 ---------------- doc/man/man1/cdist-deploy-to.text | 43 -------------------- doc/man/man1/cdist-dir.text | 38 ----------------- doc/man/man1/cdist-explorer-run-global.text | 31 -------------- doc/man/man1/cdist-manifest-run-init.text | 32 --------------- doc/man/man1/cdist-manifest-run.text | 31 -------------- doc/man/man1/cdist-mass-deploy.text | 41 ------------------- doc/man/man1/cdist-object-all.text | 31 -------------- doc/man/man1/cdist-object-code-run.text | 32 --------------- doc/man/man1/cdist-object-explorer-run.text | 31 -------------- doc/man/man1/cdist-object-gencode-run.text | 32 --------------- doc/man/man1/cdist-object-gencode.text | 33 --------------- doc/man/man1/cdist-object-manifest-run.text | 31 -------------- doc/man/man1/cdist-object-prepare.text | 35 ---------------- doc/man/man1/cdist-object-push.text | 31 -------------- doc/man/man1/cdist-object-run.text | 36 ---------------- doc/man/man1/cdist-remote-explorer-run.text | 33 --------------- doc/man/man1/cdist-run-remote.text | 33 --------------- doc/man/man1/cdist-type-build-emulation.text | 33 --------------- 20 files changed, 672 deletions(-) delete mode 100644 doc/man/man1/cdist-cache.text delete mode 100644 doc/man/man1/cdist-code-run.text delete mode 100644 doc/man/man1/cdist-deploy-to.text delete mode 100644 doc/man/man1/cdist-dir.text delete mode 100644 doc/man/man1/cdist-explorer-run-global.text delete mode 100644 doc/man/man1/cdist-manifest-run-init.text delete mode 100644 doc/man/man1/cdist-manifest-run.text delete mode 100644 doc/man/man1/cdist-mass-deploy.text delete mode 100644 doc/man/man1/cdist-object-all.text delete mode 100644 doc/man/man1/cdist-object-code-run.text delete mode 100644 doc/man/man1/cdist-object-explorer-run.text delete mode 100644 doc/man/man1/cdist-object-gencode-run.text delete mode 100644 doc/man/man1/cdist-object-gencode.text delete mode 100644 doc/man/man1/cdist-object-manifest-run.text delete mode 100644 doc/man/man1/cdist-object-prepare.text delete mode 100644 doc/man/man1/cdist-object-push.text delete mode 100644 doc/man/man1/cdist-object-run.text delete mode 100644 doc/man/man1/cdist-remote-explorer-run.text delete mode 100644 doc/man/man1/cdist-run-remote.text delete mode 100644 doc/man/man1/cdist-type-build-emulation.text diff --git a/doc/man/man1/cdist-cache.text b/doc/man/man1/cdist-cache.text deleted file mode 100644 index 54619199..00000000 --- a/doc/man/man1/cdist-cache.text +++ /dev/null @@ -1,31 +0,0 @@ -cdist-cache(1) -============== -Nico Schottelius - - -NAME ----- -cdist-cache - Cache output of last run - - -SYNOPSIS --------- -cdist-cache TARGET_HOST - - -DESCRIPTION ------------ -cdist-cache moves away the objects created during last run so the -next run can use the previous information and compare them with -the current status. - - -SEE ALSO --------- -cdist(7) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-code-run.text b/doc/man/man1/cdist-code-run.text deleted file mode 100644 index e5d8c976..00000000 --- a/doc/man/man1/cdist-code-run.text +++ /dev/null @@ -1,34 +0,0 @@ -cdist-code-run(1) -================= -Nico Schottelius - - -NAME ----- -cdist-code-run - Run explorer remotely - - -SYNOPSIS --------- -cdist-code-run OBJECT_DIR OBJECT TYPE - - -DESCRIPTION ------------ -cdist-code-run executes generated code from a given OBJECT. -The OBJECT must be located below OBJECT_DIR. -TYPE must be either local or remote and determines which -code part is to be executed. - - -SEE ALSO --------- -- cdist(7) -- cdist-object-gencode(1) -- cdist-object-gencode-all(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-deploy-to.text b/doc/man/man1/cdist-deploy-to.text deleted file mode 100644 index f42a0509..00000000 --- a/doc/man/man1/cdist-deploy-to.text +++ /dev/null @@ -1,43 +0,0 @@ -cdist-deploy-to(1) -================== -Nico Schottelius - - -NAME ----- -cdist-deploy-to - Deploy configuration to host - - -SYNOPSIS --------- -cdist-deploy-to HOSTNAME - - -DESCRIPTION ------------ -Deploy configurations to the specified host, as configured in the initial -manifest. This script triggers the execution of several other scripts, in so -called stages. It is intented to run either from the command line or from cron. - - -ENVIRONMENT ------------ -If the environment variable **__cdist_conf_dir** is not set, the -configuration is read from /conf. The local output directory can -be changed by the variable **__cdist_local_base_dir**. All environment -variables are handled by cdist-config. - - -SEE ALSO --------- -- cdist(7) -- cdist-config(1) -- cdist-mass-deploy(1) -- cdist-reference(7) -- cdist-stages(7) - - -COPYING -------- -Copyright \(C) 2010-2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-dir.text b/doc/man/man1/cdist-dir.text deleted file mode 100644 index 223bc779..00000000 --- a/doc/man/man1/cdist-dir.text +++ /dev/null @@ -1,38 +0,0 @@ -cdist-dir(1) -============ -Nico Schottelius - - -NAME ----- -cdist-dir - Poor man's directory synchronisation - - -SYNOPSIS --------- -cdist-dir TARGET_HOST SRC_DIR DST_DIR - - -DESCRIPTION ------------ -cdist-dir either pushes a local directory to the target host -or pulls a remote directory from a target host to the local host. - -In the push case SRC_DIR is local, in the pull case remote. -In the push case DST_DIR is remote, in the pull case local. - -cdist-dir does not cleanup DST_DIR and thus it may contain old -stuff if used multiple times. - -cdist-dir does not rely on rsync or other high level tools, because -it cannot expect its existence on the local or target host. - -SEE ALSO --------- -cdist(7) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-explorer-run-global.text b/doc/man/man1/cdist-explorer-run-global.text deleted file mode 100644 index f4b32dfb..00000000 --- a/doc/man/man1/cdist-explorer-run-global.text +++ /dev/null @@ -1,31 +0,0 @@ -cdist-explorer-run-global(1) -============================ -Nico Schottelius - - -NAME ----- -cdist-explorer-run-global - Run the global explorers - - -SYNOPSIS --------- -cdist-explorer-run-global HOSTNAME - - -DESCRIPTION ------------ -Transfer the global explorers to HOSTNAME, execute them and transfer -back the results. - - -SEE ALSO --------- -- cdist(7) -- cdist-deploy-to(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-manifest-run-init.text b/doc/man/man1/cdist-manifest-run-init.text deleted file mode 100644 index 3a3265dc..00000000 --- a/doc/man/man1/cdist-manifest-run-init.text +++ /dev/null @@ -1,32 +0,0 @@ -cdist-manifest-run-init(1) -========================== -Nico Schottelius - - -NAME ----- -cdist-manifest-run-init - Run the initial manifest - - -SYNOPSIS --------- -cdist-manifest-run-init HOSTNAME - - -DESCRIPTION ------------ -cdist-manifest-run-init executes the initial manifest, which creates -the first objects. - - -SEE ALSO --------- -- cdist(7) -- cdist-deploy-to(1) -- cdist-manifest-run-all(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-manifest-run.text b/doc/man/man1/cdist-manifest-run.text deleted file mode 100644 index 8cd6b513..00000000 --- a/doc/man/man1/cdist-manifest-run.text +++ /dev/null @@ -1,31 +0,0 @@ -cdist-manifest-run(1) -===================== -Nico Schottelius - - -NAME ----- -cdist-manifest-run - Run a given manifest - - -SYNOPSIS --------- -cdist-manifest-run HOSTNAME MANIFEST - - -DESCRIPTION ------------ -cdist-manifest-run executes the given MANIFEST. - - -SEE ALSO --------- -- cdist(7) -- cdist-deploy-to(1) -- cdist-manifest-run-init(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-mass-deploy.text b/doc/man/man1/cdist-mass-deploy.text deleted file mode 100644 index ac495b21..00000000 --- a/doc/man/man1/cdist-mass-deploy.text +++ /dev/null @@ -1,41 +0,0 @@ -cdist-mass-deploy(1) -==================== -Nico Schottelius - - -NAME ----- -cdist-mass-deploy - Deploy configuration to many hosts - - -SYNOPSIS --------- -cdist-mass-deploy [-p] HOSTNAME [HOSTNAME ...] - - -DESCRIPTION ------------ -cdist-mass-deploy is essentially a wrapper around cdist-deploy-to to -be able to deploy to many hosts on one command line. - - -EXAMPLES --------- -Deploy in parallel to all hosts specfied in the dsh group configuration ikr, -which is prefixed by "root@": - --------------------------------------------------------------------------------- -cdist-mass-deploy -p $(cat ~/.dsh/group/ikr | sed 's/^root@//') --------------------------------------------------------------------------------- - - -SEE ALSO --------- -- cdist(7) -- cdist-deploy-to(1) - - -COPYING -------- -Copyright \(C) 2010-2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-all.text b/doc/man/man1/cdist-object-all.text deleted file mode 100644 index 06d45268..00000000 --- a/doc/man/man1/cdist-object-all.text +++ /dev/null @@ -1,31 +0,0 @@ -cdist-object-all(1) -=================== -Steven Armstrong - - -NAME ----- -cdist-object-all - Run the given command on all objects - - -SYNOPSIS --------- -cdist-object-all HOSTNAME COMMAND - - -DESCRIPTION ------------ -Iterates over all defined objects and executes the given command on each -of them. - - -SEE ALSO --------- -- cdist(7) -- cdist-type(1) - - -COPYING -------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-code-run.text b/doc/man/man1/cdist-object-code-run.text deleted file mode 100644 index f8bae6a4..00000000 --- a/doc/man/man1/cdist-object-code-run.text +++ /dev/null @@ -1,32 +0,0 @@ -cdist-object-code-run(1) -======================== -Nico Schottelius - - -NAME ----- -cdist-object-code-run - Execute the generated code for a object - - -SYNOPSIS --------- -cdist-object-code-run HOSTNAME OBJECT - - -DESCRIPTION ------------ -Execute the local and remote code for the given object. - - -SEE ALSO --------- -- cdist(7) -- cdist-object-run(1) -- cdist-code-run(1) -- cdist-run-remote(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius, Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-explorer-run.text b/doc/man/man1/cdist-object-explorer-run.text deleted file mode 100644 index a791681e..00000000 --- a/doc/man/man1/cdist-object-explorer-run.text +++ /dev/null @@ -1,31 +0,0 @@ -cdist-object-explorer-run(1) -============================ -Nico Schottelius - - -NAME ----- -cdist-object-explorer-run - Run type explorers for a object - - -SYNOPSIS --------- -cdist-object-explorer-run HOSTNAME OBJECT - - -DESCRIPTION ------------ -Runs the explorers for the given object on the target host. - - -SEE ALSO --------- -- cdist(7) -- cdist-deploy-to(1) -- cdist-remote-explorer-run(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius, Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-gencode-run.text b/doc/man/man1/cdist-object-gencode-run.text deleted file mode 100644 index 7705815c..00000000 --- a/doc/man/man1/cdist-object-gencode-run.text +++ /dev/null @@ -1,32 +0,0 @@ -cdist-object-gencode-run(1) -=========================== -Nico Schottelius - - -NAME ----- -cdist-object-gencode-run - Generate code for a object - - -SYNOPSIS --------- -cdist-object-gencode-run HOSTNAME OBJECT - - -DESCRIPTION ------------ -For the given object, generate the code for local and remote execution. - - -SEE ALSO --------- -- cdist(7) -- cdist-code-run(1) -- cdist-object-run(1) -- cdist-object-gencode(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius, Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-gencode.text b/doc/man/man1/cdist-object-gencode.text deleted file mode 100644 index 83f4b4c1..00000000 --- a/doc/man/man1/cdist-object-gencode.text +++ /dev/null @@ -1,33 +0,0 @@ -cdist-object-gencode(1) -======================= -Nico Schottelius - - -NAME ----- -cdist-object-gencode - Generate code for a given object - - -SYNOPSIS --------- -cdist-object-gencode HOSTNAME OBJECT - - -DESCRIPTION ------------ -For the given object, run the gencode executable. The output of this -executable on stdout will be used by cdist-object-gencode-all(1). - - -SEE ALSO --------- -- cdist(7) -- cdist-code-run(1) -- cdist-deploy-to(1) -- cdist-object-gencode-all(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-manifest-run.text b/doc/man/man1/cdist-object-manifest-run.text deleted file mode 100644 index a6f12f78..00000000 --- a/doc/man/man1/cdist-object-manifest-run.text +++ /dev/null @@ -1,31 +0,0 @@ -cdist-object-manifest-run(1) -============================ -Nico Schottelius - - -NAME ----- -cdist-object-manifest-run - Run an objects manifest - - -SYNOPSIS --------- -cdist-object-manifest-run HOSTNAME OBJECT - - -DESCRIPTION ------------ -Run the manifest for the given object. - - -SEE ALSO --------- -- cdist(7) -- cdist-deploy-to(1) -- cdist-manifest-run(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius, Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-prepare.text b/doc/man/man1/cdist-object-prepare.text deleted file mode 100644 index c91a7b2e..00000000 --- a/doc/man/man1/cdist-object-prepare.text +++ /dev/null @@ -1,35 +0,0 @@ -cdist-object-prepare(1) -======================= -Steven Armstrong - - -NAME ----- -cdist-object-prepare - Prepare an object - - -SYNOPSIS --------- -cdist-object-prepare HOSTNAME OBJECT - - -DESCRIPTION ------------ -Prepare the given object by running it through stage 3 (object information -retrieval) and stage 4 (run the object manifest). -See related man pages for details. - - -SEE ALSO --------- -- cdist(7) -- cdist-stages(7) -- cdist-object-explorer-run(1) -- cdist-object-manifest-run(1) -- cdist-type(1) - - -COPYING -------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is granted -under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-push.text b/doc/man/man1/cdist-object-push.text deleted file mode 100644 index 4c960eaa..00000000 --- a/doc/man/man1/cdist-object-push.text +++ /dev/null @@ -1,31 +0,0 @@ -cdist-object-push(1) -==================== -Nico Schottelius - - -NAME ----- -cdist-object-push - Transfer a object to the target host - - -SYNOPSIS --------- -cdist-object-push HOSTNAME OBJECT - - -DESCRIPTION ------------ -Transfers the given object to the target host. - - -SEE ALSO --------- -- cdist(7) -- cdist-object-run(1) -- cdist-type(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius, Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-object-run.text b/doc/man/man1/cdist-object-run.text deleted file mode 100644 index fc85a05f..00000000 --- a/doc/man/man1/cdist-object-run.text +++ /dev/null @@ -1,36 +0,0 @@ -cdist-object-run(1) -=================== -Steven Armstrong - - -NAME ----- -cdist-object-run - Run an object - - -SYNOPSIS --------- -cdist-object-run HOSTNAME OBJECT - - -DESCRIPTION ------------ -Applies the given object on the target host by running it through stage 5 -(code generation) and stage 6 (code execution). -See related man pages for details. - - -SEE ALSO --------- -- cdist(7) -- cdist-stages(7) -- cdist-object-gencode-run(1) -- cdist-object-push(1) -- cdist-object-code-run(1) -- cdist-type(1) - - -COPYING -------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-remote-explorer-run.text b/doc/man/man1/cdist-remote-explorer-run.text deleted file mode 100644 index 64951e2c..00000000 --- a/doc/man/man1/cdist-remote-explorer-run.text +++ /dev/null @@ -1,33 +0,0 @@ -cdist-remote-explorer-run(1) -============================ -Nico Schottelius - - -NAME ----- -cdist-remote-explorer-run - Run explorer remotely - - -SYNOPSIS --------- -cdist-remote-explorer-run VARIABLE_NAME EXPLORER_DIR OUT_DIR - - -DESCRIPTION ------------ -cdist-remote-explorer-run is executed on the target. -It sets up the variable VARIABLE_NAME to point to the given -EXPLORER_DIR and runs all explorer found in EXPLORER_DIR. -The output of every run explorer is saved into OUT_DIR. - - -SEE ALSO --------- -- cdist(7) -- cdist-explorer-run-global(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-run-remote.text b/doc/man/man1/cdist-run-remote.text deleted file mode 100644 index ee7a6337..00000000 --- a/doc/man/man1/cdist-run-remote.text +++ /dev/null @@ -1,33 +0,0 @@ -cdist-run-remote(1) -=================== -Nico Schottelius - - -NAME ----- -cdist-run-remote - Execute something on the target - - -SYNOPSIS --------- -cdist-run-remote HOSTNAME EXECUTABLE [ARGUMENTS FOR EXECUTABLE] - - -DESCRIPTION ------------ -cdist-run-remote runs the given executable on the remote host. -It ensures PATH is setup correctly on the target side. - - -SEE ALSO --------- -- cdist(7) -- cdist-object-code-run(1) -- cdist-deploy-to(1) -- cdist-remote-code-run-all(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/doc/man/man1/cdist-type-build-emulation.text b/doc/man/man1/cdist-type-build-emulation.text deleted file mode 100644 index 81c56e7c..00000000 --- a/doc/man/man1/cdist-type-build-emulation.text +++ /dev/null @@ -1,33 +0,0 @@ -cdist-type-build-emulation(1) -============================= -Nico Schottelius - - -NAME ----- -cdist-type-build-emulation - Build executables for types - - -SYNOPSIS --------- -cdist-type-build-emulation OUT_DIR - - -DESCRIPTION ------------ -cdist-type-build-emulation creates a link to cdist-type-emulator -for every TYPE. These links are placed in a OUT_DIR, which -is prepended into $PATH. This way the user can use TYPE in the -manifests like any other executable. - - -SEE ALSO --------- -- cdist(7) -- cdist-type-emulator(1) - - -COPYING -------- -Copyright \(C) 2011 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). From 4aff17610619bf7f8c8d53319f30bc4daab31d22 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 00:30:45 +0200 Subject: [PATCH 128/166] flush and also append \n to header Signed-off-by: Nico Schottelius --- bin/cdist | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 5eb678be..0e85a79c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -52,7 +52,7 @@ REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") -CODE_HEADER = "#!/bin/sh -e" +CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" VERSION = "2.0.0" @@ -501,7 +501,10 @@ class Cdist: os.path.basename(bin)[3:]) outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write outfile_fd.write(CODE_HEADER) + outfile_fd.flush() self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) outfile_fd.close() From e07328f569fbea0130442a71a22b976317306bf2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 01:14:35 +0200 Subject: [PATCH 129/166] initial support for parallel running Signed-off-by: Nico Schottelius --- bin/cdist | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/bin/cdist b/bin/cdist index 0e85a79c..c50775f8 100755 --- a/bin/cdist +++ b/bin/cdist @@ -23,6 +23,7 @@ import argparse import datetime import logging +import multiprocessing import os import subprocess import shutil @@ -573,6 +574,15 @@ class Cdist: self.target_host, duration.total_seconds()) + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() + + +def foo(): + print("test") + if __name__ == "__main__": parser = argparse.ArgumentParser(description='cdist ' + VERSION) parser.add_argument('host', nargs='*', help='one or more hosts to operate on') @@ -587,13 +597,12 @@ if __name__ == "__main__": parser.add_argument('-i', '--initial-manifest', help='Path to a cdist manifest or - to read from stdin', dest='manifest', required=False) -# parser.add_argument('-p', '--parallel', -# help='Operate on multiple hosts in parallel', -# action='store_true', dest='parallel') -# parser.add_argument('-s', '--sequential', -# help='Operate on multiple hosts sequentially', -# action='store_false', dest='parallel') - + parser.add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser.add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially', + action='store_false', dest='parallel') parser.add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + VERSION) @@ -606,10 +615,26 @@ if __name__ == "__main__": banner() sys.exit(0) + process = {} + time_start = datetime.datetime.now() try: for host in args.host: c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home) - c.deploy_to() - c.cleanup() + if args.parallel: + log.info("Starting child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + log.debug("After process for %s", host) + else: + c.deploy_and_cleanup() + + if args.parallel: + for p in process.keys(): + log.debug("Joining %s", p) + process[p].join() + + time_end = datetime.datetime.now() + log.info("Total processing time: %s", (time_end - time_start).total_seconds()) + except KeyboardInterrupt: sys.exit(0) From 5ba6e0bb142edf0f0fc54c9eedcf0a4a933f2355 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 01:38:51 +0200 Subject: [PATCH 130/166] more hints on parallel running Signed-off-by: Nico Schottelius --- bin/cdist | 3 ++- doc/dev/todo/niconext | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index c50775f8..f8500ca7 100755 --- a/bin/cdist +++ b/bin/cdist @@ -618,10 +618,11 @@ if __name__ == "__main__": process = {} time_start = datetime.datetime.now() try: + log.info("Deploying to %s hosts", len(args.host)) for host in args.host: c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home) if args.parallel: - log.info("Starting child process for %s", host) + log.debug("Starting child process for %s", host) process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) process[host].start() log.debug("After process for %s", host) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 3968b0ef..2321bf03 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,4 +1,6 @@ - Remove all FIXME entries +- Support parallel execution + - and maximum number of parallel runs (-p X) - Write cdist-manpage - Remove obsolete manpages - Support different home instead of ../ From 25190e86a37f47bd889948e6bb0d13bfc80f4d69 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 01:39:56 +0200 Subject: [PATCH 131/166] todo for 2.0.0 Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 2321bf03..2b1bb703 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,8 +1,13 @@ -- Remove all FIXME entries +For 2.0.0: + + - Write cdist-manpage + - Remove all FIXME entries + - Remove obsolete manpages + +-------------------------------------------------------------------------------- + - Support parallel execution - and maximum number of parallel runs (-p X) -- Write cdist-manpage -- Remove obsolete manpages - Support different home instead of ../ - Bug: os.path.join() may be wrong for the remote side! -> does not matter for now! From 7de51e8d45c43d43ab3f97aec53016562ae47d87 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 10:52:04 +0200 Subject: [PATCH 132/166] print help in case no hosts are given Signed-off-by: Nico Schottelius --- bin/cdist | 11 ++++++++--- doc/dev/todo/niconext | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index f8500ca7..628cbca4 100755 --- a/bin/cdist +++ b/bin/cdist @@ -616,9 +616,13 @@ if __name__ == "__main__": sys.exit(0) process = {} - time_start = datetime.datetime.now() try: - log.info("Deploying to %s hosts", len(args.host)) + if len(args.host) == 0: + parser.print_help() + sys.exit(1) + + time_start = datetime.datetime.now() + for host in args.host: c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home) if args.parallel: @@ -635,7 +639,8 @@ if __name__ == "__main__": process[p].join() time_end = datetime.datetime.now() - log.info("Total processing time: %s", (time_end - time_start).total_seconds()) + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) except KeyboardInterrupt: sys.exit(0) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 2b1bb703..ef861cec 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -8,6 +8,7 @@ For 2.0.0: - Support parallel execution - and maximum number of parallel runs (-p X) + - error handling / report failed hosts - Support different home instead of ../ - Bug: os.path.join() may be wrong for the remote side! -> does not matter for now! From 1742936dc9a6fc8a4273bc3f3b7a8e98a1faec64 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 10:58:10 +0200 Subject: [PATCH 133/166] add initial version of manpage for cdist Signed-off-by: Nico Schottelius --- doc/man/man1/cdist.text | 63 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/man/man1/cdist.text diff --git a/doc/man/man1/cdist.text b/doc/man/man1/cdist.text new file mode 100644 index 00000000..368d9097 --- /dev/null +++ b/doc/man/man1/cdist.text @@ -0,0 +1,63 @@ +cdist(1) +======== +Nico Schottelius + + +NAME +---- +cdist - Configuration management + + +SYNOPSIS +-------- +cdist [-h] [-b] [-c CDIST_HOME] [-d] [-i MANIFEST] [-p] [-s] [-V] [host [host ...]] + + +DESCRIPTION +----------- +cdist is the frontend executable to the cdist configuration management. + + +OPTIONS +------- +-h, --help:: + Show the help screen + +-d, --debug:: + Enable debug output + +-p, --parallel:: + Parallelise backup processes + +-V, --version:: + Show version and exit + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Configure ikq05.ethz.ch with debug enabled +cdist -d ikq05.ethz.ch + +__motd +-------------------------------------------------------------------------------- + +In both cases, cdist-type-emulator is called instead of a real type. +In the first case, the object id "/tmp/linetest" is recorded and the +parameter "line" stored with the content "test". + +In the second case, __motd must be decleared as a singleton, as the +object id is missing. + + +SEE ALSO +-------- +- cdist(7) +- cdist-type-build-emulation(1) + + +COPYING +------- +Copyright \(C) 2011 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). From b470a47b15a4b5f77e7a73514f972aedb05f8e8c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 11:18:18 +0200 Subject: [PATCH 134/166] add working cdist manpage Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- doc/man/man1/cdist.text | 30 +++++++++++++++++++++--------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 628cbca4..7fb73e65 100755 --- a/bin/cdist +++ b/bin/cdist @@ -601,7 +601,7 @@ if __name__ == "__main__": help='Operate on multiple hosts in parallel', action='store_true', dest='parallel') parser.add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially', + help='Operate on multiple hosts sequentially (default)', action='store_false', dest='parallel') parser.add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + VERSION) diff --git a/doc/man/man1/cdist.text b/doc/man/man1/cdist.text index 368d9097..f45d4495 100644 --- a/doc/man/man1/cdist.text +++ b/doc/man/man1/cdist.text @@ -23,11 +23,24 @@ OPTIONS -h, --help:: Show the help screen +-b, --banner:: + Show cdist banner + +-c CDIST_HOME, --cdist-home CDIST_HOME:: + Instead of using the parent of the bin directory as cdist home, + use the specified directory + -d, --debug:: Enable debug output +-i MANIFEST, --initial-manifest MANIFEST:: + Path to a cdist manifest or - to read from stdin + -p, --parallel:: - Parallelise backup processes + Operate on multiple hosts in parallel + +-s, --sequential:: + Operate on multiple hosts sequentially -V, --version:: Show version and exit @@ -40,16 +53,15 @@ EXAMPLES # Configure ikq05.ethz.ch with debug enabled cdist -d ikq05.ethz.ch -__motd +# Configure hosts in parallel and use a different home directory +cdist -c ~/p/cdist-nutzung -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch + +# Does not need a comment, plain works +cdist --banner +cdist --help +cdist --version -------------------------------------------------------------------------------- -In both cases, cdist-type-emulator is called instead of a real type. -In the first case, the object id "/tmp/linetest" is recorded and the -parameter "line" stored with the content "test". - -In the second case, __motd must be decleared as a singleton, as the -object id is missing. - SEE ALSO -------- From 4e95535b7c9912729b50a97156e27399f046843b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 11:27:20 +0200 Subject: [PATCH 135/166] definitely cleanup remote directory first, so we see more bugs :-) Signed-off-by: Nico Schottelius --- bin/cdist | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7fb73e65..98e92c9a 100755 --- a/bin/cdist +++ b/bin/cdist @@ -327,6 +327,7 @@ class Cdist: def transfer_global_explorers(self): """Transfer the global explorers""" + self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) def transfer_type_explorers(self, type): @@ -406,12 +407,9 @@ class Cdist: def init_deploy(self): log.debug("Creating clean directory structure") - # Ensure there is no old stuff, neither local nor remote - # remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"]) - # - # # Create base directories - # remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"]) - # + self.remove_remote_dir(REMOTE_BASE_DIR) + self.remote_mkdir(REMOTE_BASE_DIR) + # # Link configuraion source directory - consistent with remote # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) From 8b37daf5ff291d0857f9ebb7e3388c532179ea07 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 11:27:30 +0200 Subject: [PATCH 136/166] --todo for 2.0.0 Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index ef861cec..22f9f7b7 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,6 +1,5 @@ For 2.0.0: - - Write cdist-manpage - Remove all FIXME entries - Remove obsolete manpages From a09a618c781327b36aa7303a7acdff4f3c488a58 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Sep 2011 11:37:20 +0200 Subject: [PATCH 137/166] prepare cdist-type-emulator transformation Signed-off-by: Nico Schottelius --- bin/cdist | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bin/cdist b/bin/cdist index 98e92c9a..dc6a5a5f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -65,6 +65,34 @@ def banner(): print(BANNER) +class TypeEmulator: + + def type_emulator(): + type = basename(sys.argv[0]) + + type_is_singleton(type) + + # Check object id + + # Prevent double slash if id begins with / + + # Record parameter: opt_file="${opt#--}" + # [ $# -ge 1 ] || __cdist_usage "Missing value for $opt" + # echo "${value}" > "${__cdist_parameter_dir}/${opt_file}" + + # Record requirements + # echo $requirement >> "$(__cdist_object_require "$__cdist_object_self")" + + # Ensure required parameters are given + # Ensure that only optional or required parameters are given + # [ "$is_valid" ] || __cdist_usage "Unknown parameter $parameter" + + # Merge object (creating twice with the same parameter + requirements == allowed) + + # diff -ru "${__cdist_new_object_dir}/${__cdist_name_parameter} + # # Add "I was here message" + # _cdist_object_source_add "${__cdist_object_dir}" + class Cdist: """Cdist main class to hold arbitrary data""" From 99b3b0789bbc7385a2882dbcba52514d6baa4782 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 00:02:04 +0200 Subject: [PATCH 138/166] remove tmpdir, keep it only if debug is enabled Signed-off-by: Nico Schottelius --- bin/cdist | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/bin/cdist b/bin/cdist index dc6a5a5f..e20a9f49 100755 --- a/bin/cdist +++ b/bin/cdist @@ -66,8 +66,12 @@ def banner(): class TypeEmulator: + def __init__(self, name): + self.name = name + self.type = os.path.basename(name) - def type_emulator(): + + def type_emulator(self): type = basename(sys.argv[0]) type_is_singleton(type) @@ -96,13 +100,16 @@ class TypeEmulator: class Cdist: """Cdist main class to hold arbitrary data""" - def __init__(self, target_host, initial_manifest=False, remote_user="root", home=None): + def __init__(self, target_host, + initial_manifest=False, remote_user="root", + home=None, debug=False): self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] # Setup directory paths self.temp_dir = tempfile.mkdtemp() + self.debug = debug if home: self.base_dir = home @@ -148,8 +155,10 @@ class Cdist: # "other globals referenced by the __del__() method may already have been deleted # or in the process of being torn down (e.g. the import machinery shutting down)" # - print("I should cleanup " + self.temp_dir) - # shutil.rmtree(self.temp_dir) + if self.debug: + log.debug("Skipping removal of " + self.temp_dir) + else: + shutil.rmtree(self.temp_dir) def exit_error(self, *args): log.error(*args) @@ -650,7 +659,7 @@ if __name__ == "__main__": time_start = datetime.datetime.now() for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home) + c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) if args.parallel: log.debug("Starting child process for %s", host) process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) From 4e3fec1e9e6acb8874ca7aee5573f91555c3a796 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 16:10:03 +0200 Subject: [PATCH 139/166] add hint to pythons octal mode Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index e20a9f49..f50bb1fb 100755 --- a/bin/cdist +++ b/bin/cdist @@ -551,7 +551,7 @@ class Cdist: if status.st_size == len(CODE_HEADER): os.unlink(outfile) else: - # Add header and make executable + # Add header and make executable - identically to 0o700 os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) if mode == "code": From a43474cd68ba1256db5389d7f8c7301c9527f29d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 16:10:46 +0200 Subject: [PATCH 140/166] add logfiles Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-09-12 | 67 +++++++++++++++++++++++++++++++++++++++++ doc/dev/logs/2011-09-13 | 3 ++ 2 files changed, 70 insertions(+) create mode 100644 doc/dev/logs/2011-09-12 create mode 100644 doc/dev/logs/2011-09-13 diff --git a/doc/dev/logs/2011-09-12 b/doc/dev/logs/2011-09-12 new file mode 100644 index 00000000..391b836f --- /dev/null +++ b/doc/dev/logs/2011-09-12 @@ -0,0 +1,67 @@ +Benchmark from home/X201 (2 cores, 4 threads): + + ikq (1): 72.192397 + ikq (2): INFO: Total processing time for 2 hosts: 74.1845 + ikq* (6): INFO: Total processing time: 117.572312 + ikq* + ikr3 (9): INFO: Total processing time: 120.307662 + ikq* + ikr (14): INFO: Total processing time: 139.769807 + ikq* + ikr (18): INFO: Total processing time: 186.354398 + ikq* + ikr (22): INFO: Total processing time: 225.793533 + ikq* + ikr (26): INFO: Total processing time: 237.06687 + ikq* + ikr (31): INFO: Total processing time: 276.912414 + + +ikr07.ethz.ch ikr09.ethz.ch ikr10.ethz.ch ikr11.ethz.ch +ikr13.ethz.ch ikr14.ethz.ch ikr15.ethz.ch ikr16.ethz.ch +ikr17.ethz.ch ikr19.ethz.ch ikr20.ethz.ch ikr21.ethz.ch +ikr23.ethz.ch ikr24.ethz.ch ikr25.ethz.ch ikr26.ethz.ch +ikr27.ethz.ch ikr28.ethz.ch ikr29.ethz.ch ikr30.ethz.ch +ikr31.ethz.ch + +cdist -c ~/p/cdist-nutzung -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch ikq05.ethz.ch ikq06.ethz.ch ikq07.ethz.ch ikr01.ethz.ch ikr02.ethz.ch ikr03.ethz.ch ikr05.ethz.ch ikr07.ethz.ch ikr09.ethz.ch ikr10.ethz.ch ikr11.ethz.ch ikr13.ethz.ch ikr14.ethz.ch ikr15.ethz.ch ikr16.ethz.ch ikr17.ethz.ch ikr19.ethz.ch ikr20.ethz.ch ikr21.ethz.ch + + +-------------------------------------------------------------------------------- +INFO: Total processing time for 1 hosts: 72.166661 +INFO: Total processing time for 2 hosts: 76.633228 +INFO: Total processing time for 3 host(s): 77.199817 +INFO: Total processing time for 4 host(s): 94.045175 +INFO: Total processing time for 5 host(s): 103.226354 +INFO: Total processing time for 6 host(s): 107.76097 +INFO: Total processing time for 7 host(s): 101.571705 +INFO: Total processing time for 8 host(s): 107.600093 +INFO: Total processing time for 9 host(s): 116.500371 +INFO: Total processing time for 10 host(s): 119.445805 +INFO: Total processing time for 11 host(s): 123.944385 +INFO: Total processing time for 12 host(s): 130.499098 +INFO: Total processing time for 13 host(s): 137.250861 +INFO: Total processing time for 14 host(s): 154.9841 +INFO: Total processing time for 15 host(s): 139.659637 +INFO: Total processing time for 16 host(s): 142.70005 +INFO: Total processing time for 17 host(s): 148.541452 +INFO: Total processing time for 18 host(s): 159.360809 +INFO: Total processing time for 19 host(s): 171.907864 +INFO: Total processing time for 20 host(s): 178.76695 +INFO: Total processing time for 21 host(s): 183.856671 +INFO: Total processing time for 22 host(s): 194.504221 +INFO: Total processing time for 23 host(s): 207.314842 +INFO: Total processing time for 24 host(s): 215.846502 +INFO: Total processing time for 25 host(s): 217.223581 +INFO: Total processing time for 26 host(s): 238.591705 +INFO: Total processing time for 27 host(s): 238.478493 +INFO: Total processing time for 28 host(s): 246.058718 +INFO: Total processing time for 29 host(s): 264.208372 +INFO: Total processing time for 30 host(s): 265.560685 +INFO: Total processing time for 31 host(s): 282.264488 + +-------------------------------------------------------------------------------- +Use: + grep "^INFO: Total processing time" doc/dev/logs/2011-09-12 | sed 's/.*: //' + octave + times = [ /* paste here ] + plot(times) + # keep the graph + hold on + # Scale linearly with the single host value + plot((1:31)*times(1)) + diff --git a/doc/dev/logs/2011-09-13 b/doc/dev/logs/2011-09-13 new file mode 100644 index 00000000..b55a2d4e --- /dev/null +++ b/doc/dev/logs/2011-09-13 @@ -0,0 +1,3 @@ +Name for installer: + installer (flag) + $__installer (variable) - gesetzt oder nicht From a5b56f0c4111d28a1614db9d01ea321725694b19 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 16:19:43 +0200 Subject: [PATCH 141/166] remove useless function Signed-off-by: Nico Schottelius --- bin/cdist | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/bin/cdist b/bin/cdist index f50bb1fb..fda94888 100755 --- a/bin/cdist +++ b/bin/cdist @@ -352,16 +352,6 @@ class Cdist: self.transfer_dir(self.object_parameter_dir(cdist_object), self.remote_object_parameter_dir(cdist_object)) - def transfer_object_code(self, cdist_object): - FIXME - """Transfer the object code to the remote destination""" - # Create base path before using mkdir -p - self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) - - # Synchronise parameter dir afterwards - self.transfer_file(self.object_code_path(cdist_object), - self.remote_object_parameter_dir(cdist_object)) - def transfer_global_explorers(self): """Transfer the global explorers""" self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) @@ -555,7 +545,6 @@ class Cdist: os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) if mode == "code": -# paths = self.object_code_paths(cdist_object) local_dir = self.object_dir(cdist_object) remote_dir = self.remote_object_dir(cdist_object) From 59afce35b53419a20aac42d73d7f563fd71e8e85 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 16:25:39 +0200 Subject: [PATCH 142/166] do not do more checks on the manifest, current ones are good Signed-off-by: Nico Schottelius --- bin/cdist | 1 - doc/dev/todo/niconext | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index fda94888..873fbb1a 100755 --- a/bin/cdist +++ b/bin/cdist @@ -451,7 +451,6 @@ class Cdist: manifest = self.type_manifest_path(type) log.debug("%s: Running %s", cdist_object, manifest) - # FIXME: add more sensible checks for manifest if os.path.exists(manifest): env = { "__object" : self.object_dir(cdist_object), "__object_id": self.get_object_id_from_object(cdist_object), diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 22f9f7b7..738cd692 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -2,6 +2,8 @@ For 2.0.0: - Remove all FIXME entries - Remove obsolete manpages + - support subcommands! + deploy? config? -------------------------------------------------------------------------------- @@ -15,3 +17,4 @@ For 2.0.0: - Rewrite cdist-type-emulator - Remove legacy code in cdist - Remove cdist-config + From f4faff6b33a05fb2887d521651e705208e928431 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 16:29:33 +0200 Subject: [PATCH 143/166] --todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 738cd692..b1b77a27 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,7 +1,5 @@ For 2.0.0: - - Remove all FIXME entries - - Remove obsolete manpages - support subcommands! deploy? config? @@ -17,4 +15,6 @@ For 2.0.0: - Rewrite cdist-type-emulator - Remove legacy code in cdist - Remove cdist-config + - Remove man1/cdist-type-emulator.text +- Replace bin/cdist-type-template From 5cbe34ee336ee51c5666b774654afcd706f61579 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 17:40:28 +0200 Subject: [PATCH 144/166] begin to implement subcommand handling Signed-off-by: Nico Schottelius --- bin/cdist | 98 ++++++++++++++++++++++++++----------------- doc/dev/todo/niconext | 3 +- 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/bin/cdist b/bin/cdist index 873fbb1a..99bc775f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -60,10 +60,6 @@ VERSION = "2.0.0" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() -def banner(): - """Guess what :-)""" - print(BANNER) - class TypeEmulator: def __init__(self, name): @@ -602,42 +598,13 @@ class Cdist: self.deploy_to() self.cleanup() +def banner(*args): + """Guess what :-)""" + print(BANNER) + sys.exit(0) -def foo(): - print("test") - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='cdist ' + VERSION) - parser.add_argument('host', nargs='*', help='one or more hosts to operate on') - parser.add_argument('-b', '--banner', - help='Show cdist banner', - action='store_true', dest='banner') - parser.add_argument('-c', '--cdist-home', - help='Change cdist home (default: .. from bin directory)', - action='store') - parser.add_argument('-d', '--debug', help='Set log level to debug', - action='store_true') - parser.add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest or - to read from stdin', - dest='manifest', required=False) - parser.add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser.add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') - parser.add_argument('-V', '--version', help='Show version', - action='version', version='%(prog)s ' + VERSION) - - args = parser.parse_args(sys.argv[1:]) - log.debug(args) - if args.debug: - logging.root.setLevel(logging.DEBUG) - - if args.banner: - banner() - sys.exit(0) - +def config(*args, **kargs): + """Configure remote system""" process = {} try: if len(args.host) == 0: @@ -667,3 +634,56 @@ if __name__ == "__main__": except KeyboardInterrupt: sys.exit(0) + +if __name__ == "__main__": + # Construct parser others can reuse + parser = {} + # Options _all_ parsers have in common + parser['most'] = argparse.ArgumentParser(add_help=False) + parser['most'].add_argument('-d', '--debug', + help='Set log level to debug', + action='store_true') + parser['most'].add_argument('-V', '--version', + help='Show version', + action='version', + version='%(prog)s ' + VERSION) + + # Main subcommand parser + parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) + parser['sub'] = parser['main'].add_subparsers() + + + parser['banner'] = parser['sub'].add_parser('banner') +# parser['banner'].add_argument('all', nargs='*', +# help='Show cdist banner') + parser['banner'].set_defaults(func=banner) + + parser['config'] = parser['sub'].add_parser('config', + aliases=['deploy'], + parents=[parser['most']]) + parser['config'].add_argument('host', nargs='*', + help='one or more hosts to operate on') + parser['config'].add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store') + parser['config'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest or - to read from stdin', + dest='manifest', required=False) + parser['config'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['config'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') + parser['config'].set_defaults(func=config) + + + args = parser['main'].parse_args(sys.argv[1:]) + + # Most subcommands to have --debug, so handle it here + if 'debug' in args: + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) + + args.func(args) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index b1b77a27..6b87b8fe 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,7 +1,8 @@ For 2.0.0: - support subcommands! - deploy? config? + config (deploy) + install (provision) -------------------------------------------------------------------------------- From df7f27f94b881f5e5b4dbc03f0393c74434dc148 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 17:42:43 +0200 Subject: [PATCH 145/166] catch keyboard interrupt for all subcommands Signed-off-by: Nico Schottelius --- bin/cdist | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/bin/cdist b/bin/cdist index 99bc775f..fad85dca 100755 --- a/bin/cdist +++ b/bin/cdist @@ -603,10 +603,10 @@ def banner(*args): print(BANNER) sys.exit(0) -def config(*args, **kargs): +def config(*args): """Configure remote system""" + print(*args) process = {} - try: if len(args.host) == 0: parser.print_help() sys.exit(1) @@ -632,8 +632,6 @@ def config(*args, **kargs): log.info("Total processing time for %s host(s): %s", len(args.host), (time_end - time_start).total_seconds()) - except KeyboardInterrupt: - sys.exit(0) if __name__ == "__main__": # Construct parser others can reuse @@ -652,10 +650,7 @@ if __name__ == "__main__": parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) parser['sub'] = parser['main'].add_subparsers() - parser['banner'] = parser['sub'].add_parser('banner') -# parser['banner'].add_argument('all', nargs='*', -# help='Show cdist banner') parser['banner'].set_defaults(func=banner) parser['config'] = parser['sub'].add_parser('config', @@ -677,7 +672,6 @@ if __name__ == "__main__": action='store_false', dest='parallel') parser['config'].set_defaults(func=config) - args = parser['main'].parse_args(sys.argv[1:]) # Most subcommands to have --debug, so handle it here @@ -686,4 +680,7 @@ if __name__ == "__main__": logging.root.setLevel(logging.DEBUG) log.debug(args) - args.func(args) + try: + args.func(args) + except KeyboardInterrupt: + sys.exit(0) From 8d1b4236d4b822a8cf2774020cedf5434769a9b9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 17:59:11 +0200 Subject: [PATCH 146/166] fixup most arguments for config Signed-off-by: Nico Schottelius --- bin/cdist | 55 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/bin/cdist b/bin/cdist index fad85dca..339ecfa8 100755 --- a/bin/cdist +++ b/bin/cdist @@ -598,46 +598,48 @@ class Cdist: self.deploy_to() self.cleanup() -def banner(*args): +def banner(args): """Guess what :-)""" print(BANNER) sys.exit(0) -def config(*args): +def config(args): """Configure remote system""" - print(*args) + print(args) process = {} - if len(args.host) == 0: - parser.print_help() - sys.exit(1) - time_start = datetime.datetime.now() + if len(args.host) == 0: + parser.print_help() + sys.exit(1) - for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) - if args.parallel: - log.debug("Starting child process for %s", host) - process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) - process[host].start() - log.debug("After process for %s", host) - else: - c.deploy_and_cleanup() + time_start = datetime.datetime.now() + for host in args.host: + c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) if args.parallel: - for p in process.keys(): - log.debug("Joining %s", p) - process[p].join() + log.debug("Starting child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + log.debug("After process for %s", host) + else: + c.deploy_and_cleanup() - time_end = datetime.datetime.now() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start).total_seconds()) + if args.parallel: + for p in process.keys(): + log.debug("Joining %s", p) + process[p].join() + + time_end = datetime.datetime.now() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) if __name__ == "__main__": # Construct parser others can reuse parser = {} # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) + parser['most'] = argparse.ArgumentParser(add_help=False, + epilog="Get cdist at http://www.nico.schottelius.org/software/cdist/") parser['most'].add_argument('-d', '--debug', help='Set log level to debug', action='store_true') @@ -645,9 +647,12 @@ if __name__ == "__main__": help='Show version', action='version', version='%(prog)s ' + VERSION) + #version='%(prog)s ' + VERSION) + # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) + parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION, + parents=[parser['most']]) parser['sub'] = parser['main'].add_subparsers() parser['banner'] = parser['sub'].add_parser('banner') @@ -656,7 +661,7 @@ if __name__ == "__main__": parser['config'] = parser['sub'].add_parser('config', aliases=['deploy'], parents=[parser['most']]) - parser['config'].add_argument('host', nargs='*', + parser['config'].add_argument('host', nargs='+', help='one or more hosts to operate on') parser['config'].add_argument('-c', '--cdist-home', help='Change cdist home (default: .. from bin directory)', From 4d065443da3e9f676ff8b0973b2073243d664d76 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 18:00:41 +0200 Subject: [PATCH 147/166] remove alias deploy for config, does not make things better Signed-off-by: Nico Schottelius --- bin/cdist | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 339ecfa8..7e3e9a1c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -659,7 +659,6 @@ if __name__ == "__main__": parser['banner'].set_defaults(func=banner) parser['config'] = parser['sub'].add_parser('config', - aliases=['deploy'], parents=[parser['most']]) parser['config'].add_argument('host', nargs='+', help='one or more hosts to operate on') From 581562aae96890c027fe367478ea429ff0b122d9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 18:05:15 +0200 Subject: [PATCH 148/166] cleanups Signed-off-by: Nico Schottelius --- bin/cdist | 22 +++++++--------------- doc/dev/todo/niconext | 10 ++-------- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7e3e9a1c..42838093 100755 --- a/bin/cdist +++ b/bin/cdist @@ -428,14 +428,12 @@ class Cdist: output_fd.close() def init_deploy(self): + """Ensure the base directories are cleaned up""" log.debug("Creating clean directory structure") self.remove_remote_dir(REMOTE_BASE_DIR) self.remote_mkdir(REMOTE_BASE_DIR) - # # Link configuraion source directory - consistent with remote - # run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"]) - def run_initial_manifest(self): """Run the initial manifest""" env = { "__manifest" : self.manifest_dir } @@ -605,7 +603,6 @@ def banner(args): def config(args): """Configure remote system""" - print(args) process = {} if len(args.host) == 0: @@ -633,7 +630,6 @@ def config(args): log.info("Total processing time for %s host(s): %s", len(args.host), (time_end - time_start).total_seconds()) - if __name__ == "__main__": # Construct parser others can reuse parser = {} @@ -641,14 +637,10 @@ if __name__ == "__main__": parser['most'] = argparse.ArgumentParser(add_help=False, epilog="Get cdist at http://www.nico.schottelius.org/software/cdist/") parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', - action='store_true') + help='Set log level to debug', action='store_true') parser['most'].add_argument('-V', '--version', - help='Show version', - action='version', - version='%(prog)s ' + VERSION) - #version='%(prog)s ' + VERSION) - + help='Show version', action='version', + version='%(prog)s ' + VERSION) # Main subcommand parser parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION, @@ -659,14 +651,14 @@ if __name__ == "__main__": parser['banner'].set_defaults(func=banner) parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most']]) + parents=[parser['most']]) parser['config'].add_argument('host', nargs='+', help='one or more hosts to operate on') parser['config'].add_argument('-c', '--cdist-home', help='Change cdist home (default: .. from bin directory)', action='store') parser['config'].add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest or - to read from stdin', + help='Path to a cdist manifest', dest='manifest', required=False) parser['config'].add_argument('-p', '--parallel', help='Operate on multiple hosts in parallel', @@ -678,7 +670,7 @@ if __name__ == "__main__": args = parser['main'].parse_args(sys.argv[1:]) - # Most subcommands to have --debug, so handle it here + # Most subcommands have --debug, so handle it here if 'debug' in args: if args.debug: logging.root.setLevel(logging.DEBUG) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 6b87b8fe..63f9b629 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,11 +1,3 @@ -For 2.0.0: - - - support subcommands! - config (deploy) - install (provision) - --------------------------------------------------------------------------------- - - Support parallel execution - and maximum number of parallel runs (-p X) - error handling / report failed hosts @@ -19,3 +11,5 @@ For 2.0.0: - Remove man1/cdist-type-emulator.text - Replace bin/cdist-type-template + +- Allow manifest to be read from stdin From 780fd5625e2efdfa86bbafe766186143bcb9ce76 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 18:15:04 +0200 Subject: [PATCH 149/166] re-implement correct caching Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 42838093..530618f1 100755 --- a/bin/cdist +++ b/bin/cdist @@ -113,6 +113,8 @@ class Cdist: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") self.lib_dir = os.path.join(self.base_dir, "lib") self.manifest_dir = os.path.join(self.conf_dir, "manifest") @@ -151,10 +153,11 @@ class Cdist: # "other globals referenced by the __del__() method may already have been deleted # or in the process of being torn down (e.g. the import machinery shutting down)" # - if self.debug: - log.debug("Skipping removal of " + self.temp_dir) - else: - shutil.rmtree(self.temp_dir) + log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) + # Remove previous cache + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + os.rename(self.temp_dir, self.cache_dir) def exit_error(self, *args): log.error(*args) From 7e9d2342197d3b8f6564a27aed61cc7aa505df1a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 18:18:12 +0200 Subject: [PATCH 150/166] remove obsolete length check Signed-off-by: Nico Schottelius --- bin/cdist | 4 ---- 1 file changed, 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index 530618f1..a468e2fc 100755 --- a/bin/cdist +++ b/bin/cdist @@ -608,10 +608,6 @@ def config(args): """Configure remote system""" process = {} - if len(args.host) == 0: - parser.print_help() - sys.exit(1) - time_start = datetime.datetime.now() for host in args.host: From e1421280d4700f505f320e126abc4de8bafab347 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 18:18:56 +0200 Subject: [PATCH 151/166] remove debug Signed-off-by: Nico Schottelius --- bin/cdist | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index a468e2fc..ad5584af 100755 --- a/bin/cdist +++ b/bin/cdist @@ -613,10 +613,9 @@ def config(args): for host in args.host: c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) if args.parallel: - log.debug("Starting child process for %s", host) + log.debug("Creating child process for %s", host) process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) process[host].start() - log.debug("After process for %s", host) else: c.deploy_and_cleanup() From 9f6a3933c7eb535f751f85ad64d013615976e441 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 18:24:24 +0200 Subject: [PATCH 152/166] remove obsolete HACKERS_README, we are stable now Signed-off-by: Nico Schottelius --- HACKERS_README | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100755 HACKERS_README diff --git a/HACKERS_README b/HACKERS_README deleted file mode 100755 index ad3fe3fd..00000000 --- a/HACKERS_README +++ /dev/null @@ -1,40 +0,0 @@ -cat << eof - -Hey hackers, - -this README is for you, for those who want to dig into cdist, hack it or try -to get a deeper understanding. Please read doc/man/man7/cdist-hacker.text. - -I hope you have a lot of fun with cdist, because it was also a lot of fun to -develop it! - - -- Nico, 20110324 - -## Running cdist when developing - -This file is suitable for execution and saving the objects and -explorers from cdist. I usually do it like this: - - % ./HACKERS_README - -################################################################################ -eof - -set -x -# Tell the user what we do, so this script makes sense during execution - -# prepare use (only from top level directory) -export PATH="$(pwd -P)/bin:$PATH" -export __cdist_conf_dir="$(pwd -P)/conf" - -# Allow user to supply hostname -target="${1:-localhost}" - -# And use hostname as basedir (dangerous, but hackers know what they do) -export __cdist_local_base_dir="/tmp/$target" - -# Run the real script -cdist-deploy-to "$target" - -# Display results -find "${__cdist_local_base_dir}" From 90bdf0433672456653d2d3425b5c0689bb52135b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 21:30:17 +0200 Subject: [PATCH 153/166] add epilog to parsers, make it look beautiful Signed-off-by: Nico Schottelius --- bin/cdist | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index ad5584af..5333cb3a 100755 --- a/bin/cdist +++ b/bin/cdist @@ -632,20 +632,20 @@ if __name__ == "__main__": # Construct parser others can reuse parser = {} # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False, - epilog="Get cdist at http://www.nico.schottelius.org/software/cdist/") + parser['most'] = argparse.ArgumentParser(add_help=False) parser['most'].add_argument('-d', '--debug', help='Set log level to debug', action='store_true') - parser['most'].add_argument('-V', '--version', - help='Show version', action='version', - version='%(prog)s ' + VERSION) # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION, - parents=[parser['most']]) + parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) + + parser['main'].add_argument('-V', '--version', + help='Show version', action='version', + version='%(prog)s ' + VERSION) parser['sub'] = parser['main'].add_subparsers() - parser['banner'] = parser['sub'].add_parser('banner') + parser['banner'] = parser['sub'].add_parser('banner', + add_help=False) parser['banner'].set_defaults(func=banner) parser['config'] = parser['sub'].add_parser('config', @@ -666,6 +666,9 @@ if __name__ == "__main__": action='store_false', dest='parallel') parser['config'].set_defaults(func=config) + for p in parser: + parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" + args = parser['main'].parse_args(sys.argv[1:]) # Most subcommands have --debug, so handle it here From 9b18b3d79e93b2b0aef84e102c0f39962b7e9801 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 21:35:57 +0200 Subject: [PATCH 154/166] update manpage and support commands title Signed-off-by: Nico Schottelius --- bin/cdist | 5 +++-- doc/man/man1/cdist.text | 38 ++++++++++++++++++++++++++------------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/bin/cdist b/bin/cdist index 5333cb3a..21a35c22 100755 --- a/bin/cdist +++ b/bin/cdist @@ -638,16 +638,17 @@ if __name__ == "__main__": # Main subcommand parser parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) - parser['main'].add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + VERSION) - parser['sub'] = parser['main'].add_subparsers() + parser['sub'] = parser['main'].add_subparsers(title="Commands") + # Banner parser['banner'] = parser['sub'].add_parser('banner', add_help=False) parser['banner'].set_defaults(func=banner) + # Config parser['config'] = parser['sub'].add_parser('config', parents=[parser['most']]) parser['config'].add_argument('host', nargs='+', diff --git a/doc/man/man1/cdist.text b/doc/man/man1/cdist.text index f45d4495..7f585953 100644 --- a/doc/man/man1/cdist.text +++ b/doc/man/man1/cdist.text @@ -10,22 +10,34 @@ cdist - Configuration management SYNOPSIS -------- -cdist [-h] [-b] [-c CDIST_HOME] [-d] [-i MANIFEST] [-p] [-s] [-V] [host [host ...]] +cdist [-V|--version] +cdist banner +cdist config [-h] [-d] [-V] [-c CDIST_HOME] [-i MANIFEST] [-p] [-s] host [host ...] + DESCRIPTION ----------- cdist is the frontend executable to the cdist configuration management. +cdist supports different as explained below. The options to the main +program are: + +-V, --version:: + Show version and exit -OPTIONS +BANNER ------- +Displays the cdist banner. + + +CONFIG +------ +Configure a system + -h, --help:: Show the help screen --b, --banner:: - Show cdist banner - -c CDIST_HOME, --cdist-home CDIST_HOME:: Instead of using the parent of the bin directory as cdist home, use the specified directory @@ -42,23 +54,25 @@ OPTIONS -s, --sequential:: Operate on multiple hosts sequentially --V, --version:: - Show version and exit - EXAMPLES -------- -------------------------------------------------------------------------------- # Configure ikq05.ethz.ch with debug enabled -cdist -d ikq05.ethz.ch +cdist config -d ikq05.ethz.ch # Configure hosts in parallel and use a different home directory -cdist -c ~/p/cdist-nutzung -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch +cdist config -c ~/p/cdist-nutzung \ + -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch -# Does not need a comment, plain works -cdist --banner +# Display banner +cdist banner + +# Show help cdist --help + +# Show Version cdist --version -------------------------------------------------------------------------------- From c8b238df064df105f032237d52738c4a8936ceb5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 22:07:21 +0200 Subject: [PATCH 155/166] update cdist homepage Signed-off-by: Nico Schottelius --- README | 11 +++++++---- doc/dev/todo/niconext | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README b/README index 295b9152..521d70a2 100644 --- a/README +++ b/README @@ -38,13 +38,13 @@ Design | Define target state, do not focus on methods or scripts Design | Push architecture: Instantly apply your changes Small core | cdist's core is very small - less code, less bugs Fast development | Focus on straightforwardness of type creation is a main development objective +Modern Programming Language | cdist is written in Python Requirements, Scalability | No central server needed, cdist operates in push mode and can be run from any computer Requirements, Scalability, Upgrade | cdist only needs to be updated on the master, not on the target hosts Requirements, Security | Uses well-know [SSH](http://www.openssh.com/) as transport protocol Requirements, Simplicity | Requires only shell and SSH server on the target UNIX | Reuse of existing tools like cat, find, mv, ... UNIX, familar environment, documentation | Is available as manpages and HTML -UNIX, simplicity, familar environment | cdist is written in POSIX shell UNIX, simplicity, familar environment | cdist is configured in POSIX shell """]] @@ -76,6 +76,7 @@ cdist was tested or is know to run on at least ### Server * A posix like shell + * Python (>= 3.2 required) * SSH-Client ### Client ("target host") @@ -110,10 +111,11 @@ how to use cdist. There are at least the following branches available: * Development: master - * Current Stable: 1.7 (Bugfixes, cleanups, new type and explorer rename) + * 2.0: Python rewrite Old versions: + * 1.7: Bugfixes, cleanups, new type and explorer rename * 1.6: New types, cleaned up \_\_package* types, internal cleanup * 1.5: Focus on object orientation instead of global stage orientation * 1.4: Support for redefiniton of objects (if equal) @@ -129,13 +131,12 @@ may vanish at any point. To select a specific branch use git checkout -b origin/ # Stay on a specific version - version=1.7 + version=2.0 git checkout -b $version origin/$version ### Mirrors * git://github.com/telmich/cdist.git ([github](https://github.com/telmich/cdist)) - * git://git.sans.ethz.ch/cdist ([sans](http://git.sans.ethz.ch/?p=cdist;a=summary)) ## Update @@ -151,6 +152,8 @@ If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break. The master branch on the other hand is the development branch and may not be working, break your setup or eat the tree in your garden. +### Upgrading from 1.7 to 2.0 + ### Upgrading from 1.6 to 1.7 * If you used the global explorer **hardware_type**, you need to change diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 63f9b629..accef050 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -13,3 +13,5 @@ - Replace bin/cdist-type-template - Allow manifest to be read from stdin +- Create new video for cdist 2.0.0 + http://www.youtube.com/watch?v=PRMjzy48eTI From 2f30ebf1f0c7da3284071e17ab0ee123036bd882 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 22:11:51 +0200 Subject: [PATCH 156/166] more todo, yeah! Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index accef050..beae3cb2 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -9,6 +9,7 @@ - Remove legacy code in cdist - Remove cdist-config - Remove man1/cdist-type-emulator.text + - Remove the PATH=... part from the README - Replace bin/cdist-type-template From 24cbc1b717442000e1831281b1c0e932b2981b46 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 22:23:28 +0200 Subject: [PATCH 157/166] changes for 2.0 Signed-off-by: Nico Schottelius --- README | 5 +++++ doc/man/cdist-reference.text.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README b/README index 521d70a2..3808faac 100644 --- a/README +++ b/README @@ -154,6 +154,11 @@ working, break your setup or eat the tree in your garden. ### Upgrading from 1.7 to 2.0 +* Ensure python (>= 3.2) is installed on the server +* Use "cdist config host" instead of "cdist-deploy-to host" +* Use "cdist config -p host1 host2" instead of "cdist-mass-deploy" +* Use **\_\_object_fq** instead of **\_\_self** in manifests + ### Upgrading from 1.6 to 1.7 * If you used the global explorer **hardware_type**, you need to change diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index a823891b..218b0f57 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -171,7 +171,7 @@ __object:: __object_id:: The type unique object id. Available for: type manifest, type explorer, type gencode -__self:: +__object_fq:: The full qualified name of the current object. Available for: type manifest, type explorer, type gencode __target_host:: From 159ace1ea2bee34919c6806e7dc1aeaf73713df4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 22:40:10 +0200 Subject: [PATCH 158/166] more important changes for 2.0 Signed-off-by: Nico Schottelius --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index 3808faac..3e00b497 100644 --- a/README +++ b/README @@ -157,6 +157,7 @@ working, break your setup or eat the tree in your garden. * Ensure python (>= 3.2) is installed on the server * Use "cdist config host" instead of "cdist-deploy-to host" * Use "cdist config -p host1 host2" instead of "cdist-mass-deploy" +* Use "cdist banner" for fun * Use **\_\_object_fq** instead of **\_\_self** in manifests ### Upgrading from 1.6 to 1.7 From fece6ddd6aaee0a0fdc82fee6fa7f195537b4f9d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 22:42:47 +0200 Subject: [PATCH 159/166] create subdirs man1 and man7 on the web Signed-off-by: Nico Schottelius --- build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index a6897d49..8ebfa168 100755 --- a/build.sh +++ b/build.sh @@ -96,10 +96,14 @@ case "$1" in web) cp README ${WEBDIR}/${WEBPAGE} - rm -rf ${WEBDIR}/${WEBBASE}/man && mkdir ${WEBDIR}/${WEBBASE}/man + rm -rf ${WEBDIR}/${WEBBASE}/man + mkdir ${WEBDIR}/${WEBBASE}/man/man1 + mkdir ${WEBDIR}/${WEBBASE}/man/man7 + rm -rf ${WEBDIR}/${WEBBASE}/speeches && mkdir ${WEBDIR}/${WEBBASE}/speeches - cp ${MAN1DSTDIR}/*.html ${MAN7DSTDIR}/*.html ${WEBDIR}/${WEBBASE}/man + cp ${MAN1DSTDIR}/*.html ${WEBDIR}/${WEBBASE}/man/man1 + cp ${MAN7DSTDIR}/*.html ${WEBDIR}/${WEBBASE}/man/man7 cp ${SPEECHESDIR}/*.pdf ${WEBDIR}/${WEBBASE}/speeches git describe > ${WEBDIR}/${WEBBASE}/man/VERSION From a0058a181827d1122c76fef71efa1449950b49d0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 22:46:15 +0200 Subject: [PATCH 160/166] really create the directories Signed-off-by: Nico Schottelius --- build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 8ebfa168..bef4393b 100755 --- a/build.sh +++ b/build.sh @@ -97,8 +97,7 @@ case "$1" in web) cp README ${WEBDIR}/${WEBPAGE} rm -rf ${WEBDIR}/${WEBBASE}/man - mkdir ${WEBDIR}/${WEBBASE}/man/man1 - mkdir ${WEBDIR}/${WEBBASE}/man/man7 + mkdir -p ${WEBDIR}/${WEBBASE}/man/man1 ${WEBDIR}/${WEBBASE}/man/man7 rm -rf ${WEBDIR}/${WEBBASE}/speeches && mkdir ${WEBDIR}/${WEBBASE}/speeches From 2e96ba793a1fe85cd10b7ae4a6ac694c47e1e4d7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 22:48:14 +0200 Subject: [PATCH 161/166] reformat cdist manpage Signed-off-by: Nico Schottelius --- doc/man/man1/cdist.text | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/man/man1/cdist.text b/doc/man/man1/cdist.text index 7f585953..104163af 100644 --- a/doc/man/man1/cdist.text +++ b/doc/man/man1/cdist.text @@ -10,8 +10,10 @@ cdist - Configuration management SYNOPSIS -------- -cdist [-V|--version] +cdist [-h] [-V] + cdist banner + cdist config [-h] [-d] [-V] [-c CDIST_HOME] [-i MANIFEST] [-p] [-s] host [host ...] @@ -22,6 +24,9 @@ cdist is the frontend executable to the cdist configuration management. cdist supports different as explained below. The options to the main program are: +-h, --help:: + Show the help screen + -V, --version:: Show version and exit From 41099a35960f0f97c4b8f5e7bbb7da8b8f77b20d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Sep 2011 22:49:42 +0200 Subject: [PATCH 162/166] update references Signed-off-by: Nico Schottelius --- doc/man/man1/cdist.text | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/man/man1/cdist.text b/doc/man/man1/cdist.text index 104163af..5a45b8c1 100644 --- a/doc/man/man1/cdist.text +++ b/doc/man/man1/cdist.text @@ -85,7 +85,8 @@ cdist --version SEE ALSO -------- - cdist(7) -- cdist-type-build-emulation(1) +- cdist-type-emulator(1) +- cdist-reference(7) COPYING From 9b3a95e244c4624cdd1c1268a47cf73a842f5a98 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 14 Sep 2011 01:49:00 +0200 Subject: [PATCH 163/166] ++logs +beginning of benchmark Signed-off-by: Nico Schottelius --- doc/dev/benchmark-parallel-deploy | 12 ++++++ ...11-09-08 => 2011-09-08.obsolete_debugging} | 0 doc/dev/logs/2011-09-12 | 43 +++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100755 doc/dev/benchmark-parallel-deploy rename doc/dev/logs/{2011-09-08 => 2011-09-08.obsolete_debugging} (100%) diff --git a/doc/dev/benchmark-parallel-deploy b/doc/dev/benchmark-parallel-deploy new file mode 100755 index 00000000..82d2240c --- /dev/null +++ b/doc/dev/benchmark-parallel-deploy @@ -0,0 +1,12 @@ +#!/bin/sh + +outfile="$1"; shift + +( + for host in "$@"; do + hosts="$hosts $host" + cdist config -c ~/p/cdist-nutzung -p $hosts + done +) | tee "$outfile" +echo "----------" +grep 'INFO: Total processing time for' "$outfile" | sed 's/.*: //' diff --git a/doc/dev/logs/2011-09-08 b/doc/dev/logs/2011-09-08.obsolete_debugging similarity index 100% rename from doc/dev/logs/2011-09-08 rename to doc/dev/logs/2011-09-08.obsolete_debugging diff --git a/doc/dev/logs/2011-09-12 b/doc/dev/logs/2011-09-12 index 391b836f..1ace466f 100644 --- a/doc/dev/logs/2011-09-12 +++ b/doc/dev/logs/2011-09-12 @@ -65,3 +65,46 @@ Use: # Scale linearly with the single host value plot((1:31)*times(1)) +-------------------------------------------------------------------------------- +code: + +octave +time = [ +72.166661 +76.633228 +77.199817 +94.045175 +103.226354 +107.76097 +101.571705 +107.600093 +116.500371 +119.445805 +123.944385 +130.499098 +137.250861 +154.9841 +139.659637 +142.70005 +148.541452 +159.360809 +171.907864 +178.76695 +183.856671 +194.504221 +207.314842 +215.846502 +217.223581 +238.591705 +238.478493 +246.058718 +264.208372 +265.560685 +282.264488 +] +plot(times, "-;cdist;", times(1)*[1:length(times)]', "-;linear;") +title("Configuration duration (cdist-2.0.0-rc4)") +ylabel("Number of hosts") +xlabel("Time in seconds") +print('cdist-2.0.0-rc4.png', '-dpng') + From 03e312dcb7de0da7a24a185150fd0870bc07e054 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 14 Sep 2011 01:50:18 +0200 Subject: [PATCH 164/166] use shutil.move() instead of os.rename() to avoid OSError: [Errno 18] Invalid cross-device link Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 21a35c22..92161908 100755 --- a/bin/cdist +++ b/bin/cdist @@ -157,7 +157,7 @@ class Cdist: # Remove previous cache if os.path.exists(self.cache_dir): shutil.rmtree(self.cache_dir) - os.rename(self.temp_dir, self.cache_dir) + shutil.move(self.temp_dir, self.cache_dir) def exit_error(self, *args): log.error(*args) From b76c4a5105d21cc76caa9d4ecf3980e1afe6bfc5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 14 Sep 2011 08:45:55 +0200 Subject: [PATCH 165/166] make shell exec error and normal error look the same for easier grepping in logfiles Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- doc/dev/benchmark-parallel-deploy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 92161908..f7ad2f72 100755 --- a/bin/cdist +++ b/bin/cdist @@ -197,7 +197,7 @@ class Cdist: print(script_fd.read()) script_fd.close() - self.exit_error("Non-Zero exit code exit of " + " ".join(*args)) + self.exit_error("Command failed (shell): " + " ".join(*args)) def run_or_fail(self, *args, **kargs): if "remote" in kargs: diff --git a/doc/dev/benchmark-parallel-deploy b/doc/dev/benchmark-parallel-deploy index 82d2240c..0e7c4a2f 100755 --- a/doc/dev/benchmark-parallel-deploy +++ b/doc/dev/benchmark-parallel-deploy @@ -5,7 +5,7 @@ outfile="$1"; shift ( for host in "$@"; do hosts="$hosts $host" - cdist config -c ~/p/cdist-nutzung -p $hosts + cdist config -c ~/p/cdist-nutzung -p $hosts 2>&1 done ) | tee "$outfile" echo "----------" From 1f9ae130779903302d3bf56929c25a515f75b279 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Sep 2011 12:09:22 +0200 Subject: [PATCH 166/166] only the core is python, not the manifests Signed-off-by: Nico Schottelius --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 3e00b497..5e60a093 100644 --- a/README +++ b/README @@ -111,7 +111,7 @@ how to use cdist. There are at least the following branches available: * Development: master - * 2.0: Python rewrite + * 2.0: Python rewrite of cdist core Old versions: