From 7882b4a3ac11b0866819f92b50e9231d11166aed Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 16:03:53 +0200 Subject: [PATCH 01/20] only print env, if existent Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index f95ba941..b0f98b56 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -38,7 +38,10 @@ def shell_run_or_debug_fail(script, *args, **kargs): del kargs["remote_prefix"] log.debug("Shell exec cmd: %s", args) - log.debug("Shell exec env: %s", kargs['env']) + + if 'env' in kargs: + log.debug("Shell exec env: %s", kargs['env']) + try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: From 24b8a57df51160463c501f92523303dcdf050269 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 16:04:27 +0200 Subject: [PATCH 02/20] write first tests for the exec module Signed-off-by: Nico Schottelius --- test.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/test.py b/test.py index ad56df11..b5b80dc9 100755 --- a/test.py +++ b/test.py @@ -23,6 +23,8 @@ import os import sys +import shutil +import tempfile import unittest sys.path.insert(0, os.path.abspath( @@ -33,6 +35,38 @@ import cdist.config import cdist.exec class Exec(unittest.TestCase): + def setUp(self): + """Create shell code and co.""" + + self.temp_dir = tempfile.mkdtemp() + self.shell_false = os.path.join(self.temp_dir, "shell_false") + self.shell_true = os.path.join(self.temp_dir, "shell_true") + + true_fd = open(self.shell_false, "w") + true_fd.writelines(["!/bin/sh", "/bin/true"]) + true_fd.close() + + false_fd = open(self.shell_false, "w") + false_fd.writelines(["!/bin/sh", "/bin/false"]) + false_fd.close() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_local_success_shell(self): + try: + cdist.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail_shell(self): + self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail, + self.shell_false, [self.shell_false]) + def test_local_success(self): try: cdist.exec.run_or_fail(["/bin/true"]) @@ -44,15 +78,7 @@ class Exec(unittest.TestCase): self.assertFalse(failed) def test_local_fail(self): - try: - cdist.exec.run_or_fail(["/bin/false"]) - except cdist.Error: - failed = True - else: - failed = False - - self.assertTrue(failed) - + self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) if __name__ == '__main__': From 6e73572a9514583c32f6643ed4d21d09ab8cde74 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 16:21:56 +0200 Subject: [PATCH 03/20] easier check for remote_prefix Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index b0f98b56..7985bdb3 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -26,16 +26,13 @@ log = logging.getLogger(__name__) import cdist -def shell_run_or_debug_fail(script, *args, **kargs): +def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] - remote = False - if "remote_prefix" in kargs: - remote = True - args[0][:0] = kargs["remote_prefix"] - del kargs["remote_prefix"] + if remote_prefix: + args[0][:0] = remote_prefix log.debug("Shell exec cmd: %s", args) @@ -46,9 +43,9 @@ def shell_run_or_debug_fail(script, *args, **kargs): subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: log.error("Code that raised the error:\n") - if remote: - # FIXME: included in Path! - remote_cat(script) + if remote_prefix: + run_or_fail(["cat", script], remote_prefix=remote_prefix) + else: try: script_fd = open(script) @@ -62,10 +59,9 @@ def shell_run_or_debug_fail(script, *args, **kargs): raise cdist.Error(" ".join(*args) + ": " + error.args[1]) -def run_or_fail(*args, **kargs): - if "remote_prefix" in kargs: - args[0][:0] = kargs["remote_prefix"] - del kargs["remote_prefix"] +def run_or_fail(*args, remote_prefix=False, **kargs): + if remote_prefix: + args[0][:0] = remote_prefix log.debug("Exec: " + " ".join(*args)) try: From c687dbdc70951e15d0ba8e7adbd486e8c147a493 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 16:22:09 +0200 Subject: [PATCH 04/20] remove remote_cat, as it's only used in exec module Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 0fa753a8..277b1401 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -125,10 +125,6 @@ class Path: """Create directory on remote side""" cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix) - def remote_cat(filename): - """Use cat on the remote side for output""" - cdist.exec.run_or_fail(["cat", filename], remote_prefix=self.remote_prefix) - def remove_remote_dir(self, destination): cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix) From 5e862c9ede969ec026dbf19e2a3ea158182a8ede Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 17:33:26 +0200 Subject: [PATCH 05/20] make cdist-tutorial compile Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-tutorial.text | 55 ++++++++++++-------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index e146f1a8..dfda5325 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -1,40 +1,23 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Give the user an introduction into cdist -# +cdist-tutorial(7) +================= +Nico Schottelius -. cdist-config -set -eu -banner="cdist-quickstart>" -continue="Press enter to continue or ctrl-c to abort." -create_continue="Press enter to create the described files/directories" +NAME +---- +cdist-tutorial - a guided introduction into cdist -__prompt() -{ - echo -n "$banner" "$@" - read answer -} -################################################################################ +INTRODUCTION +------------ +This tutorial is aimed at people learning cdist and shows +typical approaches as well as gives an easy start into +the world of configuration management. + + +NOT MIGRATED +------------ + # Intro of quickstart # cat << eof @@ -305,6 +288,10 @@ That's it, this is the end of the cdist-quickstart. I hope you've got some impression on how cdist works, here are again some pointers on where to continue to read: -cdist(7), cdist-deploy-to(1), cdist-type(7), cdist-stages(7) eof + +SEE ALSO +-------- +cdist(1), cdist-type(7), cdist-stages(7) + From d5df740f17c2207fdf0e92181c2f74b6671fbe1b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 20:14:04 +0200 Subject: [PATCH 06/20] always compile documentation with UTF-8 encoding Signed-off-by: Nico Schottelius --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index bef4393b..68abafde 100755 --- a/build.sh +++ b/build.sh @@ -27,7 +27,7 @@ #set -e # Manpage and HTML -A2XM="a2x -f manpage --no-xmllint" +A2XM="a2x -f manpage --no-xmllint -a encoding=UTF-8" A2XH="a2x -f xhtml --no-xmllint" # Developer webbase From df84480fcc5efb0763f34b420b9568396a0e375a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:02:04 +0200 Subject: [PATCH 07/20] BUGFIX: emulator compared parameter with value Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index ecdfff25..a97959e6 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -105,11 +105,11 @@ def emulator(argv): sys.exit(1) else: param_fd = open(file, "r") - param_old = param_fd.readlines() + value_old = param_fd.readlines() param_fd.close() - if(param_old != param): - print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + if(value_old != value): + print("Parameter " + param + " differs: " + " ".join(value_old) + " vs. " + value) print("Sources: " + " ".join(old_object_source) + " and " + object_source) sys.exit(1) else: From 1cfb6bf5a8f6707b30e9239f69d94c06df446290 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:11:59 +0200 Subject: [PATCH 08/20] make manpage compile for __partition_msdos_apply Signed-off-by: Nico Schottelius --- conf/type/__partition_msdos_apply/man.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__partition_msdos_apply/man.text b/conf/type/__partition_msdos_apply/man.text index 4d4f127c..6cc53b77 100644 --- a/conf/type/__partition_msdos_apply/man.text +++ b/conf/type/__partition_msdos_apply/man.text @@ -5,7 +5,7 @@ Steven Armstrong NAME ---- -cdist-type__partition_msdos_apply +cdist-type__partition_msdos_apply - Apply dos partition settings DESCRIPTION From 87937ecec92232fc8e4aa402c35b9cd0fb988121 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:12:11 +0200 Subject: [PATCH 09/20] also always use utf-8 for html Signed-off-by: Nico Schottelius --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 68abafde..d20e0211 100755 --- a/build.sh +++ b/build.sh @@ -28,7 +28,7 @@ # Manpage and HTML A2XM="a2x -f manpage --no-xmllint -a encoding=UTF-8" -A2XH="a2x -f xhtml --no-xmllint" +A2XH="a2x -f xhtml --no-xmllint -a encoding=UTF-8" # Developer webbase WEBDIR=$HOME/niconetz From 877d8fed309fac3df8691f4533f0ee5385b5a406 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:34:41 +0200 Subject: [PATCH 10/20] verify that corrupted manifests are detected as such Signed-off-by: Nico Schottelius --- test.py | 49 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/test.py b/test.py index b5b80dc9..d1d62d25 100755 --- a/test.py +++ b/test.py @@ -42,12 +42,12 @@ class Exec(unittest.TestCase): self.shell_false = os.path.join(self.temp_dir, "shell_false") self.shell_true = os.path.join(self.temp_dir, "shell_true") - true_fd = open(self.shell_false, "w") - true_fd.writelines(["!/bin/sh", "/bin/true"]) + true_fd = open(self.shell_true, "w") + true_fd.writelines(["#!/bin/sh", "/bin/true"]) true_fd.close() false_fd = open(self.shell_false, "w") - false_fd.writelines(["!/bin/sh", "/bin/false"]) + false_fd.writelines(["#!/bin/sh", "/bin/false"]) false_fd.close() def tearDown(self): @@ -55,7 +55,7 @@ class Exec(unittest.TestCase): def test_local_success_shell(self): try: - cdist.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) + cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) except cdist.Error: failed = True else: @@ -80,6 +80,47 @@ class Exec(unittest.TestCase): def test_local_fail(self): self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) +class Config(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.init_manifest = os.path.join(self.temp_dir, "manifest") + self.config = cdist.config.Config("localhost", + initial_manifest=self.init_manifest) + + def test_initial_manifest_different_parameter(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh", + "__file " + self.temp_dir + "--mode 0700", + "__file " + self.temp_dir + "--mode 0600", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + + def test_initial_manifest_parameter_added(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh", + "__file " + self.temp_dir, + "__file " + self.temp_dir + "--mode 0600", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + + def test_initial_manifest_parameter_removed(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh", + "__file " + self.temp_dir + "--mode 0600", + "__file " + self.temp_dir, + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + +# Todo: +# fail if parameter in manifest given are different +# fail if parameter in manifest given are absent once/given once +# succeed if same parameter is specified twice if __name__ == '__main__': unittest.main() From 603f1c3ae075c7d5e47f1bfd472e89cec884cda4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:38:08 +0200 Subject: [PATCH 11/20] also check that giving a paramter twice works Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-tutorial.text | 65 +++++++++++++++++++++++++++++++- test.py | 20 ++++++++-- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index dfda5325..8abcbad4 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -15,8 +15,69 @@ typical approaches as well as gives an easy start into the world of configuration management. -NOT MIGRATED ------------- + +QUICK START +----------- +For those who just want to configure a system with the +cdist configuration management and do not need (or want) +to understand everything. + +Cdist uses **ssh** for communication and transportation +and usually logs into the **target host** as the +**root** user. So you need to configure the **ssh server** +of the target host to allow root logins: Edit +the file **/etc/ssh/sshd_config** and add one of the following +lines: + +-------------------------------------------------------------------------------- +# Allow login only via public key +PermitRootLogin without-password + +# Allow login via password and public key +PermitRootLogin yes +-------------------------------------------------------------------------------- + + +Before you can start using cdist, you need to ensure that +you can login +sshd config! + + + + + +You can copy and paste the following +code into your shell to get started and even configure your system. + +-------------------------------------------------------------------------------- +# Get cdist +git clone git://git.schottelius.org/cdist + +# Create manifest (maps configuration to host(s) +cd cdist +echo '__file /etc/cdist-configured' > conf/manifest/init +chmod 0700 conf/manifest/init + +echo 'Ensure that you can login as root to localhost without password' +echo '(i.e. via public key) and then press return' +read tmp + +# Configure localhost +./bin/cdist config localhost + +# Find out that cdist created /etc/cdist-configured +ls -l /etc/cdist-configured +-------------------------------------------------------------------------------- + +The file 'conf/manifest/init' is usually the entry point for cdist, +to find out what to configure on which host. All manifests are +essentially shell scripts. Every manifest can use the types known to +cdist, which are usually underline prefixed (\_\_). + + + +Everything you specify in manifests + # Intro of quickstart # diff --git a/test.py b/test.py index d1d62d25..2eb0b89f 100755 --- a/test.py +++ b/test.py @@ -117,10 +117,22 @@ class Config(unittest.TestCase): self.assertRaises(cdist.Error, self.config.run_initial_manifest()) -# Todo: -# fail if parameter in manifest given are different -# fail if parameter in manifest given are absent once/given once -# succeed if same parameter is specified twice + def test_initial_manifest_parameter_twice(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh", + "__file " + self.temp_dir + "--mode 0600", + "__file " + self.temp_dir + "--mode 0600", + ]) + manifest_fd.close() + + try: + self.config.run_initial_manifest() + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) if __name__ == '__main__': unittest.main() From 326ff6728365cd94b8e29d3dbbb49d0d704ebbe2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:48:29 +0200 Subject: [PATCH 12/20] break tests by fixing them Signed-off-by: Nico Schottelius --- test.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test.py b/test.py index 2eb0b89f..a3652c41 100755 --- a/test.py +++ b/test.py @@ -89,9 +89,9 @@ class Config(unittest.TestCase): def test_initial_manifest_different_parameter(self): manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh", - "__file " + self.temp_dir + "--mode 0700", - "__file " + self.temp_dir + "--mode 0600", + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + "--mode 0700\n", + "__file " + self.temp_dir + "--mode 0600\n", ]) manifest_fd.close() @@ -99,9 +99,9 @@ class Config(unittest.TestCase): def test_initial_manifest_parameter_added(self): manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh", - "__file " + self.temp_dir, - "__file " + self.temp_dir + "--mode 0600", + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + '\n', + "__file " + self.temp_dir + "--mode 0600\n", ]) manifest_fd.close() @@ -109,9 +109,9 @@ class Config(unittest.TestCase): def test_initial_manifest_parameter_removed(self): manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh", - "__file " + self.temp_dir + "--mode 0600", - "__file " + self.temp_dir, + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + "--mode 0600\n", + "__file " + self.temp_dir + "\n", ]) manifest_fd.close() @@ -119,9 +119,9 @@ class Config(unittest.TestCase): def test_initial_manifest_parameter_twice(self): manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh", - "__file " + self.temp_dir + "--mode 0600", - "__file " + self.temp_dir + "--mode 0600", + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + "--mode 0600\n", + "__file " + self.temp_dir + "--mode 0600\n", ]) manifest_fd.close() From 2c0a7adf74578ce6151468afcf724ddddbcf46fc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:49:12 +0200 Subject: [PATCH 13/20] replace CdistError with cdist.Error Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 3 ++- lib/cdist/exec.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index a97959e6..bd97f69b 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -24,6 +24,7 @@ import logging import os import sys +import cdist import cdist.path log = logging.getLogger(__name__) @@ -87,7 +88,7 @@ def emulator(argv): try: os.makedirs(param_out_dir, exist_ok=True) except OSError as error: - raise CdistError(param_out_dir + ": " + error.args[1]) + raise cdist.Error(param_out_dir + ": " + error.args[1]) # Record parameter params = vars(args) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 7985bdb3..9cedefcc 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -58,7 +58,6 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): except OSError as error: raise cdist.Error(" ".join(*args) + ": " + error.args[1]) - def run_or_fail(*args, remote_prefix=False, **kargs): if remote_prefix: args[0][:0] = remote_prefix From 1dcc3b771e42c7593f65890aa41fd723957693a7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:58:16 +0200 Subject: [PATCH 14/20] +some ssh hints, localhost hint Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-tutorial.text | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index 8abcbad4..9b6e7492 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -14,6 +14,10 @@ This tutorial is aimed at people learning cdist and shows typical approaches as well as gives an easy start into the world of configuration management. +This tutorial assumes you are configuring **localhost**, because +it is always available. Just repace **localhost** with your target +host for real life usage. + QUICK START @@ -37,6 +41,19 @@ PermitRootLogin without-password PermitRootLogin yes -------------------------------------------------------------------------------- +As cdist uses ssh intensively, it is recommended to setup authentication +with public keys: + +-------------------------------------------------------------------------------- +# Generate pubkey pair as a normal user +ssh-keygen + +# Copy pubkey over to target host +ssh-copy-id root@target_host +-------------------------------------------------------------------------------- + +As soon as you are able to login to the target host + Before you can start using cdist, you need to ensure that you can login @@ -75,6 +92,9 @@ essentially shell scripts. Every manifest can use the types known to cdist, which are usually underline prefixed (\_\_). +SSH HINTS +--------- +Control master, ssh agent Everything you specify in manifests From d87deba30eccd70ce37b9b4844e2a25e873d9513 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:01:12 +0200 Subject: [PATCH 15/20] cleanup tutorial, fillup todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 280 ++++++++++++++++++++++++++++ doc/man/man7/cdist-tutorial.text | 301 +------------------------------ 2 files changed, 283 insertions(+), 298 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 4e07dd96..91b418a2 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -8,6 +8,286 @@ - and that ssh will wait for answer of prompt - nasty if used in parallel mode (scroll up!) + +SSH HINTS +--------- +Control master, ssh agent + +Everything you specify in manifests + + +# Intro of quickstart +# +cat << eof +$banner cdist version $__cdist_version + +Welcome to the interactive guide to cdist! +This is the interactive tutorial and beginners help for cdist and here's +our schedule: + + - Stages: How cdist operates + - Explorer: Explore facts of the target host + - Manifest: Map configurations to hosts + - Types: Bundled functionality + - Deploy a configuration to the local host! + +eof +__prompt "$continue" + +################################################################################ +# Stages +# +cat << eof + +To deploy configurations to a host, you call + + cdist-deploy-to + +which makes calls to other scripts, which realise the so called "stages". +Usually you'll not notice this, but in case you want to debug or hack cdist, +you can run each stage on its own. Besides that, you just need to remember +that the command cdist-deploy-to is the main cdist command. + +See also: + + Source of cdist-deploy-to(1), cdist-stages(7) + +eof +__prompt "$continue" + +################################################################################ +# Explorer +# +cat << eof + +The first thing cdist always does is running different explorers on the +target host. The explorers can be found in the directory + + ${__cdist_explorer_dir} + +An explorer is executed on the target host and its output is saved to a file. +You can use these files later to decide what or how to configure the host. + +For a demonstration, we'll call the OS explorer locally now, but remember: +This is only for demonstration, normally it is run on the target host. +The os explorer will which either displays the detected operating system or +nothing if it does not know your OS. + +See also: + + cdist-explorer(7) + +eof +explorer="${__cdist_explorer_dir}/os" + +__prompt "Press enter to execute $explorer" + +set -x +"$explorer" +set +x + +################################################################################ +# Manifest +# +cat << eof + +The initial manifest is the entry point for cdist to find out, what you would +like to have configured. It is located at + + ${__cdist_manifest_init} + +And can be as simple as + +-------------------------------------------------------------------------------- +__file /etc/cdist-configured --type file +-------------------------------------------------------------------------------- + +See also: + + cdist-manifest(7) + +eof +__prompt "$continue" + +cat << eof + +Let's take a deeper look at the initial manifest to understand what it means: + + __file /etc/cdist-configured --type file + | | | \\ + | | The parameter type \\ With the value file + | | + | | + | | This is the object id + | + __file is a so called "type" + + +This essentially looks like a standard command executed in the shell. +eof +__prompt "$continue" + +cat << eof + +And that's exactly true. Manifests are shell snippets that can use +types as commands with arguments. cdist prepends a special path +that contain links to the cdist-type-emulator, to \$PATH, so you +can use your types as a command. + +This is also the reason why types should always be prefixed with +"__", to prevent collisions with existing binaries. + +The object id is unique per type and used to prevent you from creating +the same object twice. + +Parameters are type specific and are always specified as --parameter . + +See also: + + cdist-type-build-emulation(1), cdist-type-emulator(1) + +eof +__prompt "$continue" + +################################################################################ +# Types +# +cat << eof + +Types are bundled functionality and are the main component of cdist. +If you want to have a feature x, you write the type __x. Types are stored in + + ${__cdist_type_dir} + +And cdist ships with some types already! + +See also: + + cdist-type(7) + +eof +__prompt "Press enter to see available types" + +set -x +ls ${__cdist_type_dir} +set +x + +cat << eof + +Types consist of the following parts: + + - ${__cdist_name_parameter} (${__cdist_name_parameter_required}/${__cdist_name_parameter_optional} + - ${__cdist_name_manifest} + - ${__cdist_name_explorer} + - ${__cdist_name_gencode} + +eof +__prompt "$continue" + + +cat << eof + +Every type must have a directory named ${__cdist_name_parameter}, which +contains required or optional parameters (in newline seperated files). + +If an object of a specific type was created in the initial manifest, +the manifest of the type is run and may create other objects. + +A type may have ${__cdist_name_explorer}, which are very similar to the +${__cdist_name_explorer} seen above, but with a different purpose: +They are specific to the type and are not relevant for other types. + +You may use them for instance to find out details on the target host, +so you can decide what to do on the target host eventually. + +After the ${__cdist_name_manifest} and the ${__cdist_name_explorer} of +a type have been run, ${__cdist_name_gencode} is executed, which creates +code to be executed on the target on stdout. + +eof +__prompt "$continue" + +################################################################################ +# Deployment +# + +cat << eof + +Now you've got some basic knowledge about cdist, let's configure your a host! + +Ensure that you have a ssh server running on the host and that you can login as root. + +eof + +__prompt "Enter hostname or press enter for localhost: " + +if [ "$answer" ]; then + host="$answer" +else + host="localhost" +fi + +manifestinit="conf/manifest/init" +cat << eof + +I'll now setup $manifestinit, containing the following code: + +-------------------------------------------------------------------------------- +# Every machine becomes a marker, so sysadmins know that automatic +# configurations are happening +__file /etc/cdist-configured + +case "\$__target_host" in + $host) + __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic + __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" + ;; +esac +-------------------------------------------------------------------------------- + +WARNING: This will overwrite ${manifestinit}. + +eof + +cat > "$__cdist_abs_mydir/../$manifestinit" << eof + +# Every machine becomes a marker, so sysadmins know that automatic +# configurations are happening +__file /etc/cdist-configured + +case "\$__target_host" in + $host) + __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic + __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" + ;; +esac + +eof + +chmod u+x "$__cdist_abs_mydir/../$manifestinit" + +cmd="cdist-deploy-to $host" + +__prompt "Press enter to run \"$cmd\"" + +# No quotes, we need field splitting +$cmd + +################################################################################ +# End +# + +cat << eof + + +-------------------------------------------------------------------------------- +That's it, this is the end of the cdist-quickstart. + +I hope you've got some impression on how cdist works, here are again some +pointers on where to continue to read: + + +eof -------------------------------------------------------------------------------- - Initial install support diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index 9b6e7492..15f2a81d 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -52,19 +52,9 @@ ssh-keygen ssh-copy-id root@target_host -------------------------------------------------------------------------------- -As soon as you are able to login to the target host - - -Before you can start using cdist, you need to ensure that -you can login -sshd config! - - - - - -You can copy and paste the following -code into your shell to get started and even configure your system. +As soon as you are able to login without passwort to the target host, +we can use cdist, to configure it. You can copy and paste the following +code into your shell to get started and configure localhost: -------------------------------------------------------------------------------- # Get cdist @@ -75,10 +65,6 @@ cd cdist echo '__file /etc/cdist-configured' > conf/manifest/init chmod 0700 conf/manifest/init -echo 'Ensure that you can login as root to localhost without password' -echo '(i.e. via public key) and then press return' -read tmp - # Configure localhost ./bin/cdist config localhost @@ -92,287 +78,6 @@ essentially shell scripts. Every manifest can use the types known to cdist, which are usually underline prefixed (\_\_). -SSH HINTS ---------- -Control master, ssh agent - -Everything you specify in manifests - - -# Intro of quickstart -# -cat << eof -$banner cdist version $__cdist_version - -Welcome to the interactive guide to cdist! -This is the interactive tutorial and beginners help for cdist and here's -our schedule: - - - Stages: How cdist operates - - Explorer: Explore facts of the target host - - Manifest: Map configurations to hosts - - Types: Bundled functionality - - Deploy a configuration to the local host! - -eof -__prompt "$continue" - -################################################################################ -# Stages -# -cat << eof - -To deploy configurations to a host, you call - - cdist-deploy-to - -which makes calls to other scripts, which realise the so called "stages". -Usually you'll not notice this, but in case you want to debug or hack cdist, -you can run each stage on its own. Besides that, you just need to remember -that the command cdist-deploy-to is the main cdist command. - -See also: - - Source of cdist-deploy-to(1), cdist-stages(7) - -eof -__prompt "$continue" - -################################################################################ -# Explorer -# -cat << eof - -The first thing cdist always does is running different explorers on the -target host. The explorers can be found in the directory - - ${__cdist_explorer_dir} - -An explorer is executed on the target host and its output is saved to a file. -You can use these files later to decide what or how to configure the host. - -For a demonstration, we'll call the OS explorer locally now, but remember: -This is only for demonstration, normally it is run on the target host. -The os explorer will which either displays the detected operating system or -nothing if it does not know your OS. - -See also: - - cdist-explorer(7) - -eof -explorer="${__cdist_explorer_dir}/os" - -__prompt "Press enter to execute $explorer" - -set -x -"$explorer" -set +x - -################################################################################ -# Manifest -# -cat << eof - -The initial manifest is the entry point for cdist to find out, what you would -like to have configured. It is located at - - ${__cdist_manifest_init} - -And can be as simple as - --------------------------------------------------------------------------------- -__file /etc/cdist-configured --type file --------------------------------------------------------------------------------- - -See also: - - cdist-manifest(7) - -eof -__prompt "$continue" - -cat << eof - -Let's take a deeper look at the initial manifest to understand what it means: - - __file /etc/cdist-configured --type file - | | | \\ - | | The parameter type \\ With the value file - | | - | | - | | This is the object id - | - __file is a so called "type" - - -This essentially looks like a standard command executed in the shell. -eof -__prompt "$continue" - -cat << eof - -And that's exactly true. Manifests are shell snippets that can use -types as commands with arguments. cdist prepends a special path -that contain links to the cdist-type-emulator, to \$PATH, so you -can use your types as a command. - -This is also the reason why types should always be prefixed with -"__", to prevent collisions with existing binaries. - -The object id is unique per type and used to prevent you from creating -the same object twice. - -Parameters are type specific and are always specified as --parameter . - -See also: - - cdist-type-build-emulation(1), cdist-type-emulator(1) - -eof -__prompt "$continue" - -################################################################################ -# Types -# -cat << eof - -Types are bundled functionality and are the main component of cdist. -If you want to have a feature x, you write the type __x. Types are stored in - - ${__cdist_type_dir} - -And cdist ships with some types already! - -See also: - - cdist-type(7) - -eof -__prompt "Press enter to see available types" - -set -x -ls ${__cdist_type_dir} -set +x - -cat << eof - -Types consist of the following parts: - - - ${__cdist_name_parameter} (${__cdist_name_parameter_required}/${__cdist_name_parameter_optional} - - ${__cdist_name_manifest} - - ${__cdist_name_explorer} - - ${__cdist_name_gencode} - -eof -__prompt "$continue" - - -cat << eof - -Every type must have a directory named ${__cdist_name_parameter}, which -contains required or optional parameters (in newline seperated files). - -If an object of a specific type was created in the initial manifest, -the manifest of the type is run and may create other objects. - -A type may have ${__cdist_name_explorer}, which are very similar to the -${__cdist_name_explorer} seen above, but with a different purpose: -They are specific to the type and are not relevant for other types. - -You may use them for instance to find out details on the target host, -so you can decide what to do on the target host eventually. - -After the ${__cdist_name_manifest} and the ${__cdist_name_explorer} of -a type have been run, ${__cdist_name_gencode} is executed, which creates -code to be executed on the target on stdout. - -eof -__prompt "$continue" - -################################################################################ -# Deployment -# - -cat << eof - -Now you've got some basic knowledge about cdist, let's configure your a host! - -Ensure that you have a ssh server running on the host and that you can login as root. - -eof - -__prompt "Enter hostname or press enter for localhost: " - -if [ "$answer" ]; then - host="$answer" -else - host="localhost" -fi - -manifestinit="conf/manifest/init" -cat << eof - -I'll now setup $manifestinit, containing the following code: - --------------------------------------------------------------------------------- -# Every machine becomes a marker, so sysadmins know that automatic -# configurations are happening -__file /etc/cdist-configured - -case "\$__target_host" in - $host) - __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic - __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" - ;; -esac --------------------------------------------------------------------------------- - -WARNING: This will overwrite ${manifestinit}. - -eof - -cat > "$__cdist_abs_mydir/../$manifestinit" << eof - -# Every machine becomes a marker, so sysadmins know that automatic -# configurations are happening -__file /etc/cdist-configured - -case "\$__target_host" in - $host) - __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic - __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" - ;; -esac - -eof - -chmod u+x "$__cdist_abs_mydir/../$manifestinit" - -cmd="cdist-deploy-to $host" - -__prompt "Press enter to run \"$cmd\"" - -# No quotes, we need field splitting -$cmd - -################################################################################ -# End -# - -cat << eof - - --------------------------------------------------------------------------------- -That's it, this is the end of the cdist-quickstart. - -I hope you've got some impression on how cdist works, here are again some -pointers on where to continue to read: - - -eof - SEE ALSO -------- cdist(1), cdist-type(7), cdist-stages(7) - From cdd4e0968343cc72d79f3228fd7e2f56c611c404 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:12:09 +0200 Subject: [PATCH 16/20] prefix/localhost in tutorial Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-tutorial.text | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index 15f2a81d..80135da9 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -49,7 +49,7 @@ with public keys: ssh-keygen # Copy pubkey over to target host -ssh-copy-id root@target_host +ssh-copy-id root@localhost -------------------------------------------------------------------------------- As soon as you are able to login without passwort to the target host, @@ -75,7 +75,7 @@ ls -l /etc/cdist-configured The file 'conf/manifest/init' is usually the entry point for cdist, to find out what to configure on which host. All manifests are essentially shell scripts. Every manifest can use the types known to -cdist, which are usually underline prefixed (\_\_). +cdist, which are usually underline prefixed (__). SEE ALSO From 3e49b12a6d30dd4621aa5150948c3662cc18aaf0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:17:32 +0200 Subject: [PATCH 17/20] update cdist-hacker Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-hacker.text | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/doc/man/man7/cdist-hacker.text b/doc/man/man7/cdist-hacker.text index efd6ef7d..b9f79d01 100644 --- a/doc/man/man7/cdist-hacker.text +++ b/doc/man/man7/cdist-hacker.text @@ -26,26 +26,12 @@ in the latest version, drop a mail to the cdist mailing list, subject prefixed with "[BUG] ". -UNDERSTANDING CDIST INTERNALS ------------------------------ -IF you are interested in how cdist internally works, you can open -bin/cdist-config and bin/cdist-deploy-to in your favorite editor and -read the scripts bin/cdist-deploy-to calls. The magnificent HACKERS_README -may be of great help as well. - - CODING CONVENTIONS (EVERYWHERE) ------------------------------- If something should be better done or needs to fixed, add the word FIXME nearby, so grepping for FIXME gives all positions that need to be fixed. -CODING CONVENTIONS (CORE) -------------------------- -- All variables exported by cdist are prefixed with a double underscore (__) -- All cdist-internal variables are prefixed with __cdist_ and are generally not exported. - - HOW TO SUBMIT STUFF FOR INCLUSION INTO UPSTREAM CDIST ----------------------------------------------------- If you did some cool changes to cdist, which you value as a benefit for From ae4d6002f06364a12598d449262c3872746bc7b2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:21:16 +0200 Subject: [PATCH 18/20] permissions/rephrase Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-explorer.text | 2 +- doc/man/man7/cdist-tutorial.text | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 doc/man/man7/cdist-tutorial.text diff --git a/doc/man/man7/cdist-explorer.text b/doc/man/man7/cdist-explorer.text index 63c7a5c9..e1909ab5 100644 --- a/doc/man/man7/cdist-explorer.text +++ b/doc/man/man7/cdist-explorer.text @@ -26,7 +26,7 @@ $__explorer/ (general and type explorer) or $__type_explorer/ (type explorer). In case of significant errors, the explorer may exit non-zero and return an -error message on stderr, which will cause the cdist run to abort. +error message on stderr, which will cause cdist to abort. You can also use stderr for debugging purposes while developing a new explorer. diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text old mode 100755 new mode 100644 From 75023a4ac9061f0eca8792622e972d5d4b57e124 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:22:45 +0200 Subject: [PATCH 19/20] begin to cleanup cdist-stages Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 + doc/man/man7/cdist-stages.text | 35 +--------------------------------- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 91b418a2..14098099 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -8,6 +8,7 @@ - and that ssh will wait for answer of prompt - nasty if used in parallel mode (scroll up!) +- rewrite cdist-stages SSH HINTS --------- diff --git a/doc/man/man7/cdist-stages.text b/doc/man/man7/cdist-stages.text index 294dffc7..8ac30015 100644 --- a/doc/man/man7/cdist-stages.text +++ b/doc/man/man7/cdist-stages.text @@ -32,11 +32,6 @@ explorers. Every existing explorer is run on the target and the output of all explorers are copied back into the local cache. The results can be used by manifests and types. -Related documentation: - - cdist-explorer-run-global(1) - - cdist-remote-explorer-run(1) - - cdist-explorer(7) - STAGE 2: RUN THE INITIAL MANIFEST --------------------------------- @@ -46,11 +41,6 @@ the objects as defined in the manifest for the specific host. In this stage, no conflicts may occur, i.e. no object of the same type with the same id may be created. -Related documentation: - - cdist-manifest-run-init(1) - - cdist-manifest-run(1) - - cdist-manifest(7) - STAGE 3: OBJECT INFORMATION RETRIEVAL ------------------------------------- @@ -59,12 +49,6 @@ transfered to the target host and executed. The results are transfered back and can be used in the following stages to decide what changes need to be made on the target to implement the desired state. -Related documentation: - - cdist-object-explorer-run(1) - - cdist-remote-explorer-run(1) - - cdist-type(7) - - cdist-explorer(7) - STAGE 4: RUN THE OBJECT MANIFEST -------------------------------- @@ -79,11 +63,6 @@ The newly created objects are merged back into the existing tree. No conflicts may occur during the merge. A conflict would mean that two different objects try to create the same object, which indicates a broken configuration. -Related documentation: - - cdist-object-manifest-run(1) - - cdist-manifest-run(1) - - cdist-type(7) - STAGE 5: CODE GENERATION ------------------------ @@ -92,29 +71,17 @@ gencode scripts. The gencode scripts generate the code to be executed on the target on stdout. If the gencode executables fail, they must print diagnostic messages on stderr and exit non-zero. -Related documentation: - - cdist-object-gencode-run(1) - - cdist-object-gencode(1) - - cdist-type(7) - STAGE 6: CODE EXECUTION ----------------------- For every object the resulting code from the previous stage is transferred to the target host and executed there to apply the configuration changes. -Related documentation: - - cdist-object-code-run(1) - - cdist-code-run(1) - STAGE 7: CACHE -------------- The cache stores the information from the current run for later use. -Related documentation: - - cdist-cache(1) - SUMMARY ------- @@ -126,8 +93,8 @@ in correct order. SEE ALSO -------- +- cdist(1) - cdist(7) -- cdist-deploy-to(1) - cdist-reference(7) From 4879f744c168b536843764a1e40a99a6821ed279 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:25:04 +0200 Subject: [PATCH 20/20] manpage cleanups Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 4 +++- doc/man/man7/cdist-type.text | 6 +----- doc/man/man7/cdist.text | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 14098099..b7748949 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -8,7 +8,9 @@ - and that ssh will wait for answer of prompt - nasty if used in parallel mode (scroll up!) -- rewrite cdist-stages +- rewrite cdist-stages, remove +- update man7! +- exec flag is not true for manifest anymore SSH HINTS --------- diff --git a/doc/man/man7/cdist-type.text b/doc/man/man7/cdist-type.text index 1af386fb..2439876c 100644 --- a/doc/man/man7/cdist-type.text +++ b/doc/man/man7/cdist-type.text @@ -35,10 +35,6 @@ __file /etc/cdist-configured --type file __package tree --state installed -------------------------------------------------------------------------------- -Internally cdist-type-emulator(1) will be called from cdist-manifest-run(1) to -save the given parameters into a cconfig database, so they can be accessed by -the manifest and gencode scripts of the type (see below). - A list of supported types can be found in the cdist-reference(7) manpage. SINGLETON TYPES @@ -111,7 +107,7 @@ __package_$type "$@" -------------------------------------------------------------------------------- As you can see, the type can reference different environment variables, -which are documented in cdist-environment-variables(7). +which are documented in cdist-reference(7). Always ensure the manifest is executable, otherwise cdist will not be able to execute it. diff --git a/doc/man/man7/cdist.text b/doc/man/man7/cdist.text index 9f7dbbab..2a5d1fe5 100644 --- a/doc/man/man7/cdist.text +++ b/doc/man/man7/cdist.text @@ -35,12 +35,11 @@ pull mechanism (client requests configuration). SEE ALSO -------- - Website: http://www.nico.schottelius.org/software/cdist/[] -- cdist-best-practise(7) -- cdist-deploy-to(1) - cdist-hacker(7) - cdist-manifest(7) -- cdist-quickstart(1) - cdist-type(7) +- cdist(1) +- cdist(7) COPYING