stephan/cdist
1
0
Fork 0
forked from ungleich-public/cdist
Code Issues Pull requests Projects Releases Wiki Activity

Merge remote-tracking branch 'telmich/master'

Browse source
This commit is contained in:
Steven Armstrong 2011-10-04 16:30:05 +02:00
commit b142435abd
13 changed files with 250 additions and 94 deletions
Download patch file Download diff file Expand all files Collapse all files

43
README
View file

@ -15,7 +15,7 @@
"P' "" ""
[[!toc levels=2]]
[[!toc levels=3]]
## Introduction
@ -86,13 +86,46 @@ cdist was tested or is know to run on at least
* SSH-Server
## Getting cdist
## Installation
### Preperation
Ensure you have Python 3.x and the **argparse** module installed on
the machine you use to **deploy to the targets**.
#### Archlinux
Archlinux already has python >= 3.2, so you only need to do:
pacman -S python
#### Debian
aptitude install python3 python3-setuptools
easy_install3 argparse
#### Gentoo
Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3).
If you want to ensure nothing breaks you must set back the python version to what was default before.
emerge -av =python-3.2.2 --autounmask-write
emerge -av =python-3.2.2
eselect python list
eselect python list set python3.2
#### Max OS X
Ensure you have port installed and configured (http://www.macports.org/install.php).
port install python32
ln -s /opt/local/bin/python3.2 /opt/local/bin/python3
### Get cdist
You can clone cdist from git, which gives you the advantage of having
a version control in place for development of your own stuff as well.
### Installation
To install cdist, execute the following commands:
git clone git://git.schottelius.org/cdist

34
bin/cdist
View file

@ -26,7 +26,7 @@ import os
import re
import sys
log = logging.getLogger(__name__)
log = logging.getLogger("cdist")
# Ensure our /lib/ is included into PYTHON_PATH
sys.path.insert(0, os.path.abspath(
@ -39,12 +39,17 @@ def commandline():
# Construct parser others can reuse
parser = {}
# Options _all_ parsers have in common
parser['most'] = argparse.ArgumentParser(add_help=False)
parser['most'].add_argument('-d', '--debug',
help='Set log level to debug', action='store_true')
parser['loglevel'] = argparse.ArgumentParser(add_help=False)
parser['loglevel'].add_argument('-d', '--debug',
help='Set log level to debug', action='store_true',
default=False)
parser['loglevel'].add_argument('-v', '--verbose',
help='Set log level to info, be more verbose',
action='store_true', default=False)
# Main subcommand parser
parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION)
parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION,
parents=[parser['loglevel']])
parser['main'].add_argument('-V', '--version',
help='Show version', action='version',
version='%(prog)s ' + cdist.VERSION)
@ -52,7 +57,7 @@ def commandline():
# Banner
parser['banner'] = parser['sub'].add_parser('banner',
add_help=False)
parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
# Config and install (common stuff)
@ -74,12 +79,12 @@ def commandline():
# Config
parser['config'] = parser['sub'].add_parser('config',
parents=[parser['most'], parser['configinstall']])
parents=[parser['loglevel'], parser['configinstall']])
parser['config'].set_defaults(func=cdist.config.config)
# Install
parser['install'] = parser['sub'].add_parser('install',
parents=[parser['most'], parser['configinstall']])
parents=[parser['loglevel'], parser['configinstall']])
parser['install'].set_defaults(func=cdist.install.install)
for p in parser:
@ -87,17 +92,18 @@ def commandline():
args = parser['main'].parse_args(sys.argv[1:])
# Most subcommands have --debug, so handle it here
if 'debug' in args:
if args.debug:
logging.root.setLevel(logging.DEBUG)
log.debug(args)
# Loglevels are handled globally in here and debug wins over verbose
if args.verbose:
logging.root.setLevel(logging.INFO)
if args.debug:
logging.root.setLevel(logging.DEBUG)
log.debug(args)
args.func(args)
if __name__ == "__main__":
try:
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
logging.basicConfig(format='%(levelname)s: %(message)s')
if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])):
import cdist.emulator

8
build.sh
View file

@ -126,6 +126,14 @@ case "$1" in
| xargs rm -f
;;
test)
python3 -m unittest discover test 'test_*.py'
;;
test-all)
python3 -m unittest discover test '*.py'
;;
*)
echo ''
echo 'Welcome to cdist!'

7
doc/changelog
View file

@ -1,5 +1,10 @@
2.0.2:
2.0.3:
* Improved logging, added --verbose, by more quiet by default
2.0.2: 2011-09-27
* Add support for detection of OpenWall Linux (Matthias Teege)
* Add support for __debug variable in manifests
* Bugfix core: Various issues with type emulator
2.0.1: 2011-09-23
* Bugfix core: Always print source of error in case of exec errors

2
doc/dev/todo/TAKEME
View file

@ -34,6 +34,8 @@ USER INTERFACE
-> given after manifest run already!
- use absent/present for state by default?
- buggy output with packages that don't exist in archlinux and fedora:
python3 vs. python
TYPES
------

4
doc/dev/todo/niconext
View file

@ -311,15 +311,13 @@ eof
via __global/
- Support parallel execution
- and maximum number of parallel runs (-p X)
- error handling / report failed hosts
- Allow manifest to be read from stdin
- Create new video for cdist 2.0.0
http://www.youtube.com/watch?v=PRMjzy48eTI
- Setup __debug, if -d is given, so other tools can reuse it
(-> non core feature!
- implement everywhere to external!
- remote_prefix:
scp vs. ssh issue

5
doc/man/cdist-reference.text.sh
View file

@ -166,6 +166,11 @@ changed::
ENVIRONMENT VARIABLES
---------------------
__debug::
If this variable is setup, cdist runs in debug mode.
You can use this information, to only output stuff in debug
mode as well.
Available for: initial manifest, type manifest
__explorer::
Directory that contains all global explorers.
Available for: explorer

2
lib/cdist/__init__.py
View file

@ -19,7 +19,7 @@
#
#
VERSION = "2.0.2"
VERSION = "2.0.3"
class Error(Exception):
"""Base exception class for this project"""

22
lib/cdist/config.py
View file

@ -65,6 +65,7 @@ class Config:
def run_global_explores(self):
"""Run global explorers"""
log.info("Running global explorers")
explorers = self.path.list_global_explorers()
if(len(explorers) == 0):
raise CdistError("No explorers found in", self.path.global_explorer_dir)
@ -107,18 +108,22 @@ class Config:
cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
output_fd.close()
def link_emulator(self):
"""Link emulator to types"""
cdist.emulator.link(self.exec_path,
self.path.bin_dir, self.path.list_types())
def init_deploy(self):
"""Ensure the base directories are cleaned up"""
log.debug("Creating clean directory structure")
self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR)
self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR)
cdist.emulator.link(self.exec_path,
self.path.bin_dir, self.path.list_types())
self.link_emulator()
def run_initial_manifest(self):
"""Run the initial manifest"""
log.info("Running initial manifest %s", self.path.initial_manifest)
env = { "__manifest" : self.path.manifest_dir }
self.run_manifest(self.path.initial_manifest, extra_env=env)
@ -146,6 +151,10 @@ class Config:
env['__target_host'] = self.target_host
env['__global'] = self.path.out_dir
# Submit debug flag to manifest, can be used by emulator and types
if self.debug:
env['__debug'] = "yes"
# Required for recording source
env['__cdist_manifest'] = manifest
@ -235,12 +244,13 @@ class Config:
self.run_global_explores()
self.run_initial_manifest()
log.info("Running object manifests and type explorers")
old_objects = []
objects = self.path.list_objects()
# Continue process until no new objects are created anymore
while old_objects != objects:
log.debug("Prepare stage")
old_objects = list(objects)
for cdist_object in objects:
if cdist_object in self.objects_prepared:
@ -255,7 +265,7 @@ class Config:
def stage_run(self):
"""The final (and real) step of deployment"""
log.debug("Actual run objects")
log.info("Generating and executing code")
# Now do the final steps over the existing objects
for cdist_object in self.path.list_objects():
log.debug("Run object: %s", cdist_object)
@ -298,7 +308,7 @@ def config(args):
if args.parallel:
for p in process.keys():
log.debug("Joining %s", p)
log.debug("Joining process %s", p)
process[p].join()
time_end = datetime.datetime.now()

25
lib/cdist/emulator.py
View file

@ -36,20 +36,21 @@ def run(argv):
global_dir = os.environ['__global']
object_source = os.environ['__cdist_manifest']
if '__debug' in os.environ:
logging.root.setLevel(logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser(add_help=False)
# Setup optional parameters
for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")):
argument = "--" + parameter
parser.add_argument(argument, action='store', required=False)
# Setup required parameters
for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")):
argument = "--" + parameter
parser.add_argument(argument, action='store', required=True)
# Setup positional parameter, if not singleton
# If not singleton support one positional parameter
if not os.path.isfile(os.path.join(type_dir, "singleton")):
parser.add_argument("object_id", nargs=1)
@ -67,6 +68,10 @@ def run(argv):
if object_id[0] == '/':
object_id = object_id[1:]
# Prefix output by object_self
logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s'
logging.basicConfig(format=logformat)
# FIXME: verify object id
log.debug(args)
@ -110,12 +115,12 @@ def run(argv):
value_old = param_fd.readlines()
param_fd.close()
if(value_old != value):
raise cdist.Error("Parameter + \"" + param +
if(value_old[0] != value):
raise cdist.Error("Parameter\"" + param +
"\" differs: " + " ".join(value_old) + " vs. " +
value +
"\nSource = " + " ".join(old_object_source)
+ " new =" + object_source)
+ " new = " + object_source)
else:
param_fd = open(file, "w")
param_fd.writelines(value)
@ -124,7 +129,7 @@ def run(argv):
# Record requirements
if "__require" in os.environ:
requirements = os.environ['__require']
print(object_id + ":Writing requirements: " + requirements)
log.debug(object_id + ":Writing requirements: " + requirements)
require_fd = open(os.path.join(object_dir, "require"), "a")
require_fd.writelines(requirements.split(" "))
require_fd.close()
@ -134,7 +139,7 @@ def run(argv):
source_fd.writelines(object_source)
source_fd.close()
print("Finished " + type + "/" + object_id + repr(params))
log.debug("Finished " + type + "/" + object_id + repr(params))
def link(exec_path, bin_dir, type_list):

45
test/nico_ui.py Executable file
View file

@ -0,0 +1,45 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import subprocess
import unittest
cdist_commands=["banner", "config", "install"]
cdist_exec_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "../bin/cdist"))
class UI(unittest.TestCase):
def test_banner(self):
self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0)
def test_help(self):
for cmd in cdist_commands:
self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0)
# FIXME: mockup needed
def test_config_localhost(self):
for cmd in cdist_commands:
self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0)

67
test.py → test/test_config.py Executable file → Normal file
View file

@ -20,68 +20,19 @@
#
#
import os
import sys
import shutil
import tempfile
import unittest
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')))
os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
import cdist.config
cdist_exec_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist"))
import cdist
import cdist.config
import cdist.exec
class Exec(unittest.TestCase):
def setUp(self):
"""Create shell code and co."""
self.temp_dir = tempfile.mkdtemp()
self.shell_false = os.path.join(self.temp_dir, "shell_false")
self.shell_true = os.path.join(self.temp_dir, "shell_true")
true_fd = open(self.shell_true, "w")
true_fd.writelines(["#!/bin/sh\n", "/bin/true"])
true_fd.close()
false_fd = open(self.shell_false, "w")
false_fd.writelines(["#!/bin/sh\n", "/bin/false"])
false_fd.close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_local_success_shell(self):
try:
cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail_shell(self):
self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail,
self.shell_false, [self.shell_false])
def test_local_success(self):
try:
cdist.exec.run_or_fail(["/bin/true"])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail(self):
self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"])
class Config(unittest.TestCase):
def setUp(self):
@ -90,6 +41,7 @@ class Config(unittest.TestCase):
self.config = cdist.config.Config("localhost",
initial_manifest=self.init_manifest,
exec_path=cdist_exec_path)
self.config.link_emulator()
def test_initial_manifest_different_parameter(self):
manifest_fd = open(self.init_manifest, "w")
@ -121,6 +73,14 @@ class Config(unittest.TestCase):
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_non_existent_command(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"thereisdefinitelynosuchcommend"])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_parameter_twice(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
@ -138,5 +98,4 @@ class Config(unittest.TestCase):
self.assertFalse(failed)
if __name__ == '__main__':
unittest.main()

80
test/test_exec.py Executable file
View file

@ -0,0 +1,80 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import sys
import shutil
import subprocess
import tempfile
import unittest
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
import cdist.exec
class Exec(unittest.TestCase):
def setUp(self):
"""Create shell code and co."""
self.temp_dir = tempfile.mkdtemp()
self.shell_false = os.path.join(self.temp_dir, "shell_false")
self.shell_true = os.path.join(self.temp_dir, "shell_true")
true_fd = open(self.shell_true, "w")
true_fd.writelines(["#!/bin/sh\n", "/bin/true"])
true_fd.close()
false_fd = open(self.shell_false, "w")
false_fd.writelines(["#!/bin/sh\n", "/bin/false"])
false_fd.close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_local_success_shell(self):
try:
cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail_shell(self):
self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail,
self.shell_false, [self.shell_false])
def test_local_success(self):
try:
cdist.exec.run_or_fail(["/bin/true"])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail(self):
self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"])