moep
/
cdist
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge remote-tracking branch 'telmich/master'

This commit is contained in:
Steven Armstrong 2011-10-04 16:30:05 +02:00
parent fe444eb04d cadb4fa852
commit b142435abd
13 changed files with 250 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
README
View File

@ -15,7 +15,7 @@
"P' "" ""
[[!toc levels=2]]
[[!toc levels=3]]
## Introduction
@ -86,13 +86,46 @@ cdist was tested or is know to run on at least
* SSH-Server
## Getting cdist
## Installation
### Preperation
Ensure you have Python 3.x and the **argparse** module installed on
the machine you use to **deploy to the targets**.
#### Archlinux
Archlinux already has python >= 3.2, so you only need to do:
pacman -S python
#### Debian
aptitude install python3 python3-setuptools
easy_install3 argparse
#### Gentoo
Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3).
If you want to ensure nothing breaks you must set back the python version to what was default before.
emerge -av =python-3.2.2 --autounmask-write
emerge -av =python-3.2.2
eselect python list
eselect python list set python3.2
#### Max OS X
Ensure you have port installed and configured (http://www.macports.org/install.php).
port install python32
ln -s /opt/local/bin/python3.2 /opt/local/bin/python3
### Get cdist
You can clone cdist from git, which gives you the advantage of having
a version control in place for development of your own stuff as well.
### Installation
To install cdist, execute the following commands:
git clone git://git.schottelius.org/cdist

34
bin/cdist
View File

@ -26,7 +26,7 @@ import os
import re
import sys
log = logging.getLogger(__name__)
log = logging.getLogger("cdist")
# Ensure our /lib/ is included into PYTHON_PATH
sys.path.insert(0, os.path.abspath(
@ -39,12 +39,17 @@ def commandline():
# Construct parser others can reuse
parser = {}
# Options _all_ parsers have in common
parser['most'] = argparse.ArgumentParser(add_help=False)
parser['most'].add_argument('-d', '--debug',
help='Set log level to debug', action='store_true')
parser['loglevel'] = argparse.ArgumentParser(add_help=False)
parser['loglevel'].add_argument('-d', '--debug',
help='Set log level to debug', action='store_true',
default=False)
parser['loglevel'].add_argument('-v', '--verbose',
help='Set log level to info, be more verbose',
action='store_true', default=False)
# Main subcommand parser
parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION)
parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION,
parents=[parser['loglevel']])
parser['main'].add_argument('-V', '--version',
help='Show version', action='version',
version='%(prog)s ' + cdist.VERSION)
@ -52,7 +57,7 @@ def commandline():
# Banner
parser['banner'] = parser['sub'].add_parser('banner',
add_help=False)
parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
# Config and install (common stuff)
@ -74,12 +79,12 @@ def commandline():
# Config
parser['config'] = parser['sub'].add_parser('config',
parents=[parser['most'], parser['configinstall']])
parents=[parser['loglevel'], parser['configinstall']])
parser['config'].set_defaults(func=cdist.config.config)
# Install
parser['install'] = parser['sub'].add_parser('install',
parents=[parser['most'], parser['configinstall']])
parents=[parser['loglevel'], parser['configinstall']])
parser['install'].set_defaults(func=cdist.install.install)
for p in parser:
@ -87,17 +92,18 @@ def commandline():
args = parser['main'].parse_args(sys.argv[1:])
# Most subcommands have --debug, so handle it here
if 'debug' in args:
if args.debug:
logging.root.setLevel(logging.DEBUG)
log.debug(args)
# Loglevels are handled globally in here and debug wins over verbose
if args.verbose:
logging.root.setLevel(logging.INFO)
if args.debug:
logging.root.setLevel(logging.DEBUG)
log.debug(args)
args.func(args)
if __name__ == "__main__":
try:
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
logging.basicConfig(format='%(levelname)s: %(message)s')
if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])):
import cdist.emulator

8
build.sh
View File

@ -126,6 +126,14 @@ case "$1" in
| xargs rm -f
;;
test)
python3 -m unittest discover test 'test_*.py'
;;
test-all)
python3 -m unittest discover test '*.py'
;;
*)
echo ''
echo 'Welcome to cdist!'

7
doc/changelog
View File

@ -1,5 +1,10 @@
2.0.2:
2.0.3:
* Improved logging, added --verbose, by more quiet by default
2.0.2: 2011-09-27
* Add support for detection of OpenWall Linux (Matthias Teege)
* Add support for __debug variable in manifests
* Bugfix core: Various issues with type emulator
2.0.1: 2011-09-23
* Bugfix core: Always print source of error in case of exec errors

2
doc/dev/todo/TAKEME
View File

@ -34,6 +34,8 @@ USER INTERFACE
-> given after manifest run already!
- use absent/present for state by default?
- buggy output with packages that don't exist in archlinux and fedora:
python3 vs. python
TYPES
------

4
doc/dev/todo/niconext
View File

@ -311,15 +311,13 @@ eof
via __global/
- Support parallel execution
- and maximum number of parallel runs (-p X)
- error handling / report failed hosts
- Allow manifest to be read from stdin
- Create new video for cdist 2.0.0
http://www.youtube.com/watch?v=PRMjzy48eTI
- Setup __debug, if -d is given, so other tools can reuse it
(-> non core feature!
- implement everywhere to external!
- remote_prefix:
scp vs. ssh issue

5
doc/man/cdist-reference.text.sh
View File

@ -166,6 +166,11 @@ changed::
ENVIRONMENT VARIABLES
---------------------
__debug::
If this variable is setup, cdist runs in debug mode.
You can use this information, to only output stuff in debug
mode as well.
Available for: initial manifest, type manifest
__explorer::
Directory that contains all global explorers.
Available for: explorer

2
lib/cdist/__init__.py
View File

@ -19,7 +19,7 @@
#
#
VERSION = "2.0.2"
VERSION = "2.0.3"
class Error(Exception):
"""Base exception class for this project"""

22
lib/cdist/config.py
View File

@ -65,6 +65,7 @@ class Config:
def run_global_explores(self):
"""Run global explorers"""
log.info("Running global explorers")
explorers = self.path.list_global_explorers()
if(len(explorers) == 0):
raise CdistError("No explorers found in", self.path.global_explorer_dir)
@ -107,18 +108,22 @@ class Config:
cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
output_fd.close()
def link_emulator(self):
"""Link emulator to types"""
cdist.emulator.link(self.exec_path,
self.path.bin_dir, self.path.list_types())
def init_deploy(self):
"""Ensure the base directories are cleaned up"""
log.debug("Creating clean directory structure")
self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR)
self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR)
cdist.emulator.link(self.exec_path,
self.path.bin_dir, self.path.list_types())
self.link_emulator()
def run_initial_manifest(self):
"""Run the initial manifest"""
log.info("Running initial manifest %s", self.path.initial_manifest)
env = { "__manifest" : self.path.manifest_dir }
self.run_manifest(self.path.initial_manifest, extra_env=env)
@ -146,6 +151,10 @@ class Config:
env['__target_host'] = self.target_host
env['__global'] = self.path.out_dir
# Submit debug flag to manifest, can be used by emulator and types
if self.debug:
env['__debug'] = "yes"
# Required for recording source
env['__cdist_manifest'] = manifest
@ -235,12 +244,13 @@ class Config:
self.run_global_explores()
self.run_initial_manifest()
log.info("Running object manifests and type explorers")
old_objects = []
objects = self.path.list_objects()
# Continue process until no new objects are created anymore
while old_objects != objects:
log.debug("Prepare stage")
old_objects = list(objects)
for cdist_object in objects:
if cdist_object in self.objects_prepared:
@ -255,7 +265,7 @@ class Config:
def stage_run(self):
"""The final (and real) step of deployment"""
log.debug("Actual run objects")
log.info("Generating and executing code")
# Now do the final steps over the existing objects
for cdist_object in self.path.list_objects():
log.debug("Run object: %s", cdist_object)
@ -298,7 +308,7 @@ def config(args):
if args.parallel:
for p in process.keys():
log.debug("Joining %s", p)
log.debug("Joining process %s", p)
process[p].join()
time_end = datetime.datetime.now()

25
lib/cdist/emulator.py
View File

@ -36,20 +36,21 @@ def run(argv):
global_dir = os.environ['__global']
object_source = os.environ['__cdist_manifest']
if '__debug' in os.environ:
logging.root.setLevel(logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser(add_help=False)
# Setup optional parameters
for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")):
argument = "--" + parameter
parser.add_argument(argument, action='store', required=False)
# Setup required parameters
for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")):
argument = "--" + parameter
parser.add_argument(argument, action='store', required=True)
# Setup positional parameter, if not singleton
# If not singleton support one positional parameter
if not os.path.isfile(os.path.join(type_dir, "singleton")):
parser.add_argument("object_id", nargs=1)
@ -67,6 +68,10 @@ def run(argv):
if object_id[0] == '/':
object_id = object_id[1:]
# Prefix output by object_self
logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s'
logging.basicConfig(format=logformat)
# FIXME: verify object id
log.debug(args)
@ -110,12 +115,12 @@ def run(argv):
value_old = param_fd.readlines()
param_fd.close()
if(value_old != value):
raise cdist.Error("Parameter + \"" + param +
if(value_old[0] != value):
raise cdist.Error("Parameter\"" + param +
"\" differs: " + " ".join(value_old) + " vs. " +
value +
"\nSource = " + " ".join(old_object_source)
+ " new =" + object_source)
+ " new = " + object_source)
else:
param_fd = open(file, "w")
param_fd.writelines(value)
@ -124,7 +129,7 @@ def run(argv):
# Record requirements
if "__require" in os.environ:
requirements = os.environ['__require']
print(object_id + ":Writing requirements: " + requirements)
log.debug(object_id + ":Writing requirements: " + requirements)
require_fd = open(os.path.join(object_dir, "require"), "a")
require_fd.writelines(requirements.split(" "))
require_fd.close()
@ -134,7 +139,7 @@ def run(argv):
source_fd.writelines(object_source)
source_fd.close()
print("Finished " + type + "/" + object_id + repr(params))
log.debug("Finished " + type + "/" + object_id + repr(params))
def link(exec_path, bin_dir, type_list):

45
test/nico_ui.py Executable file
View File

@ -0,0 +1,45 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import subprocess
import unittest
cdist_commands=["banner", "config", "install"]
cdist_exec_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "../bin/cdist"))
class UI(unittest.TestCase):
def test_banner(self):
self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0)
def test_help(self):
for cmd in cdist_commands:
self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0)
# FIXME: mockup needed
def test_config_localhost(self):
for cmd in cdist_commands:
self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0)

67
test.py → test/test_config.py Executable file → Normal file
View File

@ -20,68 +20,19 @@
#
#
import os
import sys
import shutil
import tempfile
import unittest
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')))
os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
import cdist.config
cdist_exec_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist"))
import cdist
import cdist.config
import cdist.exec
class Exec(unittest.TestCase):
def setUp(self):
"""Create shell code and co."""
self.temp_dir = tempfile.mkdtemp()
self.shell_false = os.path.join(self.temp_dir, "shell_false")
self.shell_true = os.path.join(self.temp_dir, "shell_true")
true_fd = open(self.shell_true, "w")
true_fd.writelines(["#!/bin/sh\n", "/bin/true"])
true_fd.close()
false_fd = open(self.shell_false, "w")
false_fd.writelines(["#!/bin/sh\n", "/bin/false"])
false_fd.close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_local_success_shell(self):
try:
cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail_shell(self):
self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail,
self.shell_false, [self.shell_false])
def test_local_success(self):
try:
cdist.exec.run_or_fail(["/bin/true"])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail(self):
self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"])
class Config(unittest.TestCase):
def setUp(self):
@ -90,6 +41,7 @@ class Config(unittest.TestCase):
self.config = cdist.config.Config("localhost",
initial_manifest=self.init_manifest,
exec_path=cdist_exec_path)
self.config.link_emulator()
def test_initial_manifest_different_parameter(self):
manifest_fd = open(self.init_manifest, "w")
@ -121,6 +73,14 @@ class Config(unittest.TestCase):
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_non_existent_command(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"thereisdefinitelynosuchcommend"])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_parameter_twice(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
@ -138,5 +98,4 @@ class Config(unittest.TestCase):
self.assertFalse(failed)
if __name__ == '__main__':
unittest.main()

80
test/test_exec.py Executable file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import sys
import shutil
import subprocess
import tempfile
import unittest
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
import cdist.exec
class Exec(unittest.TestCase):
def setUp(self):
"""Create shell code and co."""
self.temp_dir = tempfile.mkdtemp()
self.shell_false = os.path.join(self.temp_dir, "shell_false")
self.shell_true = os.path.join(self.temp_dir, "shell_true")
true_fd = open(self.shell_true, "w")
true_fd.writelines(["#!/bin/sh\n", "/bin/true"])
true_fd.close()
false_fd = open(self.shell_false, "w")
false_fd.writelines(["#!/bin/sh\n", "/bin/false"])
false_fd.close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_local_success_shell(self):
try:
cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail_shell(self):
self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail,
self.shell_false, [self.shell_false])
def test_local_success(self):
try:
cdist.exec.run_or_fail(["/bin/true"])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail(self):
self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"])