sparrowhawk/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge remote-tracking branch 'cdist/master' into partition_msdos

Browse source
This commit is contained in:
Steven Armstrong 2011-10-04 13:54:35 +02:00
commit 2fd0223347
28 changed files with 1613 additions and 1184 deletions
Download patch file Download diff file Expand all files Collapse all files

8
.gitignore vendored
View file

@ -11,8 +11,8 @@ doc/man/man7/cdist-type__*.text
doc/man/man7/cdist-reference.text doc/man/man7/cdist-reference.text
doc/man/man*/docbook-xsl.css doc/man/man*/docbook-xsl.css
# Ignore cache for version control # Ignore cdist cache for version control
cache/ /cache/
# Python # Python / cache
bin/__pycache__/ __pycache__/

39
README
View file

@ -15,7 +15,7 @@
"P' "" "" "P' "" ""
[[!toc levels=2]] [[!toc levels=3]]
## Introduction ## Introduction
@ -78,6 +78,7 @@ cdist was tested or is know to run on at least
* A posix like shell * A posix like shell
* Python (>= 3.2 required) * Python (>= 3.2 required)
* SSH-Client * SSH-Client
* Asciidoc (for building the manpages)
### Client ("target host") ### Client ("target host")
@ -85,20 +86,46 @@ cdist was tested or is know to run on at least
* SSH-Server * SSH-Server
## Getting cdist ## Installation
### Preperation
Ensure you have Python 3.x and the **argparse** module installed on
the machine you use to **deploy to the targets**.
#### Archlinux
Archlinux already has python >= 3.2, so you only need to do:
pacman -S python
#### Debian
aptitude install python3 python3-setuptools
easy_install3 argparse
#### Gentoo
Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3).
If you want to ensure nothing breaks you must set back the python version to what was default before.
emerge -av =python-3.2.2 --autounmask-write
emerge -av =python-3.2.2
eselect python list
eselect python list set python3.2
### Get cdist
You can clone cdist from git, which gives you the advantage of having You can clone cdist from git, which gives you the advantage of having
a version control in place for development of your own stuff as well. a version control in place for development of your own stuff as well.
### Installation
To install cdist, execute the following commands: To install cdist, execute the following commands:
git clone git://git.schottelius.org/cdist git clone git://git.schottelius.org/cdist
cd cdist cd cdist
export PATH=$PATH:$(pwd -P)/bin export PATH=$PATH:$(pwd -P)/bin
# If you want the manpages (requires gmake and asciidoc to be installed) # If you want the manpages
./build.sh man ./build.sh man
export MANPATH=$MANPATH:$(pwd -P)/doc/man export MANPATH=$MANPATH:$(pwd -P)/doc/man

736
bin/cdist
View file

@ -21,718 +21,44 @@
# #
import argparse import argparse
import datetime
import logging import logging
import multiprocessing
import os import os
import re import re
import subprocess
import shutil
import stat
import sys import sys
import tempfile
BANNER = """ log = logging.getLogger("cdist")
.. . .x+=:. s
dF @88> z` ^% :8
'88bu. %8P . <k .88
. '*88888bu . .@8Ned8" :888ooo
.udR88N ^"*8888N .@88u .@^%8888" -*8888888
<888'888k beWE "888L ''888E` x88: `)8b. 8888
9888 'Y" 888E 888E 888E 8888N=*8888 8888
9888 888E 888E 888E %8" R88 8888
9888 888E 888F 888E @8Wou 9% .8888Lu=
?8888u../ .888N..888 888& .888888P` ^%888*
"8888P' `"888*"" R888" ` ^"F 'Y"
"P' "" ""
"""
# Given paths from installation # Ensure our /lib/ is included into PYTHON_PATH
REMOTE_BASE_DIR = "/var/lib/cdist" sys.path.insert(0, os.path.abspath(
REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object")
REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type")
REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer")
CODE_HEADER = "#!/bin/sh -e\n"
DOT_CDIST = ".cdist"
TYPE_PREFIX = "__" TYPE_PREFIX = "__"
VERSION = "2.0.0"
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
log = logging.getLogger()
def file_to_list(filename):
"""Return list from \n seperated file"""
if os.path.isfile(filename):
file_fd = open(filename, "r")
lines = file_fd.readlines()
file_fd.close()
# Remove \n from all lines
lines = map(lambda s: s.strip(), lines)
else:
lines = []
return lines
def exit_error(*args):
log.error(*args)
sys.exit(1)
class Cdist:
"""Cdist main class to hold arbitrary data"""
def __init__(self, target_host,
initial_manifest=False, remote_user="root",
home=None, debug=False):
self.target_host = target_host
self.remote_prefix = ["ssh", "root@" + self.target_host]
# Setup directory paths
self.temp_dir = tempfile.mkdtemp()
self.debug = debug
if home:
self.base_dir = home
else:
self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
self.conf_dir = os.path.join(self.base_dir, "conf")
self.cache_base_dir = os.path.join(self.base_dir, "cache")
self.cache_dir = os.path.join(self.cache_base_dir, self.target_host)
self.global_explorer_dir = os.path.join(self.conf_dir, "explorer")
self.lib_dir = os.path.join(self.base_dir, "lib")
self.manifest_dir = os.path.join(self.conf_dir, "manifest")
self.type_base_dir = os.path.join(self.conf_dir, "type")
self.out_dir = os.path.join(self.temp_dir, "out")
os.mkdir(self.out_dir)
self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer")
os.mkdir(self.global_explorer_out_dir)
self.object_base_dir = os.path.join(self.out_dir, "object")
# Setup binary directory + contents
self.bin_dir = os.path.join(self.out_dir, "bin")
os.mkdir(self.bin_dir)
self.link_type_to_emulator()
# List of type explorers transferred
self.type_explorers_transferred = {}
# objects
self.objects_prepared = []
self.remote_user = remote_user
# Mostly static, but can be overwritten on user demand
if initial_manifest:
self.initial_manifest = initial_manifest
else:
self.initial_manifest = os.path.join(self.manifest_dir, "init")
def cleanup(self):
# Do not use in __del__:
# http://docs.python.org/reference/datamodel.html#customization
# "other globals referenced by the __del__() method may already have been deleted
# or in the process of being torn down (e.g. the import machinery shutting down)"
#
log.debug("Saving" + self.temp_dir + "to " + self.cache_dir)
# Remove previous cache
if os.path.exists(self.cache_dir):
shutil.rmtree(self.cache_dir)
shutil.move(self.temp_dir, self.cache_dir)
def remote_mkdir(self, directory):
"""Create directory on remote side"""
self.run_or_fail(["mkdir", "-p", directory], remote=True)
def remote_cat(filename):
"""Use cat on the remote side for output"""
self.run_or_fail(["cat", filename], remote=True)
def shell_run_or_debug_fail(self, script, *args, **kargs):
# Manually execute /bin/sh, because sh -e does what we want
# and sh -c -e does not exit if /bin/false called
args[0][:0] = [ "/bin/sh", "-e" ]
remote = False
if "remote" in kargs:
if kargs["remote"]:
args[0][:0] = self.remote_prefix
remote = true
del kargs["remote"]
log.debug("Shell exec cmd: %s", args)
log.debug("Shell exec env: %s", kargs['env'])
try:
subprocess.check_call(*args, **kargs)
except subprocess.CalledProcessError:
log.error("Code that raised the error:\n")
if remote:
remote_cat(script)
else:
script_fd = open(script)
print(script_fd.read())
script_fd.close()
exit_error("Command failed (shell): " + " ".join(*args))
except OSError as error:
exit_error(" ".join(*args) + ": " + error.args[1])
def run_or_fail(self, *args, **kargs):
if "remote" in kargs:
if kargs["remote"]:
args[0][:0] = self.remote_prefix
del kargs["remote"]
log.debug("Exec: " + " ".join(*args))
try:
subprocess.check_call(*args, **kargs)
except subprocess.CalledProcessError:
exit_error("Command failed: " + " ".join(*args))
except OSError as error:
exit_error(" ".join(*args) + ": " + error.args[1])
def remove_remote_dir(self, destination):
self.run_or_fail(["rm", "-rf", destination], remote=True)
def transfer_dir(self, source, destination):
"""Transfer directory and previously delete the remote destination"""
self.remove_remote_dir(destination)
self.run_or_fail(["scp", "-qr", source,
self.remote_user + "@" +
self.target_host + ":" +
destination])
def transfer_file(self, source, destination):
"""Transfer file"""
self.run_or_fail(["scp", "-q", source,
self.remote_user + "@" +
self.target_host + ":" +
destination])
def global_explorer_output_path(self, explorer):
"""Returns path of the output for a global explorer"""
return os.path.join(self.global_explorer_out_dir, explorer)
def type_explorer_output_dir(self, cdist_object):
"""Returns and creates dir of the output for a type explorer"""
dir = os.path.join(self.object_dir(cdist_object), "explorer")
if not os.path.isdir(dir):
os.mkdir(dir)
return dir
def remote_global_explorer_path(self, explorer):
"""Returns path to the remote explorer"""
return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer)
def list_global_explorers(self):
"""Return list of available explorers"""
return os.listdir(self.global_explorer_dir)
def list_type_explorers(self, type):
"""Return list of available explorers for a specific type"""
dir = self.type_dir(type, "explorer")
if os.path.isdir(dir):
list = os.listdir(dir)
else:
list = []
log.debug("Explorers for %s in %s: %s", type, dir, list)
return list
def list_types(self):
return os.listdir(self.type_base_dir)
def list_object_paths(self, starting_point):
"""Return list of paths of existing objects"""
object_paths = []
for content in os.listdir(starting_point):
full_path = os.path.join(starting_point, content)
if os.path.isdir(full_path):
object_paths.extend(self.list_object_paths(starting_point = full_path))
# Directory contains .cdist -> is an object
if content == DOT_CDIST:
object_paths.append(starting_point)
return object_paths
def get_type_from_object(self, cdist_object):
"""Returns the first part (i.e. type) of an object"""
return cdist_object.split(os.sep)[0]
def get_object_id_from_object(self, cdist_object):
"""Returns everything but the first part (i.e. object_id) of an object"""
return os.sep.join(cdist_object.split(os.sep)[1:])
def object_dir(self, cdist_object):
"""Returns the full path to the object (including .cdist)"""
return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST)
def remote_object_dir(self, cdist_object):
"""Returns the remote full path to the object (including .cdist)"""
return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST)
def object_parameter_dir(self, cdist_object):
"""Returns the dir to the object parameter"""
return os.path.join(self.object_dir(cdist_object), "parameter")
def remote_object_parameter_dir(self, cdist_object):
"""Returns the remote dir to the object parameter"""
return os.path.join(self.remote_object_dir(cdist_object), "parameter")
def object_code_paths(self, cdist_object):
"""Return paths to code scripts of object"""
return [os.path.join(self.object_dir(cdist_object), "code-local"),
os.path.join(self.object_dir(cdist_object), "code-remote")]
def list_objects(self):
"""Return list of existing objects"""
objects = []
if os.path.isdir(self.object_base_dir):
object_paths = self.list_object_paths(self.object_base_dir)
for path in object_paths:
objects.append(os.path.relpath(path, self.object_base_dir))
return objects
def type_dir(self, type, *args):
"""Return directory the type"""
return os.path.join(self.type_base_dir, type, *args)
def remote_type_explorer_dir(self, type):
"""Return remote directory that holds the explorers of a type"""
return os.path.join(REMOTE_TYPE_DIR, type, "explorer")
def transfer_object_parameter(self, cdist_object):
"""Transfer the object parameter to the remote destination"""
# Create base path before using mkdir -p
self.remote_mkdir(self.remote_object_parameter_dir(cdist_object))
# Synchronise parameter dir afterwards
self.transfer_dir(self.object_parameter_dir(cdist_object),
self.remote_object_parameter_dir(cdist_object))
def transfer_global_explorers(self):
"""Transfer the global explorers"""
self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR)
self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR)
def transfer_type_explorers(self, type):
"""Transfer explorers of a type, but only once"""
if type in self.type_explorers_transferred:
log.debug("Skipping retransfer for explorers of %s", type)
return
else:
# Do not retransfer
self.type_explorers_transferred[type] = 1
src = self.type_dir(type, "explorer")
remote_base = os.path.join(REMOTE_TYPE_DIR, type)
dst = self.remote_type_explorer_dir(type)
# Only continue, if there is at least the directory
if os.path.isdir(src):
# Ensure that the path exists
self.remote_mkdir(remote_base)
self.transfer_dir(src, dst)
def link_type_to_emulator(self):
"""Link type names to cdist-type-emulator"""
source = os.path.abspath(sys.argv[0])
for type in self.list_types():
destination = os.path.join(self.bin_dir, type)
log.debug("Linking %s to %s", source, destination)
os.symlink(source, destination)
def run_global_explores(self):
"""Run global explorers"""
explorers = self.list_global_explorers()
if(len(explorers) == 0):
exit_error("No explorers found in", self.global_explorer_dir)
self.transfer_global_explorers()
for explorer in explorers:
output = self.global_explorer_output_path(explorer)
output_fd = open(output, mode='w')
cmd = []
cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR)
cmd.append(self.remote_global_explorer_path(explorer))
self.run_or_fail(cmd, stdout=output_fd, remote=True)
output_fd.close()
def run_type_explorer(self, cdist_object):
"""Run type specific explorers for objects"""
# Based on bin/cdist-object-explorer-run
# Transfering explorers for this type
type = self.get_type_from_object(cdist_object)
self.transfer_type_explorers(type)
cmd = []
cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR)
cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type))
cmd.append("__object=" + self.remote_object_dir(cdist_object))
cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object))
cmd.append("__object_fq=" + cdist_object)
# Need to transfer at least the parameters for objects to be useful
self.transfer_object_parameter(cdist_object)
explorers = self.list_type_explorers(type)
for explorer in explorers:
remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)]
output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer)
output_fd = open(output, mode='w')
log.debug("%s exploring %s using %s storing to %s",
cdist_object, explorer, remote_cmd, output)
self.run_or_fail(remote_cmd, stdout=output_fd, remote=True)
output_fd.close()
def init_deploy(self):
"""Ensure the base directories are cleaned up"""
log.debug("Creating clean directory structure")
self.remove_remote_dir(REMOTE_BASE_DIR)
self.remote_mkdir(REMOTE_BASE_DIR)
def run_initial_manifest(self):
"""Run the initial manifest"""
env = { "__manifest" : self.manifest_dir }
self.run_manifest(self.initial_manifest, extra_env=env)
def run_type_manifest(self, cdist_object):
"""Run manifest for a specific object"""
type = self.get_type_from_object(cdist_object)
manifest = self.type_dir(type, "manifest")
log.debug("%s: Running %s", cdist_object, manifest)
if os.path.exists(manifest):
env = { "__object" : self.object_dir(cdist_object),
"__object_id": self.get_object_id_from_object(cdist_object),
"__object_fq": cdist_object,
"__type": self.type_dir(type)
}
self.run_manifest(manifest, extra_env=env)
def run_manifest(self, manifest, extra_env=None):
"""Run a manifest"""
log.debug("Running manifest %s, env=%s", manifest, extra_env)
env = os.environ.copy()
env['PATH'] = self.bin_dir + ":" + env['PATH']
# Information required in every manifest
env['__target_host'] = self.target_host
env['__global'] = self.out_dir
# Legacy stuff to make cdist-type-emulator work
env['__cdist_core_dir'] = os.path.join(self.base_dir, "core")
env['__cdist_local_base_dir'] = self.temp_dir
# Submit information to new type emulator
env['__cdist_manifest'] = manifest
env['__cdist_type_base_dir'] = self.type_base_dir
# Other environment stuff
if extra_env:
env.update(extra_env)
self.shell_run_or_debug_fail(manifest, [manifest], env=env)
def object_run(self, cdist_object, mode):
"""Run gencode or code for an object"""
log.debug("Running %s from %s", mode, cdist_object)
file=os.path.join(self.object_dir(cdist_object), "require")
requirements = file_to_list(file)
type = self.get_type_from_object(cdist_object)
for requirement in requirements:
log.debug("Object %s requires %s", cdist_object, requirement)
self.object_run(requirement, mode=mode)
#
# Setup env Variable:
#
env = os.environ.copy()
env['__target_host'] = self.target_host
env['__global'] = self.out_dir
env["__object"] = self.object_dir(cdist_object)
env["__object_id"] = self.get_object_id_from_object(cdist_object)
env["__object_fq"] = cdist_object
env["__type"] = self.type_dir(type)
if mode == "gencode":
paths = [
self.type_dir(type, "gencode-local"),
self.type_dir(type, "gencode-remote")
]
for bin in paths:
if os.path.isfile(bin):
# omit "gen" from gencode and
outfile=os.path.join(self.object_dir(cdist_object),
os.path.basename(bin)[3:])
outfile_fd = open(outfile, "w")
# Need to flush to ensure our write is done before stdout write
outfile_fd.write(CODE_HEADER)
outfile_fd.flush()
self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd)
outfile_fd.close()
status = os.stat(outfile)
# Remove output if empty, else make it executable
if status.st_size == len(CODE_HEADER):
os.unlink(outfile)
else:
# Add header and make executable - identically to 0o700
os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
if mode == "code":
local_dir = self.object_dir(cdist_object)
remote_dir = self.remote_object_dir(cdist_object)
bin = os.path.join(local_dir, "code-local")
if os.path.isfile(bin):
self.run_or_fail([bin], remote=False)
local_remote_code = os.path.join(local_dir, "code-remote")
remote_remote_code = os.path.join(remote_dir, "code-remote")
if os.path.isfile(local_remote_code):
self.transfer_file(local_remote_code, remote_remote_code)
self.run_or_fail([remote_remote_code], remote=True)
def stage_prepare(self):
"""Do everything for a deploy, minus the actual code stage"""
self.init_deploy()
self.run_global_explores()
self.run_initial_manifest()
old_objects = []
objects = self.list_objects()
# Continue process until no new objects are created anymore
while old_objects != objects:
log.debug("Prepare stage")
old_objects = list(objects)
for cdist_object in objects:
if cdist_object in self.objects_prepared:
log.debug("Skipping rerun of object %s", cdist_object)
continue
else:
self.run_type_explorer(cdist_object)
self.run_type_manifest(cdist_object)
self.objects_prepared.append(cdist_object)
objects = self.list_objects()
def stage_run(self):
"""The final (and real) step of deployment"""
log.debug("Actual run objects")
# Now do the final steps over the existing objects
for cdist_object in self.list_objects():
log.debug("Run object: %s", cdist_object)
self.object_run(cdist_object, mode="gencode")
self.object_run(cdist_object, mode="code")
def deploy_to(self):
"""Mimic the old deploy to: Deploy to one host"""
log.info("Deploying to " + self.target_host)
time_start = datetime.datetime.now()
self.stage_prepare()
self.stage_run()
time_end = datetime.datetime.now()
duration = time_end - time_start
log.info("Finished run of %s in %s seconds",
self.target_host,
duration.total_seconds())
def deploy_and_cleanup(self):
"""Do what is most often done: deploy & cleanup"""
self.deploy_to()
self.cleanup()
def banner(args):
"""Guess what :-)"""
print(BANNER)
sys.exit(0)
def config(args):
"""Configure remote system"""
process = {}
time_start = datetime.datetime.now()
for host in args.host:
c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug)
if args.parallel:
log.debug("Creating child process for %s", host)
process[host] = multiprocessing.Process(target=c.deploy_and_cleanup)
process[host].start()
else:
c.deploy_and_cleanup()
if args.parallel:
for p in process.keys():
log.debug("Joining %s", p)
process[p].join()
time_end = datetime.datetime.now()
log.info("Total processing time for %s host(s): %s", len(args.host),
(time_end - time_start).total_seconds())
def install(args):
"""Install remote system"""
process = {}
def emulator():
"""Emulate type commands (i.e. __file and co)"""
type = os.path.basename(sys.argv[0])
type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type)
param_dir = os.path.join(type_dir, "parameter")
global_dir = os.environ['__global']
object_source = os.environ['__cdist_manifest']
parser = argparse.ArgumentParser(add_help=False)
# Setup optional parameters
for parameter in file_to_list(os.path.join(param_dir, "optional")):
argument = "--" + parameter
parser.add_argument(argument, action='store', required=False)
# Setup required parameters
for parameter in file_to_list(os.path.join(param_dir, "required")):
argument = "--" + parameter
parser.add_argument(argument, action='store', required=True)
# Setup positional parameter, if not singleton
if not os.path.isfile(os.path.join(type_dir, "singleton")):
parser.add_argument("object_id", nargs=1)
# And finally verify parameter
args = parser.parse_args(sys.argv[1:])
# Setup object_id
if os.path.isfile(os.path.join(type_dir, "singleton")):
object_id = "singleton"
else:
object_id = args.object_id[0]
del args.object_id
# FIXME: / hardcoded - better portable solution available?
if object_id[0] == '/':
object_id = object_id[1:]
# FIXME: verify object id
log.debug(args)
object_dir = os.path.join(global_dir, "object", type,
object_id, DOT_CDIST)
param_out_dir = os.path.join(object_dir, "parameter")
object_source_file = os.path.join(object_dir, "source")
if os.path.exists(param_out_dir):
object_exists = True
old_object_source_fd = open(object_source_file, "r")
old_object_source = old_object_source_fd.readlines()
old_object_source_fd.close()
else:
object_exists = False
try:
os.makedirs(param_out_dir, exist_ok=True)
except OSError as error:
exit_error(param_out_dir + ": " + error.args[1])
# Record parameter
params = vars(args)
for param in params:
value = getattr(args, param)
if value:
file = os.path.join(param_out_dir, param)
log.debug(file + "<-" + param + " = " + value)
# Already exists, verify all parameter are the same
if object_exists:
if not os.path.isfile(file):
print("New parameter + " + param + "specified, aborting")
print("Source = " + old_object_source + "new =" + object_source)
sys.exit(1)
else:
param_fd = open(file, "r")
param_old = param_fd.realines()
param_fd.close()
if(param_old != param):
print("Parameter differs: " + param_old + "vs," + param)
print("Source = " + old_object_source + "new =" + object_source)
sys.exit(1)
else:
param_fd = open(file, "w")
param_fd.writelines(value)
param_fd.close()
# Record requirements
if "__require" in os.environ:
requirements = os.environ['__require']
print(object_id + ":Writing requirements: " + requirements)
require_fd = open(os.path.join(object_dir, "require"), "a")
require_fd.writelines(requirements.split(" "))
require_fd.close()
# Record / Append source
source_fd = open(os.path.join(object_dir, "source"), "a")
source_fd.writelines(object_source)
source_fd.close()
# sys.exit(1)
print("Finished " + type + "/" + object_id + repr(params))
def commandline(): def commandline():
"""Parse command line""" """Parse command line"""
# Construct parser others can reuse # Construct parser others can reuse
parser = {} parser = {}
# Options _all_ parsers have in common # Options _all_ parsers have in common
parser['most'] = argparse.ArgumentParser(add_help=False) parser['loglevel'] = argparse.ArgumentParser(add_help=False)
parser['most'].add_argument('-d', '--debug', parser['loglevel'].add_argument('-d', '--debug',
help='Set log level to debug', action='store_true') help='Set log level to debug', action='store_true',
default=False)
parser['loglevel'].add_argument('-v', '--verbose',
help='Set log level to info, be more verbose',
action='store_true', default=False)
# Main subcommand parser # Main subcommand parser
parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION,
parents=[parser['loglevel']])
parser['main'].add_argument('-V', '--version', parser['main'].add_argument('-V', '--version',
help='Show version', action='version', help='Show version', action='version',
version='%(prog)s ' + VERSION) version='%(prog)s ' + cdist.VERSION)
parser['sub'] = parser['main'].add_subparsers(title="Commands") parser['sub'] = parser['main'].add_subparsers(title="Commands")
# Banner # Banner
parser['banner'] = parser['sub'].add_parser('banner', parser['banner'] = parser['sub'].add_parser('banner',
add_help=False) parents=[parser['loglevel']])
parser['banner'].set_defaults(func=banner) parser['banner'].set_defaults(func=cdist.banner.banner)
# Config and install (common stuff) # Config and install (common stuff)
parser['configinstall'] = argparse.ArgumentParser(add_help=False) parser['configinstall'] = argparse.ArgumentParser(add_help=False)
@ -753,32 +79,46 @@ def commandline():
# Config # Config
parser['config'] = parser['sub'].add_parser('config', parser['config'] = parser['sub'].add_parser('config',
parents=[parser['most'], parser['configinstall']]) parents=[parser['loglevel'], parser['configinstall']])
parser['config'].set_defaults(func=config) parser['config'].set_defaults(func=cdist.config.config)
# Install # Install
parser['install'] = parser['sub'].add_parser('install', parser['install'] = parser['sub'].add_parser('install',
parents=[parser['most'], parser['configinstall']]) parents=[parser['loglevel'], parser['configinstall']])
parser['install'].set_defaults(func=install) parser['install'].set_defaults(func=cdist.install.install)
for p in parser: for p in parser:
parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
args = parser['main'].parse_args(sys.argv[1:]) args = parser['main'].parse_args(sys.argv[1:])
# Most subcommands have --debug, so handle it here # Loglevels are handled globally in here and debug wins over verbose
if 'debug' in args: if args.verbose:
logging.root.setLevel(logging.INFO)
if args.debug: if args.debug:
logging.root.setLevel(logging.DEBUG) logging.root.setLevel(logging.DEBUG)
log.debug(args)
log.debug(args)
args.func(args) args.func(args)
if __name__ == "__main__": if __name__ == "__main__":
try: try:
logging.basicConfig(format='%(levelname)s: %(message)s')
if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])):
emulator() import cdist.emulator
cdist.emulator.run(sys.argv)
else: else:
import cdist
import cdist.banner
import cdist.config
import cdist.exec
import cdist.install
import cdist.path
commandline() commandline()
except KeyboardInterrupt: except KeyboardInterrupt:
sys.exit(0) sys.exit(0)
except cdist.Error as e:
log.error(e)
sys.exit(1)

1
bin/cdist.py
View file

@ -1 +0,0 @@
cdist

4
build.sh
View file

@ -27,8 +27,8 @@
#set -e #set -e
# Manpage and HTML # Manpage and HTML
A2XM="a2x -f manpage --no-xmllint" A2XM="a2x -f manpage --no-xmllint -a encoding=UTF-8"
A2XH="a2x -f xhtml --no-xmllint" A2XH="a2x -f xhtml --no-xmllint -a encoding=UTF-8"
# Developer webbase # Developer webbase
WEBDIR=$HOME/niconetz WEBDIR=$HOME/niconetz

7
conf/explorer/os
View file

@ -65,8 +65,13 @@ if [ -f /etc/SuSE-release ]; then
exit 0 exit 0
fi fi
if [ -f /etc/owl-release ]; then
echo owl
exit 0
fi
if [ -f /etc/cdist-preos ]; then if [ -f /etc/cdist-preos ]; then
echo preos echo cdist-preos
exit 0 exit 0
fi fi

3
conf/explorer/os_version
View file

@ -42,6 +42,9 @@ case "$($__explorer/os)" in
*bsd|solaris) *bsd|solaris)
uname -r uname -r
;; ;;
owl)
cat /etc/owl-release
;;
redhat|centos) redhat|centos)
cat /etc/redhat-release cat /etc/redhat-release
;; ;;

2
conf/type/__partition_msdos_apply/man.text
View file

@ -5,7 +5,7 @@ Steven Armstrong <steven-cdist--@--armstrong.cc>
NAME NAME
---- ----
cdist-type__partition_msdos_apply cdist-type__partition_msdos_apply - Apply dos partition settings
DESCRIPTION DESCRIPTION

14
doc/changelog
View file

@ -1,5 +1,15 @@
2.0.1: 2.0.3:
* Bugfix cdist: Always print source of error in case of exec errors * Improved logging, added --verbose, by more quiet by default
2.0.2: 2011-09-27
* Add support for detection of OpenWall Linux (Matthias Teege)
* Add support for __debug variable in manifests
* Bugfix core: Various issues with type emulator
2.0.1: 2011-09-23
* Bugfix core: Always print source of error in case of exec errors
* Bugfix core: Various smaller bugs in string concatenation
* Feature: Add marker "changed" to changed objects
2.0.0: 2011-09-16 2.0.0: 2011-09-16
* New Type: __package_rubygem (Chase Allen James) * New Type: __package_rubygem (Chase Allen James)

8
doc/dev/todo/TAKEME
View file

@ -9,6 +9,12 @@ CORE
- allow cdist to run without $PATH setup: ./bin/cdist-deploy-to - allow cdist to run without $PATH setup: ./bin/cdist-deploy-to
- support non-ssh access? - support non-ssh access?
TESTS
-----
- multiple defines of object:
- fail if different parameters
- succeed if same parameters
USER INTERFACE USER INTERFACE
-------------- --------------
- add support $__tmp? - add support $__tmp?
@ -28,6 +34,8 @@ USER INTERFACE
-> given after manifest run already! -> given after manifest run already!
- use absent/present for state by default? - use absent/present for state by default?
- buggy output with packages that don't exist in archlinux and fedora:
python3 vs. python
TYPES TYPES
------ ------

310
doc/dev/todo/niconext
View file

@ -1,18 +1,296 @@
2.0.1:
- Rewrite cdist-type-emulator
- Remove legacy code in cdist
- Remove cdist-config
- Remove man1/cdist-type-emulator.text
- Remove the PATH=... part from the README
- how to access output dir?
Test:
__cdist_type_base_dir=$(pwd -P)/conf/type __file
- Fix / rewrite cdist-quickstart - Fix / rewrite cdist-quickstart
- write tutorial!!!!!!!!!
- like ccollect!
- include ssh control master!
- add local/ hint (and add to git)
- add hint for ssh StrictHostKeyChecking no
- and that ssh will wait for answer of prompt
- nasty if used in parallel mode (scroll up!)
- rewrite cdist-stages, remove
- update man7!
- exec flag is not true for manifest anymore
SSH HINTS
---------
Control master, ssh agent
Everything you specify in manifests
# Intro of quickstart
#
cat << eof
$banner cdist version $__cdist_version
Welcome to the interactive guide to cdist!
This is the interactive tutorial and beginners help for cdist and here's
our schedule:
- Stages: How cdist operates
- Explorer: Explore facts of the target host
- Manifest: Map configurations to hosts
- Types: Bundled functionality
- Deploy a configuration to the local host!
eof
__prompt "$continue"
################################################################################
# Stages
#
cat << eof
To deploy configurations to a host, you call
cdist-deploy-to <hostname>
which makes calls to other scripts, which realise the so called "stages".
Usually you'll not notice this, but in case you want to debug or hack cdist,
you can run each stage on its own. Besides that, you just need to remember
that the command cdist-deploy-to is the main cdist command.
See also:
Source of cdist-deploy-to(1), cdist-stages(7)
eof
__prompt "$continue"
################################################################################
# Explorer
#
cat << eof
The first thing cdist always does is running different explorers on the
target host. The explorers can be found in the directory
${__cdist_explorer_dir}
An explorer is executed on the target host and its output is saved to a file.
You can use these files later to decide what or how to configure the host.
For a demonstration, we'll call the OS explorer locally now, but remember:
This is only for demonstration, normally it is run on the target host.
The os explorer will which either displays the detected operating system or
nothing if it does not know your OS.
See also:
cdist-explorer(7)
eof
explorer="${__cdist_explorer_dir}/os"
__prompt "Press enter to execute $explorer"
set -x
"$explorer"
set +x
################################################################################
# Manifest
#
cat << eof
The initial manifest is the entry point for cdist to find out, what you would
like to have configured. It is located at
${__cdist_manifest_init}
And can be as simple as
--------------------------------------------------------------------------------
__file /etc/cdist-configured --type file
--------------------------------------------------------------------------------
See also:
cdist-manifest(7)
eof
__prompt "$continue"
cat << eof
Let's take a deeper look at the initial manifest to understand what it means:
__file /etc/cdist-configured --type file
| | | \\
| | The parameter type \\ With the value file
| |
| |
| | This is the object id
|
__file is a so called "type"
This essentially looks like a standard command executed in the shell.
eof
__prompt "$continue"
cat << eof
And that's exactly true. Manifests are shell snippets that can use
types as commands with arguments. cdist prepends a special path
that contain links to the cdist-type-emulator, to \$PATH, so you
can use your types as a command.
This is also the reason why types should always be prefixed with
"__", to prevent collisions with existing binaries.
The object id is unique per type and used to prevent you from creating
the same object twice.
Parameters are type specific and are always specified as --parameter <value>.
See also:
cdist-type-build-emulation(1), cdist-type-emulator(1)
eof
__prompt "$continue"
################################################################################
# Types
#
cat << eof
Types are bundled functionality and are the main component of cdist.
If you want to have a feature x, you write the type __x. Types are stored in
${__cdist_type_dir}
And cdist ships with some types already!
See also:
cdist-type(7)
eof
__prompt "Press enter to see available types"
set -x
ls ${__cdist_type_dir}
set +x
cat << eof
Types consist of the following parts:
- ${__cdist_name_parameter} (${__cdist_name_parameter_required}/${__cdist_name_parameter_optional}
- ${__cdist_name_manifest}
- ${__cdist_name_explorer}
- ${__cdist_name_gencode}
eof
__prompt "$continue"
cat << eof
Every type must have a directory named ${__cdist_name_parameter}, which
contains required or optional parameters (in newline seperated files).
If an object of a specific type was created in the initial manifest,
the manifest of the type is run and may create other objects.
A type may have ${__cdist_name_explorer}, which are very similar to the
${__cdist_name_explorer} seen above, but with a different purpose:
They are specific to the type and are not relevant for other types.
You may use them for instance to find out details on the target host,
so you can decide what to do on the target host eventually.
After the ${__cdist_name_manifest} and the ${__cdist_name_explorer} of
a type have been run, ${__cdist_name_gencode} is executed, which creates
code to be executed on the target on stdout.
eof
__prompt "$continue"
################################################################################
# Deployment
#
cat << eof
Now you've got some basic knowledge about cdist, let's configure your a host!
Ensure that you have a ssh server running on the host and that you can login as root.
eof
__prompt "Enter hostname or press enter for localhost: "
if [ "$answer" ]; then
host="$answer"
else
host="localhost"
fi
manifestinit="conf/manifest/init"
cat << eof
I'll now setup $manifestinit, containing the following code:
--------------------------------------------------------------------------------
# Every machine becomes a marker, so sysadmins know that automatic
# configurations are happening
__file /etc/cdist-configured
case "\$__target_host" in
$host)
__link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic
__addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist"
;;
esac
--------------------------------------------------------------------------------
WARNING: This will overwrite ${manifestinit}.
eof
cat > "$__cdist_abs_mydir/../$manifestinit" << eof
# Every machine becomes a marker, so sysadmins know that automatic
# configurations are happening
__file /etc/cdist-configured
case "\$__target_host" in
$host)
__link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic
__addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist"
;;
esac
eof
chmod u+x "$__cdist_abs_mydir/../$manifestinit"
cmd="cdist-deploy-to $host"
__prompt "Press enter to run \"$cmd\""
# No quotes, we need field splitting
$cmd
################################################################################
# End
#
cat << eof
--------------------------------------------------------------------------------
That's it, this is the end of the cdist-quickstart.
I hope you've got some impression on how cdist works, here are again some
pointers on where to continue to read:
eof
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
- Initial install support - Initial install support
@ -33,11 +311,13 @@
via __global/ via __global/
- Support parallel execution - Support parallel execution
- and maximum number of parallel runs (-p X)
- error handling / report failed hosts - error handling / report failed hosts
- Allow manifest to be read from stdin
- Create new video for cdist 2.0.0 - Create new video for cdist 2.0.0
http://www.youtube.com/watch?v=PRMjzy48eTI http://www.youtube.com/watch?v=PRMjzy48eTI
- Setup __debug, if -d is given, so other tools can reuse it - Setup __debug, if -d is given, so other tools can reuse it
- implement everywhere to external!
- remote_prefix:
scp vs. ssh issue

15
doc/man/cdist-reference.text.sh
View file

@ -154,8 +154,23 @@ done
cat << eof cat << eof
OBJECTS
-------
For object to object communication and tests, the following paths are
usable within a object directory:
changed::
This empty file exists in an object directory, if the object has
code to be excuted (either remote or local)
ENVIRONMENT VARIABLES ENVIRONMENT VARIABLES
--------------------- ---------------------
__debug::
If this variable is setup, cdist runs in debug mode.
You can use this information, to only output stuff in debug
mode as well.
Available for: initial manifest, type manifest
__explorer:: __explorer::
Directory that contains all global explorers. Directory that contains all global explorers.
Available for: explorer Available for: explorer

30
doc/man/man1/cdist-deploy-stdin-to.text
View file

@ -1,30 +0,0 @@
cdist-deploy-stdin-to(1)
========================
Steven Armstrong <steven-cdist--@--armstrong.cc>
NAME
----
cdist-deploy-stdin-to - Deploy the configuration given on stdin to host
SYNOPSIS
--------
echo "__file /tmp/whatever" | cdist-deploy-stdin-to HOSTNAME
DESCRIPTION
-----------
Use stdin as the manifest for cdist-deploy-to.
SEE ALSO
--------
- cdist(7)
- cdist-deploy-to(1)
COPYING
-------
Copyright \(C) 2011 Steven Armstrong. Free use of this software is
granted under the terms of the GNU General Public License version 3 (GPLv3).

2
doc/man/man7/cdist-explorer.text
View file

@ -26,7 +26,7 @@ $__explorer/<explorer_name> (general and type explorer) or
$__type_explorer/<explorer name> (type explorer). $__type_explorer/<explorer name> (type explorer).
In case of significant errors, the explorer may exit non-zero and return an In case of significant errors, the explorer may exit non-zero and return an
error message on stderr, which will cause the cdist run to abort. error message on stderr, which will cause cdist to abort.
You can also use stderr for debugging purposes while developing a new You can also use stderr for debugging purposes while developing a new
explorer. explorer.

14
doc/man/man7/cdist-hacker.text
View file

@ -26,26 +26,12 @@ in the latest version, drop a mail to the cdist mailing list,
subject prefixed with "[BUG] ". subject prefixed with "[BUG] ".
UNDERSTANDING CDIST INTERNALS
-----------------------------
IF you are interested in how cdist internally works, you can open
bin/cdist-config and bin/cdist-deploy-to in your favorite editor and
read the scripts bin/cdist-deploy-to calls. The magnificent HACKERS_README
may be of great help as well.
CODING CONVENTIONS (EVERYWHERE) CODING CONVENTIONS (EVERYWHERE)
------------------------------- -------------------------------
If something should be better done or needs to fixed, add the word FIXME If something should be better done or needs to fixed, add the word FIXME
nearby, so grepping for FIXME gives all positions that need to be fixed. nearby, so grepping for FIXME gives all positions that need to be fixed.
CODING CONVENTIONS (CORE)
-------------------------
- All variables exported by cdist are prefixed with a double underscore (__)
- All cdist-internal variables are prefixed with __cdist_ and are generally not exported.
HOW TO SUBMIT STUFF FOR INCLUSION INTO UPSTREAM CDIST HOW TO SUBMIT STUFF FOR INCLUSION INTO UPSTREAM CDIST
----------------------------------------------------- -----------------------------------------------------
If you did some cool changes to cdist, which you value as a benefit for If you did some cool changes to cdist, which you value as a benefit for

310
doc/man/man7/cdist-quickstart.text
View file

@ -1,310 +0,0 @@
#!/bin/sh
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# Give the user an introduction into cdist
#
. cdist-config
set -eu
banner="cdist-quickstart>"
continue="Press enter to continue or ctrl-c to abort."
create_continue="Press enter to create the described files/directories"
__prompt()
{
echo -n "$banner" "$@"
read answer
}
################################################################################
# Intro of quickstart
#
cat << eof
$banner cdist version $__cdist_version
Welcome to the interactive guide to cdist!
This is the interactive tutorial and beginners help for cdist and here's
our schedule:
- Stages: How cdist operates
- Explorer: Explore facts of the target host
- Manifest: Map configurations to hosts
- Types: Bundled functionality
- Deploy a configuration to the local host!
eof
__prompt "$continue"
################################################################################
# Stages
#
cat << eof
To deploy configurations to a host, you call
cdist-deploy-to <hostname>
which makes calls to other scripts, which realise the so called "stages".
Usually you'll not notice this, but in case you want to debug or hack cdist,
you can run each stage on its own. Besides that, you just need to remember
that the command cdist-deploy-to is the main cdist command.
See also:
Source of cdist-deploy-to(1), cdist-stages(7)
eof
__prompt "$continue"
################################################################################
# Explorer
#
cat << eof
The first thing cdist always does is running different explorers on the
target host. The explorers can be found in the directory
${__cdist_explorer_dir}
An explorer is executed on the target host and its output is saved to a file.
You can use these files later to decide what or how to configure the host.
For a demonstration, we'll call the OS explorer locally now, but remember:
This is only for demonstration, normally it is run on the target host.
The os explorer will which either displays the detected operating system or
nothing if it does not know your OS.
See also:
cdist-explorer(7)
eof
explorer="${__cdist_explorer_dir}/os"
__prompt "Press enter to execute $explorer"
set -x
"$explorer"
set +x
################################################################################
# Manifest
#
cat << eof
The initial manifest is the entry point for cdist to find out, what you would
like to have configured. It is located at
${__cdist_manifest_init}
And can be as simple as
--------------------------------------------------------------------------------
__file /etc/cdist-configured --type file
--------------------------------------------------------------------------------
See also:
cdist-manifest(7)
eof
__prompt "$continue"
cat << eof
Let's take a deeper look at the initial manifest to understand what it means:
__file /etc/cdist-configured --type file
| | | \\
| | The parameter type \\ With the value file
| |
| |
| | This is the object id
|
__file is a so called "type"
This essentially looks like a standard command executed in the shell.
eof
__prompt "$continue"
cat << eof
And that's exactly true. Manifests are shell snippets that can use
types as commands with arguments. cdist prepends a special path
that contain links to the cdist-type-emulator, to \$PATH, so you
can use your types as a command.
This is also the reason why types should always be prefixed with
"__", to prevent collisions with existing binaries.
The object id is unique per type and used to prevent you from creating
the same object twice.
Parameters are type specific and are always specified as --parameter <value>.
See also:
cdist-type-build-emulation(1), cdist-type-emulator(1)
eof
__prompt "$continue"
################################################################################
# Types
#
cat << eof
Types are bundled functionality and are the main component of cdist.
If you want to have a feature x, you write the type __x. Types are stored in
${__cdist_type_dir}
And cdist ships with some types already!
See also:
cdist-type(7)
eof
__prompt "Press enter to see available types"
set -x
ls ${__cdist_type_dir}
set +x
cat << eof
Types consist of the following parts:
- ${__cdist_name_parameter} (${__cdist_name_parameter_required}/${__cdist_name_parameter_optional}
- ${__cdist_name_manifest}
- ${__cdist_name_explorer}
- ${__cdist_name_gencode}
eof
__prompt "$continue"
cat << eof
Every type must have a directory named ${__cdist_name_parameter}, which
contains required or optional parameters (in newline seperated files).
If an object of a specific type was created in the initial manifest,
the manifest of the type is run and may create other objects.
A type may have ${__cdist_name_explorer}, which are very similar to the
${__cdist_name_explorer} seen above, but with a different purpose:
They are specific to the type and are not relevant for other types.
You may use them for instance to find out details on the target host,
so you can decide what to do on the target host eventually.
After the ${__cdist_name_manifest} and the ${__cdist_name_explorer} of
a type have been run, ${__cdist_name_gencode} is executed, which creates
code to be executed on the target on stdout.
eof
__prompt "$continue"
################################################################################
# Deployment
#
cat << eof
Now you've got some basic knowledge about cdist, let's configure your a host!
Ensure that you have a ssh server running on the host and that you can login as root.
eof
__prompt "Enter hostname or press enter for localhost: "
if [ "$answer" ]; then
host="$answer"
else
host="localhost"
fi
manifestinit="conf/manifest/init"
cat << eof
I'll now setup $manifestinit, containing the following code:
--------------------------------------------------------------------------------
# Every machine becomes a marker, so sysadmins know that automatic
# configurations are happening
__file /etc/cdist-configured
case "\$__target_host" in
$host)
__link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic
__addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist"
;;
esac
--------------------------------------------------------------------------------
WARNING: This will overwrite ${manifestinit}.
eof
cat > "$__cdist_abs_mydir/../$manifestinit" << eof
# Every machine becomes a marker, so sysadmins know that automatic
# configurations are happening
__file /etc/cdist-configured
case "\$__target_host" in
$host)
__link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic
__addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist"
;;
esac
eof
chmod u+x "$__cdist_abs_mydir/../$manifestinit"
cmd="cdist-deploy-to $host"
__prompt "Press enter to run \"$cmd\""
# No quotes, we need field splitting
$cmd
################################################################################
# End
#
cat << eof
--------------------------------------------------------------------------------
That's it, this is the end of the cdist-quickstart.
I hope you've got some impression on how cdist works, here are again some
pointers on where to continue to read:
cdist(7), cdist-deploy-to(1), cdist-type(7), cdist-stages(7)
eof

35
doc/man/man7/cdist-stages.text
View file

@ -32,11 +32,6 @@ explorers. Every existing explorer is run on the target and the output of all
explorers are copied back into the local cache. The results can be used by explorers are copied back into the local cache. The results can be used by
manifests and types. manifests and types.
Related documentation:
- cdist-explorer-run-global(1)
- cdist-remote-explorer-run(1)
- cdist-explorer(7)
STAGE 2: RUN THE INITIAL MANIFEST STAGE 2: RUN THE INITIAL MANIFEST
--------------------------------- ---------------------------------
@ -46,11 +41,6 @@ the objects as defined in the manifest for the specific host. In this stage,
no conflicts may occur, i.e. no object of the same type with the same id may no conflicts may occur, i.e. no object of the same type with the same id may
be created. be created.
Related documentation:
- cdist-manifest-run-init(1)
- cdist-manifest-run(1)
- cdist-manifest(7)
STAGE 3: OBJECT INFORMATION RETRIEVAL STAGE 3: OBJECT INFORMATION RETRIEVAL
------------------------------------- -------------------------------------
@ -59,12 +49,6 @@ transfered to the target host and executed. The results are transfered back
and can be used in the following stages to decide what changes need to be made and can be used in the following stages to decide what changes need to be made
on the target to implement the desired state. on the target to implement the desired state.
Related documentation:
- cdist-object-explorer-run(1)
- cdist-remote-explorer-run(1)
- cdist-type(7)
- cdist-explorer(7)
STAGE 4: RUN THE OBJECT MANIFEST STAGE 4: RUN THE OBJECT MANIFEST
-------------------------------- --------------------------------
@ -79,11 +63,6 @@ The newly created objects are merged back into the existing tree. No conflicts
may occur during the merge. A conflict would mean that two different objects may occur during the merge. A conflict would mean that two different objects
try to create the same object, which indicates a broken configuration. try to create the same object, which indicates a broken configuration.
Related documentation:
- cdist-object-manifest-run(1)
- cdist-manifest-run(1)
- cdist-type(7)
STAGE 5: CODE GENERATION STAGE 5: CODE GENERATION
------------------------ ------------------------
@ -92,29 +71,17 @@ gencode scripts. The gencode scripts generate the code to be executed on the
target on stdout. If the gencode executables fail, they must print diagnostic target on stdout. If the gencode executables fail, they must print diagnostic
messages on stderr and exit non-zero. messages on stderr and exit non-zero.
Related documentation:
- cdist-object-gencode-run(1)
- cdist-object-gencode(1)
- cdist-type(7)
STAGE 6: CODE EXECUTION STAGE 6: CODE EXECUTION
----------------------- -----------------------
For every object the resulting code from the previous stage is transferred to For every object the resulting code from the previous stage is transferred to
the target host and executed there to apply the configuration changes. the target host and executed there to apply the configuration changes.
Related documentation:
- cdist-object-code-run(1)
- cdist-code-run(1)
STAGE 7: CACHE STAGE 7: CACHE
-------------- --------------
The cache stores the information from the current run for later use. The cache stores the information from the current run for later use.
Related documentation:
- cdist-cache(1)
SUMMARY SUMMARY
------- -------
@ -126,8 +93,8 @@ in correct order.
SEE ALSO SEE ALSO
-------- --------
- cdist(1)
- cdist(7) - cdist(7)
- cdist-deploy-to(1)
- cdist-reference(7) - cdist-reference(7)

83
doc/man/man7/cdist-tutorial.text Normal file
View file

@ -0,0 +1,83 @@
cdist-tutorial(7)
=================
Nico Schottelius <nico-cdist--@--schottelius.org>
NAME
----
cdist-tutorial - a guided introduction into cdist
INTRODUCTION
------------
This tutorial is aimed at people learning cdist and shows
typical approaches as well as gives an easy start into
the world of configuration management.
This tutorial assumes you are configuring **localhost**, because
it is always available. Just repace **localhost** with your target
host for real life usage.
QUICK START
-----------
For those who just want to configure a system with the
cdist configuration management and do not need (or want)
to understand everything.
Cdist uses **ssh** for communication and transportation
and usually logs into the **target host** as the
**root** user. So you need to configure the **ssh server**
of the target host to allow root logins: Edit
the file **/etc/ssh/sshd_config** and add one of the following
lines:
--------------------------------------------------------------------------------
# Allow login only via public key
PermitRootLogin without-password
# Allow login via password and public key
PermitRootLogin yes
--------------------------------------------------------------------------------
As cdist uses ssh intensively, it is recommended to setup authentication
with public keys:
--------------------------------------------------------------------------------
# Generate pubkey pair as a normal user
ssh-keygen
# Copy pubkey over to target host
ssh-copy-id root@localhost
--------------------------------------------------------------------------------
As soon as you are able to login without passwort to the target host,
we can use cdist, to configure it. You can copy and paste the following
code into your shell to get started and configure localhost:
--------------------------------------------------------------------------------
# Get cdist
git clone git://git.schottelius.org/cdist
# Create manifest (maps configuration to host(s)
cd cdist
echo '__file /etc/cdist-configured' > conf/manifest/init
chmod 0700 conf/manifest/init
# Configure localhost
./bin/cdist config localhost
# Find out that cdist created /etc/cdist-configured
ls -l /etc/cdist-configured
--------------------------------------------------------------------------------
The file 'conf/manifest/init' is usually the entry point for cdist,
to find out what to configure on which host. All manifests are
essentially shell scripts. Every manifest can use the types known to
cdist, which are usually underline prefixed (__).
SEE ALSO
--------
cdist(1), cdist-type(7), cdist-stages(7)

6
doc/man/man7/cdist-type.text
View file

@ -35,10 +35,6 @@ __file /etc/cdist-configured --type file
__package tree --state installed __package tree --state installed
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
Internally cdist-type-emulator(1) will be called from cdist-manifest-run(1) to
save the given parameters into a cconfig database, so they can be accessed by
the manifest and gencode scripts of the type (see below).
A list of supported types can be found in the cdist-reference(7) manpage. A list of supported types can be found in the cdist-reference(7) manpage.
SINGLETON TYPES SINGLETON TYPES
@ -111,7 +107,7 @@ __package_$type "$@"
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
As you can see, the type can reference different environment variables, As you can see, the type can reference different environment variables,
which are documented in cdist-environment-variables(7). which are documented in cdist-reference(7).
Always ensure the manifest is executable, otherwise cdist will not be able Always ensure the manifest is executable, otherwise cdist will not be able
to execute it. to execute it.

5
doc/man/man7/cdist.text
View file

@ -35,12 +35,11 @@ pull mechanism (client requests configuration).
SEE ALSO SEE ALSO
-------- --------
- Website: http://www.nico.schottelius.org/software/cdist/[] - Website: http://www.nico.schottelius.org/software/cdist/[]
- cdist-best-practise(7)
- cdist-deploy-to(1)
- cdist-hacker(7) - cdist-hacker(7)
- cdist-manifest(7) - cdist-manifest(7)
- cdist-quickstart(1)
- cdist-type(7) - cdist-type(7)
- cdist(1)
- cdist(7)
COPYING COPYING

26
lib/cdist/__init__.py Normal file
View file

@ -0,0 +1,26 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
VERSION = "2.0.3"
class Error(Exception):
"""Base exception class for this project"""
pass

46
lib/cdist/banner.py Normal file
View file

@ -0,0 +1,46 @@
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import sys
log = logging.getLogger(__name__)
BANNER = """
.. . .x+=:. s
dF @88> z` ^% :8
'88bu. %8P . <k .88
. '*88888bu . .@8Ned8" :888ooo
.udR88N ^"*8888N .@88u .@^%8888" -*8888888
<888'888k beWE "888L ''888E` x88: `)8b. 8888
9888 'Y" 888E 888E 888E 8888N=*8888 8888
9888 888E 888E 888E %8" R88 8888
9888 888E 888F 888E @8Wou 9% .8888Lu=
?8888u../ .888N..888 888& .888888P` ^%888*
"8888P' `"888*"" R888" ` ^"F 'Y"
"P' "" ""
"""
def banner(args):
"""Guess what :-)"""
print(BANNER)
sys.exit(0)

316
lib/cdist/config.py Normal file
View file

@ -0,0 +1,316 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import datetime
import logging
import os
import stat
import sys
log = logging.getLogger(__name__)
import cdist.emulator
import cdist.path
CODE_HEADER = "#!/bin/sh -e\n"
class Config:
"""Cdist main class to hold arbitrary data"""
def __init__(self, target_host,
initial_manifest=False,
remote_user="root",
home=None,
exec_path=sys.argv[0],
debug=False):
self.target_host = target_host
self.debug = debug
self.remote_user = remote_user
self.exec_path = exec_path
# FIXME: broken - construct elsewhere!
self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host]
self.path = cdist.path.Path(self.target_host,
initial_manifest=initial_manifest,
remote_user=self.remote_user,
remote_prefix=self.remote_prefix,
base_dir=home,
debug=debug)
self.objects_prepared = []
def cleanup(self):
self.path.cleanup()
def run_global_explores(self):
"""Run global explorers"""
log.info("Running global explorers")
explorers = self.path.list_global_explorers()
if(len(explorers) == 0):
raise CdistError("No explorers found in", self.path.global_explorer_dir)
self.path.transfer_global_explorers()
for explorer in explorers:
output = self.path.global_explorer_output_path(explorer)
output_fd = open(output, mode='w')
cmd = []
cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR)
cmd.append(self.path.remote_global_explorer_path(explorer))
cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
output_fd.close()
def run_type_explorer(self, cdist_object):
"""Run type specific explorers for objects"""
type = self.path.get_type_from_object(cdist_object)
self.path.transfer_type_explorers(type)
cmd = []
cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR)
cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type))
cmd.append("__object=" + self.path.remote_object_dir(cdist_object))
cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object))
cmd.append("__object_fq=" + cdist_object)
# Need to transfer at least the parameters for objects to be useful
self.path.transfer_object_parameter(cdist_object)
explorers = self.path.list_type_explorers(type)
for explorer in explorers:
remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)]
output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer)
output_fd = open(output, mode='w')
log.debug("%s exploring %s using %s storing to %s",
cdist_object, explorer, remote_cmd, output)
cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
output_fd.close()
def link_emulator(self):
"""Link emulator to types"""
cdist.emulator.link(self.exec_path,
self.path.bin_dir, self.path.list_types())
def init_deploy(self):
"""Ensure the base directories are cleaned up"""
log.debug("Creating clean directory structure")
self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR)
self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR)
self.link_emulator()
def run_initial_manifest(self):
"""Run the initial manifest"""
log.info("Running initial manifest %s", self.path.initial_manifest)
env = { "__manifest" : self.path.manifest_dir }
self.run_manifest(self.path.initial_manifest, extra_env=env)
def run_type_manifest(self, cdist_object):
"""Run manifest for a specific object"""
type = self.path.get_type_from_object(cdist_object)
manifest = self.path.type_dir(type, "manifest")
log.debug("%s: Running %s", cdist_object, manifest)
if os.path.exists(manifest):
env = { "__object" : self.path.object_dir(cdist_object),
"__object_id": self.path.get_object_id_from_object(cdist_object),
"__object_fq": cdist_object,
"__type": self.path.type_dir(type)
}
self.run_manifest(manifest, extra_env=env)
def run_manifest(self, manifest, extra_env=None):
"""Run a manifest"""
log.debug("Running manifest %s, env=%s", manifest, extra_env)
env = os.environ.copy()
env['PATH'] = self.path.bin_dir + ":" + env['PATH']
# Information required in every manifest
env['__target_host'] = self.target_host
env['__global'] = self.path.out_dir
# Submit debug flag to manifest, can be used by emulator and types
if self.debug:
env['__debug'] = "yes"
# Required for recording source
env['__cdist_manifest'] = manifest
# Required to find types
env['__cdist_type_base_dir'] = self.path.type_base_dir
# Other environment stuff
if extra_env:
env.update(extra_env)
cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env)
def object_run(self, cdist_object, mode):
"""Run gencode or code for an object"""
log.debug("Running %s from %s", mode, cdist_object)
file=os.path.join(self.path.object_dir(cdist_object), "require")
requirements = cdist.path.file_to_list(file)
type = self.path.get_type_from_object(cdist_object)
for requirement in requirements:
log.debug("Object %s requires %s", cdist_object, requirement)
self.object_run(requirement, mode=mode)
#
# Setup env Variable:
#
env = os.environ.copy()
env['__target_host'] = self.target_host
env['__global'] = self.path.out_dir
env["__object"] = self.path.object_dir(cdist_object)
env["__object_id"] = self.path.get_object_id_from_object(cdist_object)
env["__object_fq"] = cdist_object
env["__type"] = self.path.type_dir(type)
if mode == "gencode":
paths = [
self.path.type_dir(type, "gencode-local"),
self.path.type_dir(type, "gencode-remote")
]
for bin in paths:
if os.path.isfile(bin):
# omit "gen" from gencode and use it for output base
outfile=os.path.join(self.path.object_dir(cdist_object),
os.path.basename(bin)[3:])
outfile_fd = open(outfile, "w")
# Need to flush to ensure our write is done before stdout write
outfile_fd.write(CODE_HEADER)
outfile_fd.flush()
cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd)
outfile_fd.close()
status = os.stat(outfile)
# Remove output if empty, else make it executable
if status.st_size == len(CODE_HEADER):
os.unlink(outfile)
else:
# Add header and make executable - identically to 0o700
os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
# Mark object as changed
open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close()
if mode == "code":
local_dir = self.path.object_dir(cdist_object)
remote_dir = self.path.remote_object_dir(cdist_object)
bin = os.path.join(local_dir, "code-local")
if os.path.isfile(bin):
cdist.exec.run_or_fail([bin])
local_remote_code = os.path.join(local_dir, "code-remote")
remote_remote_code = os.path.join(remote_dir, "code-remote")
if os.path.isfile(local_remote_code):
self.path.transfer_file(local_remote_code, remote_remote_code)
# FIXME: remote_prefix
cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix)
def stage_prepare(self):
"""Do everything for a deploy, minus the actual code stage"""
self.init_deploy()
self.run_global_explores()
self.run_initial_manifest()
log.info("Running object manifests and type explorers")
old_objects = []
objects = self.path.list_objects()
# Continue process until no new objects are created anymore
while old_objects != objects:
old_objects = list(objects)
for cdist_object in objects:
if cdist_object in self.objects_prepared:
log.debug("Skipping rerun of object %s", cdist_object)
continue
else:
self.run_type_explorer(cdist_object)
self.run_type_manifest(cdist_object)
self.objects_prepared.append(cdist_object)
objects = self.path.list_objects()
def stage_run(self):
"""The final (and real) step of deployment"""
log.info("Generating and executing code")
# Now do the final steps over the existing objects
for cdist_object in self.path.list_objects():
log.debug("Run object: %s", cdist_object)
self.object_run(cdist_object, mode="gencode")
self.object_run(cdist_object, mode="code")
def deploy_to(self):
"""Mimic the old deploy to: Deploy to one host"""
log.info("Deploying to " + self.target_host)
time_start = datetime.datetime.now()
self.stage_prepare()
self.stage_run()
time_end = datetime.datetime.now()
duration = time_end - time_start
log.info("Finished run of %s in %s seconds",
self.target_host,
duration.total_seconds())
def deploy_and_cleanup(self):
"""Do what is most often done: deploy & cleanup"""
self.deploy_to()
self.cleanup()
def config(args):
"""Configure remote system"""
process = {}
time_start = datetime.datetime.now()
for host in args.host:
c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug)
if args.parallel:
log.debug("Creating child process for %s", host)
process[host] = multiprocessing.Process(target=c.deploy_and_cleanup)
process[host].start()
else:
c.deploy_and_cleanup()
if args.parallel:
for p in process.keys():
log.debug("Joining process %s", p)
process[p].join()
time_end = datetime.datetime.now()
log.info("Total processing time for %s host(s): %s", len(args.host),
(time_end - time_start).total_seconds())

151
lib/cdist/emulator.py Normal file
View file

@ -0,0 +1,151 @@
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import argparse
import logging
import os
import cdist
import cdist.path
log = logging.getLogger(__name__)
def run(argv):
"""Emulate type commands (i.e. __file and co)"""
type = os.path.basename(argv[0])
type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type)
param_dir = os.path.join(type_dir, "parameter")
global_dir = os.environ['__global']
object_source = os.environ['__cdist_manifest']
if '__debug' in os.environ:
logging.root.setLevel(logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser(add_help=False)
for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")):
argument = "--" + parameter
parser.add_argument(argument, action='store', required=False)
for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")):
argument = "--" + parameter
parser.add_argument(argument, action='store', required=True)
# If not singleton support one positional parameter
if not os.path.isfile(os.path.join(type_dir, "singleton")):
parser.add_argument("object_id", nargs=1)
# And finally verify parameter
args = parser.parse_args(argv[1:])
# Setup object_id
if os.path.isfile(os.path.join(type_dir, "singleton")):
object_id = "singleton"
else:
object_id = args.object_id[0]
del args.object_id
# FIXME: / hardcoded - better portable solution available?
if object_id[0] == '/':
object_id = object_id[1:]
# Prefix output by object_self
logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s'
logging.basicConfig(format=logformat)
# FIXME: verify object id
log.debug(args)
object_dir = os.path.join(global_dir, "object", type,
object_id, cdist.path.DOT_CDIST)
param_out_dir = os.path.join(object_dir, "parameter")
object_source_file = os.path.join(object_dir, "source")
if os.path.exists(param_out_dir):
object_exists = True
old_object_source_fd = open(object_source_file, "r")
old_object_source = old_object_source_fd.readlines()
old_object_source_fd.close()
else:
object_exists = False
try:
os.makedirs(param_out_dir, exist_ok=True)
except OSError as error:
raise cdist.Error(param_out_dir + ": " + error.args[1])
# Record parameter
params = vars(args)
for param in params:
value = getattr(args, param)
if value:
file = os.path.join(param_out_dir, param)
log.debug(file + "<-" + param + " = " + value)
# Already exists, verify all parameter are the same
if object_exists:
if not os.path.isfile(file):
raise cdist.Error("New parameter \"" +
param + "\" specified, aborting\n" +
"Source = " +
" ".join(old_object_source)
+ " new =" + object_source)
else:
param_fd = open(file, "r")
value_old = param_fd.readlines()
param_fd.close()
if(value_old[0] != value):
raise cdist.Error("Parameter\"" + param +
"\" differs: " + " ".join(value_old) + " vs. " +
value +
"\nSource = " + " ".join(old_object_source)
+ " new = " + object_source)
else:
param_fd = open(file, "w")
param_fd.writelines(value)
param_fd.close()
# Record requirements
if "__require" in os.environ:
requirements = os.environ['__require']
log.debug(object_id + ":Writing requirements: " + requirements)
require_fd = open(os.path.join(object_dir, "require"), "a")
require_fd.writelines(requirements.split(" "))
require_fd.close()
# Record / Append source
source_fd = open(os.path.join(object_dir, "source"), "a")
source_fd.writelines(object_source)
source_fd.close()
log.debug("Finished " + type + "/" + object_id + repr(params))
def link(exec_path, bin_dir, type_list):
"""Link type names to cdist-type-emulator"""
source = os.path.abspath(exec_path)
for type in type_list:
destination = os.path.join(bin_dir, type)
log.debug("Linking %s to %s", source, destination)
os.symlink(source, destination)

71
lib/cdist/exec.py Normal file
View file

@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import subprocess
log = logging.getLogger(__name__)
import cdist
def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs):
# Manually execute /bin/sh, because sh -e does what we want
# and sh -c -e does not exit if /bin/false called
args[0][:0] = [ "/bin/sh", "-e" ]
if remote_prefix:
args[0][:0] = remote_prefix
log.debug("Shell exec cmd: %s", args)
if 'env' in kargs:
log.debug("Shell exec env: %s", kargs['env'])
try:
subprocess.check_call(*args, **kargs)
except subprocess.CalledProcessError:
log.error("Code that raised the error:\n")
if remote_prefix:
run_or_fail(["cat", script], remote_prefix=remote_prefix)
else:
try:
script_fd = open(script)
print(script_fd.read())
script_fd.close()
except IOError as error:
raise cdist.Error(str(error))
raise cdist.Error("Command failed (shell): " + " ".join(*args))
except OSError as error:
raise cdist.Error(" ".join(*args) + ": " + error.args[1])
def run_or_fail(*args, remote_prefix=False, **kargs):
if remote_prefix:
args[0][:0] = remote_prefix
log.debug("Exec: " + " ".join(*args))
try:
subprocess.check_call(*args, **kargs)
except subprocess.CalledProcessError:
raise cdist.Error("Command failed: " + " ".join(*args))
except OSError as error:
raise cdist.Error(" ".join(*args) + ": " + error.args[1])

22
bin/cdist-deploy-stdin-to → lib/cdist/install.py Executable file → Normal file
View file

@ -1,6 +1,7 @@
#!/bin/sh #!/usr/bin/env python3
# -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -18,19 +19,12 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>. # along with cdist. If not, see <http://www.gnu.org/licenses/>.
# #
# #
# Use stdin as the manifest to deploy on the given host.
#
. cdist-config import logging
[ $# -eq 1 ] || __cdist_usage "<target host>"
set -eu
__cdist_target_host="$1" log = logging.getLogger(__name__)
shift
cat >> "$__cdist_tmp_file" def install(args):
"""Install remote system"""
process = {}
chmod +x "$__cdist_tmp_file"
export __cdist_manifest_init="$__cdist_tmp_file"
cdist-deploy-to "$__cdist_target_host"

276
lib/cdist/path.py Normal file
View file

@ -0,0 +1,276 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import os
import shutil
import sys
import tempfile
# Hardcoded paths usually not changable
REMOTE_BASE_DIR = "/var/lib/cdist"
REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf")
REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object")
REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type")
REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer")
DOT_CDIST = ".cdist"
log = logging.getLogger(__name__)
import cdist.exec
def file_to_list(filename):
"""Return list from \n seperated file"""
if os.path.isfile(filename):
file_fd = open(filename, "r")
lines = file_fd.readlines()
file_fd.close()
# Remove \n from all lines
lines = map(lambda s: s.strip(), lines)
else:
lines = []
return lines
class Path:
"""Class that handles path related configurations"""
def __init__(self,
target_host,
remote_user,
remote_prefix,
initial_manifest=False,
base_dir=None,
debug=False):
# Base and Temp Base
if base_dir:
self.base_dir = base_dir
else:
self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
self.temp_dir = tempfile.mkdtemp()
self.target_host = target_host
self.remote_user = remote_user
self.remote_prefix = remote_prefix
self.conf_dir = os.path.join(self.base_dir, "conf")
self.cache_base_dir = os.path.join(self.base_dir, "cache")
self.cache_dir = os.path.join(self.cache_base_dir, target_host)
self.global_explorer_dir = os.path.join(self.conf_dir, "explorer")
self.lib_dir = os.path.join(self.base_dir, "lib")
self.manifest_dir = os.path.join(self.conf_dir, "manifest")
self.type_base_dir = os.path.join(self.conf_dir, "type")
self.out_dir = os.path.join(self.temp_dir, "out")
os.mkdir(self.out_dir)
self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer")
os.mkdir(self.global_explorer_out_dir)
self.object_base_dir = os.path.join(self.out_dir, "object")
# Setup binary directory + contents
self.bin_dir = os.path.join(self.out_dir, "bin")
os.mkdir(self.bin_dir)
# List of type explorers transferred
self.type_explorers_transferred = {}
# objects
self.objects_prepared = []
# Mostly static, but can be overwritten on user demand
if initial_manifest:
self.initial_manifest = initial_manifest
else:
self.initial_manifest = os.path.join(self.manifest_dir, "init")
def cleanup(self):
# Do not use in __del__:
# http://docs.python.org/reference/datamodel.html#customization
# "other globals referenced by the __del__() method may already have been deleted
# or in the process of being torn down (e.g. the import machinery shutting down)"
#
log.debug("Saving" + self.temp_dir + "to " + self.cache_dir)
# Remove previous cache
if os.path.exists(self.cache_dir):
shutil.rmtree(self.cache_dir)
shutil.move(self.temp_dir, self.cache_dir)
def remote_mkdir(self, directory):
"""Create directory on remote side"""
cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix)
def remove_remote_dir(self, destination):
cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix)
def transfer_dir(self, source, destination):
"""Transfer directory and previously delete the remote destination"""
self.remove_remote_dir(destination)
cdist.exec.run_or_fail(["scp", "-qr", source,
self.remote_user + "@" +
self.target_host + ":" +
destination])
def transfer_file(self, source, destination):
"""Transfer file"""
cdist.exec.run_or_fail(["scp", "-q", source,
self.remote_user + "@" +
self.target_host + ":" +
destination])
def global_explorer_output_path(self, explorer):
"""Returns path of the output for a global explorer"""
return os.path.join(self.global_explorer_out_dir, explorer)
def type_explorer_output_dir(self, cdist_object):
"""Returns and creates dir of the output for a type explorer"""
dir = os.path.join(self.object_dir(cdist_object), "explorer")
if not os.path.isdir(dir):
os.mkdir(dir)
return dir
def remote_global_explorer_path(self, explorer):
"""Returns path to the remote explorer"""
return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer)
def list_global_explorers(self):
"""Return list of available explorers"""
return os.listdir(self.global_explorer_dir)
def list_type_explorers(self, type):
"""Return list of available explorers for a specific type"""
dir = self.type_dir(type, "explorer")
if os.path.isdir(dir):
list = os.listdir(dir)
else:
list = []
log.debug("Explorers for %s in %s: %s", type, dir, list)
return list
def list_types(self):
return os.listdir(self.type_base_dir)
def list_object_paths(self, starting_point):
"""Return list of paths of existing objects"""
object_paths = []
for content in os.listdir(starting_point):
full_path = os.path.join(starting_point, content)
if os.path.isdir(full_path):
object_paths.extend(self.list_object_paths(starting_point = full_path))
# Directory contains .cdist -> is an object
if content == DOT_CDIST:
object_paths.append(starting_point)
return object_paths
# FIXME
def get_type_from_object(self, cdist_object):
"""Returns the first part (i.e. type) of an object"""
return cdist_object.split(os.sep)[0]
def get_object_id_from_object(self, cdist_object):
"""Returns everything but the first part (i.e. object_id) of an object"""
return os.sep.join(cdist_object.split(os.sep)[1:])
def object_dir(self, cdist_object):
"""Returns the full path to the object (including .cdist)"""
return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST)
def remote_object_dir(self, cdist_object):
"""Returns the remote full path to the object (including .cdist)"""
return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST)
def object_parameter_dir(self, cdist_object):
"""Returns the dir to the object parameter"""
return os.path.join(self.object_dir(cdist_object), "parameter")
def remote_object_parameter_dir(self, cdist_object):
"""Returns the remote dir to the object parameter"""
return os.path.join(self.remote_object_dir(cdist_object), "parameter")
def object_code_paths(self, cdist_object):
"""Return paths to code scripts of object"""
return [os.path.join(self.object_dir(cdist_object), "code-local"),
os.path.join(self.object_dir(cdist_object), "code-remote")]
def list_objects(self):
"""Return list of existing objects"""
objects = []
if os.path.isdir(self.object_base_dir):
object_paths = self.list_object_paths(self.object_base_dir)
for path in object_paths:
objects.append(os.path.relpath(path, self.object_base_dir))
return objects
def type_dir(self, type, *args):
"""Return directory the type"""
return os.path.join(self.type_base_dir, type, *args)
def remote_type_explorer_dir(self, type):
"""Return remote directory that holds the explorers of a type"""
return os.path.join(REMOTE_TYPE_DIR, type, "explorer")
def transfer_object_parameter(self, cdist_object):
"""Transfer the object parameter to the remote destination"""
# Create base path before using mkdir -p
self.remote_mkdir(self.remote_object_parameter_dir(cdist_object))
# Synchronise parameter dir afterwards
self.transfer_dir(self.object_parameter_dir(cdist_object),
self.remote_object_parameter_dir(cdist_object))
def transfer_global_explorers(self):
"""Transfer the global explorers"""
self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR)
self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR)
def transfer_type_explorers(self, type):
"""Transfer explorers of a type, but only once"""
if type in self.type_explorers_transferred:
log.debug("Skipping retransfer for explorers of %s", type)
return
else:
# Do not retransfer
self.type_explorers_transferred[type] = 1
src = self.type_dir(type, "explorer")
remote_base = os.path.join(REMOTE_TYPE_DIR, type)
dst = self.remote_type_explorer_dir(type)
# Only continue, if there is at least the directory
if os.path.isdir(src):
# Ensure that the path exists
self.remote_mkdir(remote_base)
self.transfer_dir(src, dst)

171
test.py Executable file
View file

@ -0,0 +1,171 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import sys
import shutil
import subprocess
import tempfile
import unittest
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')))
cdist_exec_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist"))
cdist_commands=["banner", "config", "install"]
import cdist
import cdist.config
import cdist.exec
class Exec(unittest.TestCase):
def setUp(self):
"""Create shell code and co."""
self.temp_dir = tempfile.mkdtemp()
self.shell_false = os.path.join(self.temp_dir, "shell_false")
self.shell_true = os.path.join(self.temp_dir, "shell_true")
true_fd = open(self.shell_true, "w")
true_fd.writelines(["#!/bin/sh\n", "/bin/true"])
true_fd.close()
false_fd = open(self.shell_false, "w")
false_fd.writelines(["#!/bin/sh\n", "/bin/false"])
false_fd.close()
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_local_success_shell(self):
try:
cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail_shell(self):
self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail,
self.shell_false, [self.shell_false])
def test_local_success(self):
try:
cdist.exec.run_or_fail(["/bin/true"])
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
def test_local_fail(self):
self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"])
class Config(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
self.init_manifest = os.path.join(self.temp_dir, "manifest")
self.config = cdist.config.Config("localhost",
initial_manifest=self.init_manifest,
exec_path=cdist_exec_path)
self.config.link_emulator()
def test_initial_manifest_different_parameter(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + " --mode 0700\n",
"__file " + self.temp_dir + " --mode 0600\n",
])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_parameter_added(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + '\n',
"__file " + self.temp_dir + " --mode 0600\n",
])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_parameter_removed(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + " --mode 0600\n",
"__file " + self.temp_dir + "\n",
])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_non_existent_command(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"thereisdefinitelynosuchcommend"])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_parameter_twice(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + " --mode 0600\n",
"__file " + self.temp_dir + " --mode 0600\n",
])
manifest_fd.close()
try:
print("a")
self.config.run_initial_manifest()
print("b")
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)
class UI(unittest.TestCase):
def test_banner(self):
self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0)
def test_help(self):
for cmd in cdist_commands:
self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0)
# FIXME: mockup needed
def test_config_localhost(self):
for cmd in cdist_commands:
self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0)
if __name__ == '__main__':
unittest.main()