Merge remote-tracking branch 'steven/oo-restructure'

Conflicts:
	lib/cdist/config.py
	lib/cdist/path.py

Signed-off-by: Nico Schottelius <nico@kr.ethz.ch>
This commit is contained in:
Nico Schottelius 2011-10-06 16:57:48 +02:00
commit efe8dbfa85
29 changed files with 1302 additions and 54 deletions

View file

@ -127,11 +127,18 @@ case "$1" in
;;
test)
python3 -m unittest discover test 'test_*.py'
PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \
python3 -m cdist.test
;;
test-install)
PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \
python3 -m unittest cdist.test.test_install
;;
test-all)
python3 -m unittest discover test '*.py'
PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \
python3 -m unittest discover lib/cdist/test '*.py'
;;
*)

38
conf/type/__mkfs/gencode-remote Executable file
View file

@ -0,0 +1,38 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
device="$(cat "$__object/parameter/device")"
type="$(cat "$__object/parameter/type")"
if [ "$type" = "swap" ]; then
echo "mkswap $device"
else
command="mkfs -t $type -q"
if [ -f "$__object/parameter/options" ]; then
options="$(cat "$__object/parameter/options")"
command="$command $options"
fi
command="$command $device"
if [ -f "$__object/parameter/blocks" ]; then
blocks="$(cat "$__object/parameter/blocks")"
command="$command $blocks"
fi
echo "$command"
fi

0
conf/type/__mkfs/install Normal file
View file

57
conf/type/__mkfs/man.text Normal file
View file

@ -0,0 +1,57 @@
cdist-type__mkfs(7)
===================
Steven Armstrong <steven-cdist--@--armstrong.cc>
NAME
----
cdist-type__mkfs - build a linux file system
DESCRIPTION
-----------
This cdist type is a wrapper for the mkfs command.
REQUIRED PARAMETERS
-------------------
type::
The filesystem type to use. Same as mkfs -t.
OPTIONAL PARAMETERS
-------------------
device::
defaults to object_id
options::
file system-specific options to be passed to the mkfs command
blocks::
the number of blocks to be used for the file system
EXAMPLES
--------
--------------------------------------------------------------------------------
# reiserfs /dev/sda5
__mkfs /dev/sda5 --type reiserfs
# same thing with explicit device
__mkfs whatever --device /dev/sda5 --type reiserfs
# jfs with journal on /dev/sda2
__mkfs /dev/sda1 --type jfs --options "-j /dev/sda2"
--------------------------------------------------------------------------------
SEE ALSO
--------
- cdist-type(7)
- mkfs(8)
COPYING
-------
Copyright \(C) 2011 Steven Armstrong. Free use of this software is
granted under the terms of the GNU General Public License version 3 (GPLv3).

31
conf/type/__mkfs/manifest Executable file
View file

@ -0,0 +1,31 @@
#!/bin/sh
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
# set defaults
if [ -f "$__object/parameter/device" ]; then
device="(cat "$__object/parameter/device")"
else
device="/$__object_id"
echo "$device" > "$__object/parameter/device"
fi
type="(cat "$__object/parameter/type")"
options="(cat "$__object/parameter/options")"

View file

@ -0,0 +1,3 @@
device
options
blocks

View file

@ -0,0 +1 @@
type

View file

View file

@ -3,7 +3,8 @@ die() {
exit 1
}
debug() {
echo "[__partition_msdos_apply] $@" >&2
#echo "[__partition_msdos_apply] $@" >&2
:
}
fdisk_command() {
@ -51,7 +52,7 @@ create_partition() {
first_minor="${minor}\n"
type_minor="${minor}\n"
primary_extended="l\n"
[ "$primary_count" > "3" ] && primary_extended=""
[ "$primary_count" -gt "3" ] && primary_extended=""
fi
[ -n "${size}" ] && size="+${size}M"
fdisk_command ${device} "n\n${primary_extended}${first_minor}\n${size}\nt\n${type_minor}${type}\n"

3
doc/dev/logs/2011-10-04 Normal file
View file

@ -0,0 +1,3 @@
Testing for single tests:
PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib python3 -m unittest cdist.test.test_install.Install.test_explorer_ran

116
doc/dev/logs/2011-10-05 Normal file
View file

@ -0,0 +1,116 @@
Config/Install/Deploy/Run:
target host
remote_cmd_prefix - ssh user@bla sudo foo?????
remote_cp_prefix - cp statt scp oder so
debug -> env für alles += __debug
Storage/Metaobject/Tree? == Path?
base_dir?
nimmt objekte
Sammelt Objekte
Ist prepared hier?
Object
"Infos" / Datenhalde
Base_Dir-Abhängigkeit? - wo
out_dir - wo speichern
nur eigenes verzeichnis interessant?
-> nicht für shell code / aka gencode!
-> __global abhängigkeit
object.gencode()?
hast du type-explorer?
ja?
führe JEDEN remote aus
speichere ausgabe in object
nein:
fertig
hast du gencode-{local,remote}?
ja?
führe local oder remote aus
speichere ausgabe in s/^gen//
nein:
fertig
hast du code-{local,remote}?
ja?
führe local oder remote aus
nein:
fertig
ich habe ...
object_id
type
type.singleton() == False -> require object_id
parameter gegeben
requirements / order
type_explorer := methode zum ausführen?
cdist.object.Object(type, id)
methoden:
gen_code
code
run_manifest
manifest == ort
Type
singleton: ja / nein
install: ja / nein
type_explorer := liste
optional_parameter
required_parameter
TypeExplorer
verwandt oder == explorer
Verwandschaft klären!
sehr abhängig von base_dir!</behauptung :)>
- welche gibt es?
- was für optionen haben sie
cdist.type.Type("/path/to/type")
Tree/Path vieh, das liste von $_ speichert
Einfach iterieren
Explorer
execute(env)
env == __explorer -> nur im explorer
z.B. BaseExplorer oder andersherum GlobalExplorer
Manifest
Exec
wrapper um auszuführen,
error handling,
output redirection (variable, file, beides, socat :-)
--------------------------------------------------------------------------------
- base_dir (conf/, type, ...)
- manifest (initiale)
$methode_mit_inhalt_von_manifest?
run_manifest(code)
ob sinnvoll?
geht auch mit stdin oder datei
stdin -> muss in tmp-datei, für sh -e?
-
--------------------------------------------------------------------------------
save output of shell in buffer instead of displaying?
-> freedom to decide whether to display or not!

View file

@ -24,3 +24,13 @@ VERSION = "2.0.3"
class Error(Exception):
"""Base exception class for this project"""
pass
class MissingEnvironmentVariableError(Error):
"""Raised when a required environment variable is not set."""
def __init__(self, name)
self.name = name
def __str__(self):
return 'Missing required environment variable: {0.name}'.format(o)

View file

@ -22,10 +22,6 @@
import datetime
import logging
import os
import stat
import sys
log = logging.getLogger(__name__)
import cdist.emulator
@ -280,10 +276,8 @@ class Config:
self.target_host,
duration.total_seconds())
def deploy_and_cleanup(self):
"""Do what is most often done: deploy & cleanup"""
self.deploy_to()
self.cleanup()
class Config(cdist.config_install.ConfigInstall):
pass
def config(args):
"""Configure remote system"""

306
lib/cdist/config_install.py Normal file
View file

@ -0,0 +1,306 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import datetime
import logging
import os
import stat
import sys
import cdist.emulator
import cdist.path
log = logging.getLogger(__name__)
CODE_HEADER = "#!/bin/sh -e\n"
class ConfigInstall:
"""Class to hold install and config methods"""
def __init__(self, target_host,
initial_manifest=False,
remote_user="root",
home=None,
exec_path=sys.argv[0],
debug=False):
self.target_host = target_host
self.debug = debug
self.remote_user = remote_user
self.exec_path = exec_path
# FIXME: broken - construct elsewhere!
self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host]
self.path = cdist.path.Path(self.target_host,
initial_manifest=initial_manifest,
remote_user=self.remote_user,
remote_prefix=self.remote_prefix,
base_dir=home,
debug=debug)
self.objects_prepared = []
def cleanup(self):
self.path.cleanup()
def run_global_explorers(self):
"""Run global explorers"""
log.info("Running global explorers")
explorers = self.path.list_global_explorers()
if(len(explorers) == 0):
raise CdistError("No explorers found in", self.path.global_explorer_dir)
self.path.transfer_global_explorers()
for explorer in explorers:
output = self.path.global_explorer_output_path(explorer)
output_fd = open(output, mode='w')
cmd = []
cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR)
cmd.append(self.path.remote_global_explorer_path(explorer))
cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
output_fd.close()
# FIXME: where to call this from?
def run_type_explorer(self, cdist_object):
"""Run type specific explorers for objects"""
type = self.path.get_type_from_object(cdist_object)
self.path.transfer_type_explorers(type)
cmd = []
cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR)
cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type))
cmd.append("__object=" + self.path.remote_object_dir(cdist_object))
cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object))
cmd.append("__object_fq=" + cdist_object)
# Need to transfer at least the parameters for objects to be useful
self.path.transfer_object_parameter(cdist_object)
# FIXME: Broken due to refactoring into type.py
explorers = self.path.list_type_explorers(type)
for explorer in explorers:
remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)]
output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer)
output_fd = open(output, mode='w')
log.debug("%s exploring %s using %s storing to %s",
cdist_object, explorer, remote_cmd, output)
cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
output_fd.close()
def link_emulator(self):
"""Link emulator to types"""
cdist.emulator.link(self.exec_path,
self.path.bin_dir, self.path.list_types())
def init_deploy(self):
"""Ensure the base directories are cleaned up"""
log.debug("Creating clean directory structure")
self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR)
self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR)
self.link_emulator()
def run_initial_manifest(self):
"""Run the initial manifest"""
log.info("Running initial manifest %s", self.path.initial_manifest)
env = { "__manifest" : self.path.manifest_dir }
self.run_manifest(self.path.initial_manifest, extra_env=env)
def run_type_manifest(self, cdist_object):
"""Run manifest for a specific object"""
type = self.path.get_type_from_object(cdist_object)
manifest = self.path.type_dir(type, "manifest")
log.debug("%s: Running %s", cdist_object, manifest)
if os.path.exists(manifest):
env = { "__object" : self.path.object_dir(cdist_object),
"__object_id": self.path.get_object_id_from_object(cdist_object),
"__object_fq": cdist_object,
"__type": self.path.type_dir(type)
}
self.run_manifest(manifest, extra_env=env)
def run_manifest(self, manifest, extra_env=None):
"""Run a manifest"""
log.debug("Running manifest %s, env=%s", manifest, extra_env)
env = os.environ.copy()
env['PATH'] = self.path.bin_dir + ":" + env['PATH']
# Information required in every manifest
env['__target_host'] = self.target_host
env['__global'] = self.path.out_dir
# Submit debug flag to manifest, can be used by emulator and types
if self.debug:
env['__debug'] = "yes"
# Required for recording source
env['__cdist_manifest'] = manifest
# Required to find types
env['__cdist_type_base_dir'] = self.path.type_base_dir
# Other environment stuff
if extra_env:
env.update(extra_env)
cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env)
def object_run(self, cdist_object, mode):
"""Run gencode or code for an object"""
log.debug("Running %s from %s", mode, cdist_object)
file=os.path.join(self.path.object_dir(cdist_object), "require")
requirements = cdist.path.file_to_list(file)
type = self.path.get_type_from_object(cdist_object)
for requirement in requirements:
log.debug("Object %s requires %s", cdist_object, requirement)
self.object_run(requirement, mode=mode)
#
# Setup env Variable:
#
env = os.environ.copy()
env['__target_host'] = self.target_host
env['__global'] = self.path.out_dir
env["__object"] = self.path.object_dir(cdist_object)
env["__object_id"] = self.path.get_object_id_from_object(cdist_object)
env["__object_fq"] = cdist_object
env["__type"] = self.path.type_dir(type)
if mode == "gencode":
paths = [
self.path.type_dir(type, "gencode-local"),
self.path.type_dir(type, "gencode-remote")
]
for bin in paths:
if os.path.isfile(bin):
# omit "gen" from gencode and use it for output base
outfile=os.path.join(self.path.object_dir(cdist_object),
os.path.basename(bin)[3:])
outfile_fd = open(outfile, "w")
# Need to flush to ensure our write is done before stdout write
# FIXME: CODE_HEADER needed in our sh -e scenario?
outfile_fd.write(CODE_HEADER)
outfile_fd.flush()
cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd)
outfile_fd.close()
status = os.stat(outfile)
# Remove output if empty, else make it executable
if status.st_size == len(CODE_HEADER):
os.unlink(outfile)
else:
# Add header and make executable - identically to 0o700
os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
# Mark object as changed
open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close()
if mode == "code":
local_dir = self.path.object_dir(cdist_object)
remote_dir = self.path.remote_object_dir(cdist_object)
bin = os.path.join(local_dir, "code-local")
if os.path.isfile(bin):
cdist.exec.run_or_fail([bin])
local_remote_code = os.path.join(local_dir, "code-remote")
remote_remote_code = os.path.join(remote_dir, "code-remote")
if os.path.isfile(local_remote_code):
self.path.transfer_file(local_remote_code, remote_remote_code)
# FIXME: remote_prefix
cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix)
def stage_prepare(self):
"""Do everything for a deploy, minus the actual code stage"""
self.init_deploy()
self.run_global_explorers()
self.run_initial_manifest()
log.info("Running object manifests and type explorers")
old_objects = []
objects = self.path.list_objects()
# Continue process until no new objects are created anymore
while old_objects != objects:
old_objects = list(objects)
for cdist_object in objects:
if cdist_object in self.objects_prepared:
log.debug("Skipping rerun of object %s", cdist_object)
continue
else:
# FIXME: run_type_explorer:
# object can return type
# type has explorers
# path knows about where to save explorer output
# type = self.path.objects[object].type()
# self.path.types['type'].explorers()
# for explorer in explorers:
# output = cdist.exec.run_debug_or_fail_shell(explorer)
# if output:
# write_output_to(output, os.path.join(self.path.objects[object].explorer_dir(),explorer) )
#
self.run_type_explorer(cdist_object)
self.run_type_manifest(cdist_object)
self.objects_prepared.append(cdist_object)
objects = self.path.list_objects()
def stage_run(self):
"""The final (and real) step of deployment"""
log.info("Generating and executing code")
# Now do the final steps over the existing objects
for cdist_object in self.path.list_objects():
log.debug("Run object: %s", cdist_object)
self.object_run(cdist_object, mode="gencode")
self.object_run(cdist_object, mode="code")
def deploy_to(self):
"""Mimic the old deploy to: Deploy to one host"""
log.info("Deploying to " + self.target_host)
time_start = datetime.datetime.now()
self.stage_prepare()
self.stage_run()
time_end = datetime.datetime.now()
duration = time_end - time_start
log.info("Finished run of %s in %s seconds",
self.target_host,
duration.total_seconds())
def deploy_and_cleanup(self):
"""Do what is most often done: deploy & cleanup"""
self.deploy_to()
self.cleanup()

View file

@ -0,0 +1,25 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
__all__ = ['Type', 'Object']
from cdist.core.type import Type
from cdist.core.object import Object

115
lib/cdist/core/object.py Normal file
View file

@ -0,0 +1,115 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import cdist
import cdist.path
class Object(object):
"""Represents a cdist object.
All interaction with objects in cdist should be done through this class.
Directly accessing an object through the file system from python code is
a bug.
"""
@staticmethod
def base_dir():
"""Return the absolute path to the top level directory where objects
are defined.
Requires the environment variable '__cdist_out_dir' to be set.
"""
try:
base_dir = os.path.join(
os.environ['__cdist_out_dir'],
'object'
)
except KeyError as e:
raise cdist.MissingEnvironmentVariableError(e.args[0])
# FIXME: should directory be created elsewhere?
if not os.path.isdir(base_dir):
os.mkdir(base_dir)
return base_dir
@classmethod
def list_objects(cls):
"""Return a list of object instances"""
for object_name in cls.list_object_names():
type_name = object_name.split(os.sep)[0]
object_id = os.sep.join(object_name.split(os.sep)[1:])
yield cls(Type(type_name), object_id=object_id)
@classmethod
def list_type_names(cls):
"""Return a list of type names"""
return os.listdir(cls.base_dir())
@classmethod
def list_object_names(cls):
"""Return a list of object names"""
for path, dirs, files in os.walk(cls.base_dir()):
# FIXME: use constant instead of string
if cdist.path.DOT_CDIST in dirs:
yield os.path.relpath(path, cls.base_dir())
def __init__(self, type, object_id=None, parameter=None, requirements=None):
self.type = type # instance of Type
self.object_id = object_id
self.qualified_name = os.path.join(self.type.name, self.object_id)
self.parameter = parameter or {}
self.requirements = requirements or []
def __repr__(self):
return '<Object %s>' % self.qualified_name
@property
def path(self):
return os.path.join(
self.base_dir(),
self.qualified_name,
cdist.path.DOT_CDIST
)
@property
def changed(self):
"""Check whether the object has been changed."""
return os.path.isfile(os.path.join(self.path, "changed"))
@changed.setter
def changed(self, value):
"""Change the objects changed status."""
path = os.path.join(self.path, "changed")
if value:
open(path, "w").close()
else:
try:
os.remove(path)
except EnvironmentError:
# ignore
pass
# FIXME: implement other properties/methods

125
lib/cdist/core/type.py Normal file
View file

@ -0,0 +1,125 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import cdist
class Type(object):
@staticmethod
def base_dir():
"""Return the absolute path to the top level directory where types
are defined.
Requires the environment variable '__cdist_base_dir' to be set.
"""
try:
return os.path.join(
os.environ['__cdist_base_dir'],
'conf',
'type'
)
except KeyError as e:
raise cdist.MissingEnvironmentVariableError(e.args[0])
@classmethod
def list_types(cls):
"""Return a list of type instances"""
for type_name in cls.list_type_names():
yield cls(type_name)
@classmethod
def list_type_names(cls):
"""Return a list of type names"""
return os.listdir(cls.base_dir())
def __init__(self, name):
self.name = name
self.__explorers = None
self.__required_parameters = None
self.__optional_parameters = None
def __repr__(self):
return '<Type name=%s>' % self.name
@property
def path(self):
return os.path.join(
self.base_dir(),
self.name
)
@property
def is_singleton(self):
"""Check whether a type is a singleton."""
return os.path.isfile(os.path.join(self.path, "singleton"))
@property
def is_install(self):
"""Check whether a type is used for installation (if not: for configuration)"""
return os.path.isfile(os.path.join(self.path, "install"))
@property
def explorers(self):
"""Return a list of available explorers"""
if not self.__explorers:
try:
self.__explorers = os.listdir(os.path.join(self.path, "explorer"))
except EnvironmentError as e:
# error ignored
self.__explorers = []
return self.__explorers
@property
def required_parameters(self):
"""Return a list of required parameters"""
if not self.__required_parameters:
parameters = []
try:
with open(os.path.join(self.path, "parameter", "required")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError as e:
# error ignored
pass
finally:
self.__required_parameters = parameters
return self.__required_parameters
@property
def optional_parameters(self):
"""Return a list of optional parameters"""
if not self.__optional_parameters:
parameters = []
try:
with open(os.path.join(self.path, "parameter", "optional")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError as e:
# error ignored
pass
finally:
self.__optional_parameters = parameters
return self.__optional_parameters

View file

@ -22,8 +22,14 @@
import logging
import cdist.config_install
log = logging.getLogger(__name__)
Class Install(cdist.config_install.ConfigInstall):
pass
def install(args):
"""Install remote system"""
process = {}

51
lib/cdist/object.py Normal file
View file

@ -0,0 +1,51 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import logging
log = logging.getLogger(__name__)
class Object(object):
def __init__(self, path, remote_path, object_fq):
self.path = path
self.remote_path = remote_path
self.object_fq = object_fq
self.type = self.object_fq.split(os.sep)[0]
self.object_id = self.object_fq.split(os.sep)[1:]
self.parameter_dir = os.path.join(self.path, "parameter")
self.remote_object_parameter_dir = os.path.join(self.remote_path, "parameter")
self.object_code_paths = [
os.path.join(self.path, "code-local"),
os.path.join(self.path, "code-remote")]
@property
def type_explorer_output_dir(self):
"""Returns and creates dir of the output for a type explorer"""
if not self.__type_explorer_output_dir:
dir = os.path.join(self.path, "explorer")
if not os.path.isdir(dir):
os.mkdir(dir)
self.__type_explorer_output_dir = dir
return self.__type_explorer_output_dir

View file

@ -70,6 +70,7 @@ class Path:
self.temp_dir = tempfile.mkdtemp()
self.target_host = target_host
# Input directories
self.conf_dir = os.path.join(self.base_dir, "conf")
self.cache_base_dir = os.path.join(self.base_dir, "cache")
self.cache_dir = os.path.join(self.cache_base_dir, target_host)
@ -78,30 +79,27 @@ class Path:
self.manifest_dir = os.path.join(self.conf_dir, "manifest")
self.type_base_dir = os.path.join(self.conf_dir, "type")
self.out_dir = os.path.join(self.temp_dir, "out")
os.mkdir(self.out_dir)
self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer")
os.mkdir(self.global_explorer_out_dir)
self.object_base_dir = os.path.join(self.out_dir, "object")
# Setup binary directory + contents
self.bin_dir = os.path.join(self.out_dir, "bin")
os.mkdir(self.bin_dir)
# List of type explorers transferred
self.type_explorers_transferred = {}
# objects
self.objects_prepared = []
# Mostly static, but can be overwritten on user demand
if initial_manifest:
self.initial_manifest = initial_manifest
else:
self.initial_manifest = os.path.join(self.manifest_dir, "init")
# Output directories
self.out_dir = os.path.join(self.temp_dir, "out")
self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer")
self.object_base_dir = os.path.join(self.out_dir, "object")
self.bin_dir = os.path.join(self.out_dir, "bin")
# List of type explorers transferred
self.type_explorers_transferred = {}
# objects prepared
self.objects_prepared = []
# Create directories
self.__init_out_dirs()
def cleanup(self):
# Do not use in __del__:
# http://docs.python.org/reference/datamodel.html#customization
@ -114,29 +112,49 @@ class Path:
shutil.rmtree(self.cache_dir)
shutil.move(self.temp_dir, self.cache_dir)
def __init_out_dirs(self):
"""Initialise output directory structure"""
os.mkdir(self.out_dir)
os.mkdir(self.global_explorer_out_dir)
os.mkdir(self.bin_dir)
# Stays here
def list_types(self):
"""Retuns list of types"""
return os.listdir(self.type_base_dir)
######################################################################
# FIXME: belongs to here - clearify remote*
def remote_mkdir(self, directory):
"""Create directory on remote side"""
cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True)
# FIXME: belongs to here - clearify remote*
def remove_remote_dir(self, destination):
cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True)
# FIXME: belongs to here - clearify remote*
def transfer_dir(self, source, destination):
"""Transfer directory and previously delete the remote destination"""
self.remove_remote_dir(destination)
cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
["-r", source, self.target_host + ":" + destination])
# FIXME: belongs to here - clearify remote*
def transfer_file(self, source, destination):
"""Transfer file"""
cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
[source, self.target_host + ":" + destination])
# FIXME: Explorer or stays
def global_explorer_output_path(self, explorer):
"""Returns path of the output for a global explorer"""
return os.path.join(self.global_explorer_out_dir, explorer)
# FIXME: object
def type_explorer_output_dir(self, cdist_object):
"""Returns and creates dir of the output for a type explorer"""
dir = os.path.join(self.object_dir(cdist_object), "explorer")
@ -145,29 +163,17 @@ class Path:
return dir
# FIXME Stays here / Explorer?
def remote_global_explorer_path(self, explorer):
"""Returns path to the remote explorer"""
return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer)
# FIXME: stays here
def list_global_explorers(self):
"""Return list of available explorers"""
return os.listdir(self.global_explorer_dir)
def list_type_explorers(self, type):
"""Return list of available explorers for a specific type"""
dir = self.type_dir(type, "explorer")
if os.path.isdir(dir):
list = os.listdir(dir)
else:
list = []
log.debug("Explorers for %s in %s: %s", type, dir, list)
return list
def list_types(self):
return os.listdir(self.type_base_dir)
# Stays here
def list_object_paths(self, starting_point):
"""Return list of paths of existing objects"""
object_paths = []
@ -183,36 +189,43 @@ class Path:
return object_paths
# FIXME
# FIXME: Object
def get_type_from_object(self, cdist_object):
"""Returns the first part (i.e. type) of an object"""
return cdist_object.split(os.sep)[0]
# FIXME: Object
def get_object_id_from_object(self, cdist_object):
"""Returns everything but the first part (i.e. object_id) of an object"""
return os.sep.join(cdist_object.split(os.sep)[1:])
# FIXME: Object
def object_dir(self, cdist_object):
"""Returns the full path to the object (including .cdist)"""
return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST)
# FIXME: Object
def remote_object_dir(self, cdist_object):
"""Returns the remote full path to the object (including .cdist)"""
return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST)
# FIXME: Object
def object_parameter_dir(self, cdist_object):
"""Returns the dir to the object parameter"""
return os.path.join(self.object_dir(cdist_object), "parameter")
# FIXME: object
def remote_object_parameter_dir(self, cdist_object):
"""Returns the remote dir to the object parameter"""
return os.path.join(self.remote_object_dir(cdist_object), "parameter")
# FIXME: object
def object_code_paths(self, cdist_object):
"""Return paths to code scripts of object"""
return [os.path.join(self.object_dir(cdist_object), "code-local"),
os.path.join(self.object_dir(cdist_object), "code-remote")]
# Stays here
def list_objects(self):
"""Return list of existing objects"""
@ -225,14 +238,7 @@ class Path:
return objects
def type_dir(self, type, *args):
"""Return directory the type"""
return os.path.join(self.type_base_dir, type, *args)
def remote_type_explorer_dir(self, type):
"""Return remote directory that holds the explorers of a type"""
return os.path.join(REMOTE_TYPE_DIR, type, "explorer")
# Stays here
def transfer_object_parameter(self, cdist_object):
"""Transfer the object parameter to the remote destination"""
# Create base path before using mkdir -p
@ -242,11 +248,13 @@ class Path:
self.transfer_dir(self.object_parameter_dir(cdist_object),
self.remote_object_parameter_dir(cdist_object))
# Stays here
def transfer_global_explorers(self):
"""Transfer the global explorers"""
self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR)
self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR)
# Stays here - FIXME: adjust to type code, loop over types!
def transfer_type_explorers(self, type):
"""Transfer explorers of a type, but only once"""
if type in self.type_explorers_transferred:

View file

@ -0,0 +1,47 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import subprocess
import unittest
cdist_commands=["banner", "config", "install"]
cdist_exec_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../bin/cdist"))
def exec():
print(cdist_exec_path)
#class UI(unittest.TestCase):
# def test_banner(self):
# self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0)
#
# def test_help(self):
# for cmd in cdist_commands:
# self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0)
#
# # FIXME: mockup needed
# def test_config_localhost(self):
# for cmd in cdist_commands:
# self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0)

View file

@ -0,0 +1,46 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import sys
import cdist.test
import unittest
#class UI(unittest.TestCase):
# def test_banner(self):
# self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0)
#
# def test_help(self):
# for cmd in cdist_commands:
# self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0)
#
# # FIXME: mockup needed
# def test_config_localhost(self):
# for cmd in cdist_commands:
# self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0)
print(cdist.test.cdist_exec_path)
print(sys.argv)
suite = unittest.defaultTestLoader.discover(os.path.dirname(__file__))
unittest.TextTestRunner(verbosity=1).run(suite)

0
test/nico_ui.py → lib/cdist/test/nico_ui.py Executable file → Normal file
View file

0
test/test_exec.py → lib/cdist/test/test_exec.py Executable file → Normal file
View file

View file

@ -0,0 +1,129 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import sys
import tempfile
import unittest
sys.path.insert(0, os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
import cdist.config
cdist_exec_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist"))
class Install(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
self.init_manifest = os.path.join(self.temp_dir, "manifest")
self.config = cdist.config.Config("localhost",
initial_manifest=self.init_manifest,
exec_path=cdist_exec_path)
self.config.link_emulator()
### NEW FOR INSTALL ############################################################
def test_explorer_ran(self):
"""Check that all explorers returned a result"""
self.config.run_global_explores()
explorers = self.config.path.list_global_explorers()
for explorer in explorers:
output = self.config.path.global_explorer_output_path(explorer)
self.assertTrue(os.path.isfile(output))
def test_manifest_uses_install_types_only(self):
"""Check that objects created from manifest are only of install type"""
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + " --mode 0700\n",
"__partition_msdos /dev/null --type 82\n",
])
manifest_fd.close()
self.config.run_initial_manifest()
# FIXME: check that only __partition_msdos objects are created!
self.assertFalse(failed)
### OLD FROM CONFIG ############################################################
def test_initial_manifest_different_parameter(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + " --mode 0700\n",
"__file " + self.temp_dir + " --mode 0600\n",
])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_parameter_added(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + '\n',
"__file " + self.temp_dir + " --mode 0600\n",
])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_parameter_removed(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + " --mode 0600\n",
"__file " + self.temp_dir + "\n",
])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_non_existent_command(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"thereisdefinitelynosuchcommend"])
manifest_fd.close()
self.assertRaises(cdist.Error, self.config.run_initial_manifest)
def test_initial_manifest_parameter_twice(self):
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
"__file " + self.temp_dir + " --mode 0600\n",
"__file " + self.temp_dir + " --mode 0600\n",
])
manifest_fd.close()
try:
self.config.run_initial_manifest()
except cdist.Error:
failed = True
else:
failed = False
self.assertFalse(failed)

View file

@ -0,0 +1,78 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import shutil
import sys
import tempfile
import unittest
import cdist.path
import cdist.test
class Path(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
self.init_manifest = os.path.join(self.temp_dir, "manifest")
self.path = cdist.path.Path("localhost", "root", "ssh root@localhost",
initial_manifest=self.init_manifest,
base_dir=self.temp_dir)
os.mkdir(self.path.conf_dir)
os.mkdir(self.path.type_base_dir)
self.install_type_name = "__install_test"
self.config_type_name = "__config_test"
# Create install type
self.install_type = os.path.join(self.path.type_base_dir, self.install_type_name)
os.mkdir(self.install_type)
open(os.path.join(self.install_type, "install"), "w").close()
# Create config type
self.config_type = os.path.join(self.path.type_base_dir, self.config_type_name)
os.mkdir(self.config_type)
def tearDown(self):
self.path.cleanup()
shutil.rmtree(self.temp_dir)
def test_type_detection(self):
"""Check that a type is identified as install or configuration correctly"""
self.assertTrue(self.path.is_install_type(self.install_type))
self.assertFalse(self.path.is_install_type(self.config_type))
def test_manifest_uses_install_types_only(self):
"""Check that objects created from manifest are only of install type"""
manifest_fd = open(self.init_manifest, "w")
manifest_fd.writelines(["#!/bin/sh\n",
self.install_type_name + "testid\n",
self.config_type_name + "testid\n",
])
manifest_fd.close()
self.install.run_initial_manifest()
# FIXME: check that only __partition_msdos objects are created!
self.assertFalse(failed)

51
lib/cdist/type.py Normal file
View file

@ -0,0 +1,51 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import os
log = logging.getLogger(__name__)
class Type(object):
def __init__(self, path, remote_path):
self.path = path
self.remote_path = remote_path
def list_explorers(self):
"""Return list of available explorers"""
dir = os.path.join(self.path, "explorer")
if os.path.isdir(dir):
list = os.listdir(dir)
else:
list = []
log.debug("Explorers for %s in %s: %s", type, dir, list)
return list
def is_install(self):
"""Check whether a type is used for installation (if not: for configuration)"""
return os.path.isfile(os.path.join(self.path, "install"))
def remote_explorer_dir(self):
"""Return remote directory that holds the explorers of a type"""
return os.path.join(self.remote_path, "explorer")