Merge branch 'cleanup/remove-install' of https://github.com/asteven/cdist

Also merged in base_path patch.

Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>

Conflicts:
	cdist/config_install.py
This commit is contained in:
Nico Schottelius 2013-08-30 01:35:11 +02:00
commit 25324a5e35
21 changed files with 261 additions and 357 deletions

View file

@ -20,7 +20,252 @@
# #
# #
import cdist.config_install import logging
import os
import shutil
import sys
import time
import pprint
class Config(cdist.config_install.ConfigInstall): import cdist
pass
import cdist.exec.local
import cdist.exec.remote
from cdist import core
class Config(object):
"""Cdist main class to hold arbitrary data"""
def __init__(self, local, remote, dry_run=False):
self.local = local
self.remote = remote
self.log = logging.getLogger(self.local.target_host)
self.dry_run = dry_run
self.explorer = core.Explorer(self.local.target_host, self.local, self.remote)
self.manifest = core.Manifest(self.local.target_host, self.local)
self.code = core.Code(self.local.target_host, self.local, self.remote)
def _init_files_dirs(self):
"""Prepare files and directories for the run"""
self.local.create_files_dirs()
self.remote.create_files_dirs()
@classmethod
def commandline(cls, args):
"""Configure remote system"""
import multiprocessing
# FIXME: Refactor relict - remove later
log = logging.getLogger("cdist")
initial_manifest_tempfile = None
if args.manifest == '-':
# read initial manifest from stdin
import tempfile
try:
handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
with os.fdopen(handle, 'w') as fd:
fd.write(sys.stdin.read())
except (IOError, OSError) as e:
raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
args.manifest = initial_manifest_temp_path
import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path))
process = {}
failed_hosts = []
time_start = time.time()
for host in args.host:
if args.parallel:
log.debug("Creating child process for %s", host)
process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True))
process[host].start()
else:
try:
cls.onehost(host, args, parallel=False)
except cdist.Error as e:
failed_hosts.append(host)
# Catch errors in parallel mode when joining
if args.parallel:
for host in process.keys():
log.debug("Joining process %s", host)
process[host].join()
if not process[host].exitcode == 0:
failed_hosts.append(host)
time_end = time.time()
log.info("Total processing time for %s host(s): %s", len(args.host),
(time_end - time_start))
if len(failed_hosts) > 0:
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
@classmethod
def onehost(cls, host, args, parallel):
"""Configure ONE system"""
log = logging.getLogger(host)
try:
local = cdist.exec.local.Local(
target_host=host,
initial_manifest=args.manifest,
out_path=args.out_path,
add_conf_dirs=args.conf_dir)
remote = cdist.exec.remote.Remote(
target_host=host,
remote_exec=args.remote_exec,
remote_copy=args.remote_copy)
c = cls(local, remote)
c.run()
except cdist.Error as e:
log.error(e)
if parallel:
# We are running in our own process here, need to sys.exit!
sys.exit(1)
else:
raise
except KeyboardInterrupt:
# Ignore in parallel mode, we are existing anyway
if parallel:
sys.exit(0)
# Pass back to controlling code in sequential mode
else:
raise
def run(self):
"""Do what is most often done: deploy & cleanup"""
start_time = time.time()
self._init_files_dirs()
self.explorer.run_global_explorers(self.local.global_explorer_out_path)
self.manifest.run_initial_manifest(self.local.initial_manifest)
self.iterate_until_finished()
self.local.save_cache()
self.log.info("Finished successful run in %s seconds", time.time() - start_time)
def object_list(self):
"""Short name for object list retrieval"""
for cdist_object in core.CdistObject.list_objects(self.local.object_path,
self.local.type_path):
yield cdist_object
def iterate_once(self):
"""
Iterate over the objects once - helper method for
iterate_until_finished
"""
objects_changed = False
for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements):
"""We cannot do anything for this poor object"""
continue
if cdist_object.state == core.CdistObject.STATE_UNDEF:
"""Prepare the virgin object"""
self.object_prepare(cdist_object)
objects_changed = True
if cdist_object.requirements_unfinished(cdist_object.autorequire):
"""The previous step created objects we depend on - wait for them"""
continue
if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.object_run(cdist_object)
objects_changed = True
return objects_changed
def iterate_until_finished(self):
"""
Go through all objects and solve them
one after another
"""
objects_changed = True
while objects_changed:
objects_changed = self.iterate_once()
# Check whether all objects have been finished
unfinished_objects = []
for cdist_object in self.object_list():
if not cdist_object.state == cdist_object.STATE_DONE:
unfinished_objects.append(cdist_object)
if unfinished_objects:
info_string = []
for cdist_object in unfinished_objects:
requirement_names = []
autorequire_names = []
for requirement in cdist_object.requirements_unfinished(cdist_object.requirements):
requirement_names.append(requirement.name)
for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire):
autorequire_names.append(requirement.name)
requirements = ", ".join(requirement_names)
autorequire = ", ".join(autorequire_names)
info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire))
raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved: %s" %
("; ".join(info_string)))
def object_prepare(self, cdist_object):
"""Prepare object: Run type explorer + manifest"""
self.log.info("Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object)
self.manifest.run_type_manifest(cdist_object)
cdist_object.state = core.CdistObject.STATE_PREPARED
def object_run(self, cdist_object):
"""Run gencode and code for an object"""
self.log.debug("Trying to run object %s" % (cdist_object.name))
if cdist_object.state == core.CdistObject.STATE_DONE:
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
cdist_type = cdist_object.cdist_type
# Generate
self.log.info("Generating and executing code for %s" % (cdist_object.name))
cdist_object.code_local = self.code.run_gencode_local(cdist_object)
cdist_object.code_remote = self.code.run_gencode_remote(cdist_object)
if cdist_object.code_local or cdist_object.code_remote:
cdist_object.changed = True
# Execute
if not self.dry_run:
if cdist_object.code_local:
self.code.run_code_local(cdist_object)
if cdist_object.code_remote:
self.code.transfer_code_remote(cdist_object)
self.code.run_code_remote(cdist_object)
else:
self.log.info("Skipping code execution due to DRY RUN")
# Mark this object as done
self.log.debug("Finishing run of " + cdist_object.name)
cdist_object.state = core.CdistObject.STATE_DONE

View file

@ -1,271 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import os
import shutil
import sys
import time
import pprint
import cdist
import cdist.exec.local
import cdist.exec.remote
from cdist import core
class ConfigInstall(object):
"""Cdist main class to hold arbitrary data"""
def __init__(self, local, remote, dry_run=False):
self.local = local
self.remote = remote
self.log = logging.getLogger(self.local.target_host)
self.dry_run = dry_run
self.explorer = core.Explorer(self.local.target_host, self.local, self.remote)
self.manifest = core.Manifest(self.local.target_host, self.local)
self.code = core.Code(self.local.target_host, self.local, self.remote)
def _init_files_dirs(self):
"""Prepare files and directories for the run"""
self.local.create_files_dirs()
self.remote.create_files_dirs()
@classmethod
def commandline(cls, args):
"""Configure or install remote system"""
import multiprocessing
# FIXME: Refactor relict - remove later
log = logging.getLogger("cdist")
initial_manifest_tempfile = None
if args.manifest == '-':
# read initial manifest from stdin
import tempfile
try:
handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
with os.fdopen(handle, 'w') as fd:
fd.write(sys.stdin.read())
except (IOError, OSError) as e:
raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
args.manifest = initial_manifest_temp_path
import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path))
process = {}
failed_hosts = []
time_start = time.time()
for host in args.host:
if args.parallel:
log.debug("Creating child process for %s", host)
process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True))
process[host].start()
else:
try:
cls.onehost(host, args, parallel=False)
except cdist.Error as e:
failed_hosts.append(host)
# Catch errors in parallel mode when joining
if args.parallel:
for host in process.keys():
log.debug("Joining process %s", host)
process[host].join()
if not process[host].exitcode == 0:
failed_hosts.append(host)
time_end = time.time()
log.info("Total processing time for %s host(s): %s", len(args.host),
(time_end - time_start))
if len(failed_hosts) > 0:
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
@classmethod
def onehost(cls, host, args, parallel):
"""Configure or install ONE system"""
log = logging.getLogger(host)
try:
local = cdist.exec.local.Local(
target_host=host,
initial_manifest=args.manifest,
base_path=args.out_path,
add_conf_dirs=args.conf_dir)
remote = cdist.exec.remote.Remote(
target_host=host,
remote_exec=args.remote_exec,
remote_copy=args.remote_copy)
c = cls(local, remote)
c.run()
except cdist.Error as e:
log.error(e)
if parallel:
# We are running in our own process here, need to sys.exit!
sys.exit(1)
else:
raise
except KeyboardInterrupt:
# Ignore in parallel mode, we are existing anyway
if parallel:
sys.exit(0)
# Pass back to controlling code in sequential mode
else:
raise
def run(self):
"""Do what is most often done: deploy & cleanup"""
start_time = time.time()
self._init_files_dirs()
self.explorer.run_global_explorers(self.local.global_explorer_out_path)
self.manifest.run_initial_manifest(self.local.initial_manifest)
self.iterate_until_finished()
self.local.save_cache()
self.log.info("Finished successful run in %s seconds", time.time() - start_time)
def object_list(self):
"""Short name for object list retrieval"""
for cdist_object in core.CdistObject.list_objects(self.local.object_path,
self.local.type_path):
yield cdist_object
def iterate_once(self):
"""
Iterate over the objects once - helper method for
iterate_until_finished
"""
objects_changed = False
for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements):
"""We cannot do anything for this poor object"""
continue
if cdist_object.state == core.CdistObject.STATE_UNDEF:
"""Prepare the virgin object"""
self.object_prepare(cdist_object)
objects_changed = True
if cdist_object.requirements_unfinished(cdist_object.autorequire):
"""The previous step created objects we depend on - wait for them"""
continue
if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.object_run(cdist_object)
objects_changed = True
return objects_changed
def iterate_until_finished(self):
"""
Go through all objects and solve them
one after another
"""
objects_changed = True
while objects_changed:
objects_changed = self.iterate_once()
# Check whether all objects have been finished
unfinished_objects = []
for cdist_object in self.object_list():
if not cdist_object.state == cdist_object.STATE_DONE:
unfinished_objects.append(cdist_object)
if unfinished_objects:
info_string = []
for cdist_object in unfinished_objects:
requirement_names = []
autorequire_names = []
for requirement in cdist_object.requirements_unfinished(cdist_object.requirements):
requirement_names.append(requirement.name)
for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire):
autorequire_names.append(requirement.name)
requirements = ", ".join(requirement_names)
autorequire = ", ".join(autorequire_names)
info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire))
raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved: %s" %
("; ".join(info_string)))
def object_prepare(self, cdist_object):
"""Prepare object: Run type explorer + manifest"""
self.log.info("Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object)
self.manifest.run_type_manifest(cdist_object)
cdist_object.state = core.CdistObject.STATE_PREPARED
def object_run(self, cdist_object):
"""Run gencode and code for an object"""
self.log.debug("Trying to run object %s" % (cdist_object.name))
if cdist_object.state == core.CdistObject.STATE_DONE:
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
cdist_type = cdist_object.cdist_type
# Generate
self.log.info("Generating and executing code for %s" % (cdist_object.name))
cdist_object.code_local = self.code.run_gencode_local(cdist_object)
cdist_object.code_remote = self.code.run_gencode_remote(cdist_object)
if cdist_object.code_local or cdist_object.code_remote:
cdist_object.changed = True
# Execute
if not self.dry_run:
if cdist_object.code_local:
self.code.run_code_local(cdist_object)
if cdist_object.code_remote:
self.code.transfer_code_remote(cdist_object)
self.code.run_code_remote(cdist_object)
else:
self.log.info("Skipping code execution due to DRY RUN")
# Mark this object as done
self.log.debug("Finishing run of " + cdist_object.name)
cdist_object.state = core.CdistObject.STATE_DONE

View file

@ -99,11 +99,6 @@ class CdistType(object):
"""Check whether a type is a singleton.""" """Check whether a type is a singleton."""
return os.path.isfile(os.path.join(self.absolute_path, "singleton")) return os.path.isfile(os.path.join(self.absolute_path, "singleton"))
@property
def is_install(self):
"""Check whether a type is used for installation (if not: for configuration)"""
return os.path.isfile(os.path.join(self.absolute_path, "install"))
@property @property
def explorers(self): def explorers(self):
"""Return a list of available explorers""" """Return a list of available explorers"""

View file

@ -78,11 +78,6 @@ class Emulator(object):
def run(self): def run(self):
"""Emulate type commands (i.e. __file and co)""" """Emulate type commands (i.e. __file and co)"""
if '__install' in self.env:
if not self.cdist_type.is_install:
self.log.debug("Running in install mode, ignoring non install type")
return True
self.commandline() self.commandline()
self.setup_object() self.setup_object()
self.save_stdin() self.save_stdin()

View file

@ -1,33 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import cdist.config_install
class Install(cdist.config_install.ConfigInstall):
def __init__(self, *args, **kargs):
"""Enhance config install with install support"""
# Setup environ to be used in emulator
os.environ['__install'] = "yes"
super().__init__(*args, **kargs)

View file

@ -27,9 +27,6 @@ import subprocess
import cdist.exec.local import cdist.exec.local
# FIXME: only considering config here - enable
# command line switch for using install object
# when it is available
import cdist.config import cdist.config
log = logging.getLogger(__name__) log = logging.getLogger(__name__)

View file

@ -106,16 +106,6 @@ class TypeTestCase(test.CdistTestCase):
cdist_type = core.CdistType(base_path, '__not_singleton') cdist_type = core.CdistType(base_path, '__not_singleton')
self.assertFalse(cdist_type.is_singleton) self.assertFalse(cdist_type.is_singleton)
def test_install_is_install(self):
base_path = fixtures
cdist_type = core.CdistType(base_path, '__install')
self.assertTrue(cdist_type.is_install)
def test_not_install_is_install(self):
base_path = fixtures
cdist_type = core.CdistType(base_path, '__not_install')
self.assertFalse(cdist_type.is_install)
def test_with_explorers(self): def test_with_explorers(self):
base_path = fixtures base_path = fixtures
cdist_type = core.CdistType(base_path, '__with_explorers') cdist_type = core.CdistType(base_path, '__with_explorers')

View file

@ -38,7 +38,7 @@ object_base_path = op.join(fixtures, 'object')
type_base_path = op.join(fixtures, 'type') type_base_path = op.join(fixtures, 'type')
add_conf_dir = op.join(fixtures, 'conf') add_conf_dir = op.join(fixtures, 'conf')
class ConfigInstallRunTestCase(test.CdistTestCase): class ConfigRunTestCase(test.CdistTestCase):
def setUp(self): def setUp(self):

View file

@ -26,7 +26,6 @@ def commandline():
import cdist.banner import cdist.banner
import cdist.config import cdist.config
# import cdist.install
import cdist.shell import cdist.shell
# Construct parser others can reuse # Construct parser others can reuse
@ -53,39 +52,35 @@ def commandline():
parents=[parser['loglevel']]) parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner) parser['banner'].set_defaults(func=cdist.banner.banner)
# Config and install (common stuff) # Config
parser['configinstall'] = argparse.ArgumentParser(add_help=False) parser['config'] = parser['sub'].add_parser('config',
parser['configinstall'].add_argument('host', nargs='+', parents=[parser['loglevel']])
parser['config'].add_argument('host', nargs='+',
help='one or more hosts to operate on') help='one or more hosts to operate on')
parser['configinstall'].add_argument('-c', '--conf-dir', parser['config'].add_argument('-c', '--conf-dir',
help='Add configuration directory (can be repeated, last one wins)', help='Add configuration directory (can be repeated, last one wins)',
action='append') action='append')
parser['configinstall'].add_argument('-i', '--initial-manifest', parser['config'].add_argument('-i', '--initial-manifest',
help='Path to a cdist manifest or \'-\' to read from stdin.', help='Path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False) dest='manifest', required=False)
parser['configinstall'].add_argument('-n', '--dry-run', parser['config'].add_argument('-n', '--dry-run',
help='Do not execute code', action='store_true') help='Do not execute code', action='store_true')
parser['configinstall'].add_argument('-o', '--out-dir', parser['config'].add_argument('-o', '--out-dir',
help='Directory to save cdist output in', dest="out_path") help='Directory to save cdist output in', dest="out_path")
parser['configinstall'].add_argument('-p', '--parallel', parser['config'].add_argument('-p', '--parallel',
help='Operate on multiple hosts in parallel', help='Operate on multiple hosts in parallel',
action='store_true', dest='parallel') action='store_true', dest='parallel')
parser['configinstall'].add_argument('-s', '--sequential', parser['config'].add_argument('-s', '--sequential',
help='Operate on multiple hosts sequentially (default)', help='Operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel') action='store_false', dest='parallel')
parser['config'].add_argument('--remote-copy',
parser['configinstall'].add_argument('--remote-copy',
help='Command to use for remote copy (should behave like scp)', help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy', action='store', dest='remote_copy',
default=cdist.REMOTE_COPY) default=cdist.REMOTE_COPY)
parser['configinstall'].add_argument('--remote-exec', parser['config'].add_argument('--remote-exec',
help='Command to use for remote execution (should behave like ssh)', help='Command to use for remote execution (should behave like ssh)',
action='store', dest='remote_exec', action='store', dest='remote_exec',
default=cdist.REMOTE_EXEC) default=cdist.REMOTE_EXEC)
# Config
parser['config'] = parser['sub'].add_parser('config',
parents=[parser['loglevel'], parser['configinstall']])
parser['config'].set_defaults(func=cdist.config.Config.commandline) parser['config'].set_defaults(func=cdist.config.Config.commandline)
# Shell # Shell
@ -96,12 +91,6 @@ def commandline():
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
# Install
# 20120525/sar: commented until it actually does something
#parser['install'] = parser['sub'].add_parser('install',
# parents=[parser['loglevel'], parser['configinstall']])
#parser['install'].set_defaults(func=install)
for p in parser: for p in parser:
parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
@ -133,9 +122,6 @@ def commandline():
args.func(args) args.func(args)
#def install(args):
# configinstall(args, mode=cdist.install.Install)
def emulator(): def emulator():
"""Prepare and run emulator""" """Prepare and run emulator"""
import cdist.emulator import cdist.emulator