tom/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

there is no such thing as cdist install in master

Browse source 
remove all traces of it to prevent the ongoing merge issues when using the real thing

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>
This commit is contained in:
Steven Armstrong 2013-08-29 21:56:53 +02:00
commit 3de9b869ac
21 changed files with 261 additions and 357 deletions
Download patch file Download diff file Expand all files Collapse all files

251
cdist/config.py
View file

@ -20,7 +20,252 @@
#
#
import cdist.config_install
import logging
import os
import shutil
import sys
import time
import pprint
class Config(cdist.config_install.ConfigInstall):
pass
import cdist
import cdist.exec.local
import cdist.exec.remote
from cdist import core
class Config(object):
"""Cdist main class to hold arbitrary data"""
def __init__(self, local, remote, dry_run=False):
self.local = local
self.remote = remote
self.log = logging.getLogger(self.local.target_host)
self.dry_run = dry_run
self.explorer = core.Explorer(self.local.target_host, self.local, self.remote)
self.manifest = core.Manifest(self.local.target_host, self.local)
self.code = core.Code(self.local.target_host, self.local, self.remote)
def _init_files_dirs(self):
"""Prepare files and directories for the run"""
self.local.create_files_dirs()
self.remote.create_files_dirs()
@classmethod
def commandline(cls, args):
"""Configure remote system"""
import multiprocessing
# FIXME: Refactor relict - remove later
log = logging.getLogger("cdist")
initial_manifest_tempfile = None
if args.manifest == '-':
# read initial manifest from stdin
import tempfile
try:
handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
with os.fdopen(handle, 'w') as fd:
fd.write(sys.stdin.read())
except (IOError, OSError) as e:
raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
args.manifest = initial_manifest_temp_path
import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path))
process = {}
failed_hosts = []
time_start = time.time()
for host in args.host:
if args.parallel:
log.debug("Creating child process for %s", host)
process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True))
process[host].start()
else:
try:
cls.onehost(host, args, parallel=False)
except cdist.Error as e:
failed_hosts.append(host)
# Catch errors in parallel mode when joining
if args.parallel:
for host in process.keys():
log.debug("Joining process %s", host)
process[host].join()
if not process[host].exitcode == 0:
failed_hosts.append(host)
time_end = time.time()
log.info("Total processing time for %s host(s): %s", len(args.host),
(time_end - time_start))
if len(failed_hosts) > 0:
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
@classmethod
def onehost(cls, host, args, parallel):
"""Configure ONE system"""
log = logging.getLogger(host)
try:
local = cdist.exec.local.Local(
target_host=host,
initial_manifest=args.manifest,
out_path=args.out_path,
add_conf_dirs=args.conf_dir)
remote = cdist.exec.remote.Remote(
target_host=host,
remote_exec=args.remote_exec,
remote_copy=args.remote_copy)
c = cls(local, remote)
c.run()
except cdist.Error as e:
log.error(e)
if parallel:
# We are running in our own process here, need to sys.exit!
sys.exit(1)
else:
raise
except KeyboardInterrupt:
# Ignore in parallel mode, we are existing anyway
if parallel:
sys.exit(0)
# Pass back to controlling code in sequential mode
else:
raise
def run(self):
"""Do what is most often done: deploy & cleanup"""
start_time = time.time()
self._init_files_dirs()
self.explorer.run_global_explorers(self.local.global_explorer_out_path)
self.manifest.run_initial_manifest(self.local.initial_manifest)
self.iterate_until_finished()
self.local.save_cache()
self.log.info("Finished successful run in %s seconds", time.time() - start_time)
def object_list(self):
"""Short name for object list retrieval"""
for cdist_object in core.CdistObject.list_objects(self.local.object_path,
self.local.type_path):
yield cdist_object
def iterate_once(self):
"""
Iterate over the objects once - helper method for
iterate_until_finished
"""
objects_changed = False
for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements):
"""We cannot do anything for this poor object"""
continue
if cdist_object.state == core.CdistObject.STATE_UNDEF:
"""Prepare the virgin object"""
self.object_prepare(cdist_object)
objects_changed = True
if cdist_object.requirements_unfinished(cdist_object.autorequire):
"""The previous step created objects we depend on - wait for them"""
continue
if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.object_run(cdist_object)
objects_changed = True
return objects_changed
def iterate_until_finished(self):
"""
Go through all objects and solve them
one after another
"""
objects_changed = True
while objects_changed:
objects_changed = self.iterate_once()
# Check whether all objects have been finished
unfinished_objects = []
for cdist_object in self.object_list():
if not cdist_object.state == cdist_object.STATE_DONE:
unfinished_objects.append(cdist_object)
if unfinished_objects:
info_string = []
for cdist_object in unfinished_objects:
requirement_names = []
autorequire_names = []
for requirement in cdist_object.requirements_unfinished(cdist_object.requirements):
requirement_names.append(requirement.name)
for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire):
autorequire_names.append(requirement.name)
requirements = ", ".join(requirement_names)
autorequire = ", ".join(autorequire_names)
info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire))
raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved: %s" %
("; ".join(info_string)))
def object_prepare(self, cdist_object):
"""Prepare object: Run type explorer + manifest"""
self.log.info("Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object)
self.manifest.run_type_manifest(cdist_object)
cdist_object.state = core.CdistObject.STATE_PREPARED
def object_run(self, cdist_object):
"""Run gencode and code for an object"""
self.log.debug("Trying to run object %s" % (cdist_object.name))
if cdist_object.state == core.CdistObject.STATE_DONE:
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
cdist_type = cdist_object.cdist_type
# Generate
self.log.info("Generating and executing code for %s" % (cdist_object.name))
cdist_object.code_local = self.code.run_gencode_local(cdist_object)
cdist_object.code_remote = self.code.run_gencode_remote(cdist_object)
if cdist_object.code_local or cdist_object.code_remote:
cdist_object.changed = True
# Execute
if not self.dry_run:
if cdist_object.code_local:
self.code.run_code_local(cdist_object)
if cdist_object.code_remote:
self.code.transfer_code_remote(cdist_object)
self.code.run_code_remote(cdist_object)
else:
self.log.info("Skipping code execution due to DRY RUN")
# Mark this object as done
self.log.debug("Finishing run of " + cdist_object.name)
cdist_object.state = core.CdistObject.STATE_DONE

271
cdist/config_install.py
View file

@ -1,271 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import os
import shutil
import sys
import time
import pprint
import cdist
import cdist.exec.local
import cdist.exec.remote
from cdist import core
class ConfigInstall(object):
"""Cdist main class to hold arbitrary data"""
def __init__(self, local, remote, dry_run=False):
self.local = local
self.remote = remote
self.log = logging.getLogger(self.local.target_host)
self.dry_run = dry_run
self.explorer = core.Explorer(self.local.target_host, self.local, self.remote)
self.manifest = core.Manifest(self.local.target_host, self.local)
self.code = core.Code(self.local.target_host, self.local, self.remote)
def _init_files_dirs(self):
"""Prepare files and directories for the run"""
self.local.create_files_dirs()
self.remote.create_files_dirs()
@classmethod
def commandline(cls, args):
"""Configure or install remote system"""
import multiprocessing
# FIXME: Refactor relict - remove later
log = logging.getLogger("cdist")
initial_manifest_tempfile = None
if args.manifest == '-':
# read initial manifest from stdin
import tempfile
try:
handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
with os.fdopen(handle, 'w') as fd:
fd.write(sys.stdin.read())
except (IOError, OSError) as e:
raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
args.manifest = initial_manifest_temp_path
import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path))
process = {}
failed_hosts = []
time_start = time.time()
for host in args.host:
if args.parallel:
log.debug("Creating child process for %s", host)
process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True))
process[host].start()
else:
try:
cls.onehost(host, args, parallel=False)
except cdist.Error as e:
failed_hosts.append(host)
# Catch errors in parallel mode when joining
if args.parallel:
for host in process.keys():
log.debug("Joining process %s", host)
process[host].join()
if not process[host].exitcode == 0:
failed_hosts.append(host)
time_end = time.time()
log.info("Total processing time for %s host(s): %s", len(args.host),
(time_end - time_start))
if len(failed_hosts) > 0:
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
@classmethod
def onehost(cls, host, args, parallel):
"""Configure or install ONE system"""
log = logging.getLogger(host)
try:
local = cdist.exec.local.Local(
target_host=host,
initial_manifest=args.manifest,
out_path=args.out_path,
add_conf_dirs=args.conf_dir)
remote = cdist.exec.remote.Remote(
target_host=host,
remote_exec=args.remote_exec,
remote_copy=args.remote_copy)
c = cls(local, remote)
c.run()
except cdist.Error as e:
log.error(e)
if parallel:
# We are running in our own process here, need to sys.exit!
sys.exit(1)
else:
raise
except KeyboardInterrupt:
# Ignore in parallel mode, we are existing anyway
if parallel:
sys.exit(0)
# Pass back to controlling code in sequential mode
else:
raise
def run(self):
"""Do what is most often done: deploy & cleanup"""
start_time = time.time()
self._init_files_dirs()
self.explorer.run_global_explorers(self.local.global_explorer_out_path)
self.manifest.run_initial_manifest(self.local.initial_manifest)
self.iterate_until_finished()
self.local.save_cache()
self.log.info("Finished successful run in %s seconds", time.time() - start_time)
def object_list(self):
"""Short name for object list retrieval"""
for cdist_object in core.CdistObject.list_objects(self.local.object_path,
self.local.type_path):
yield cdist_object
def iterate_once(self):
"""
Iterate over the objects once - helper method for
iterate_until_finished
"""
objects_changed = False
for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements):
"""We cannot do anything for this poor object"""
continue
if cdist_object.state == core.CdistObject.STATE_UNDEF:
"""Prepare the virgin object"""
self.object_prepare(cdist_object)
objects_changed = True
if cdist_object.requirements_unfinished(cdist_object.autorequire):
"""The previous step created objects we depend on - wait for them"""
continue
if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.object_run(cdist_object)
objects_changed = True
return objects_changed
def iterate_until_finished(self):
"""
Go through all objects and solve them
one after another
"""
objects_changed = True
while objects_changed:
objects_changed = self.iterate_once()
# Check whether all objects have been finished
unfinished_objects = []
for cdist_object in self.object_list():
if not cdist_object.state == cdist_object.STATE_DONE:
unfinished_objects.append(cdist_object)
if unfinished_objects:
info_string = []
for cdist_object in unfinished_objects:
requirement_names = []
autorequire_names = []
for requirement in cdist_object.requirements_unfinished(cdist_object.requirements):
requirement_names.append(requirement.name)
for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire):
autorequire_names.append(requirement.name)
requirements = ", ".join(requirement_names)
autorequire = ", ".join(autorequire_names)
info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire))
raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved: %s" %
("; ".join(info_string)))
def object_prepare(self, cdist_object):
"""Prepare object: Run type explorer + manifest"""
self.log.info("Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object)
self.manifest.run_type_manifest(cdist_object)
cdist_object.state = core.CdistObject.STATE_PREPARED
def object_run(self, cdist_object):
"""Run gencode and code for an object"""
self.log.debug("Trying to run object %s" % (cdist_object.name))
if cdist_object.state == core.CdistObject.STATE_DONE:
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
cdist_type = cdist_object.cdist_type
# Generate
self.log.info("Generating and executing code for %s" % (cdist_object.name))
cdist_object.code_local = self.code.run_gencode_local(cdist_object)
cdist_object.code_remote = self.code.run_gencode_remote(cdist_object)
if cdist_object.code_local or cdist_object.code_remote:
cdist_object.changed = True
# Execute
if not self.dry_run:
if cdist_object.code_local:
self.code.run_code_local(cdist_object)
if cdist_object.code_remote:
self.code.transfer_code_remote(cdist_object)
self.code.run_code_remote(cdist_object)
else:
self.log.info("Skipping code execution due to DRY RUN")
# Mark this object as done
self.log.debug("Finishing run of " + cdist_object.name)
cdist_object.state = core.CdistObject.STATE_DONE

5
cdist/core/cdist_type.py
View file

@ -99,11 +99,6 @@ class CdistType(object):
"""Check whether a type is a singleton."""
return os.path.isfile(os.path.join(self.absolute_path, "singleton"))
@property
def is_install(self):
"""Check whether a type is used for installation (if not: for configuration)"""
return os.path.isfile(os.path.join(self.absolute_path, "install"))
@property
def explorers(self):
"""Return a list of available explorers"""

5
cdist/emulator.py
View file

@ -78,11 +78,6 @@ class Emulator(object):
def run(self):
"""Emulate type commands (i.e. __file and co)"""
if '__install' in self.env:
if not self.cdist_type.is_install:
self.log.debug("Running in install mode, ignoring non install type")
return True
self.commandline()
self.setup_object()
self.save_stdin()

33
cdist/install.py
View file

@ -1,33 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import cdist.config_install
class Install(cdist.config_install.ConfigInstall):
def __init__(self, *args, **kargs):
"""Enhance config install with install support"""
# Setup environ to be used in emulator
os.environ['__install'] = "yes"
super().__init__(*args, **kargs)

3
cdist/shell.py
View file

@ -27,9 +27,6 @@ import subprocess
import cdist.exec.local
# FIXME: only considering config here - enable
# command line switch for using install object
# when it is available
import cdist.config
log = logging.getLogger(__name__)

10
cdist/test/cdist_type/__init__.py
View file

@ -106,16 +106,6 @@ class TypeTestCase(test.CdistTestCase):
cdist_type = core.CdistType(base_path, '__not_singleton')
self.assertFalse(cdist_type.is_singleton)
def test_install_is_install(self):
base_path = fixtures
cdist_type = core.CdistType(base_path, '__install')
self.assertTrue(cdist_type.is_install)
def test_not_install_is_install(self):
base_path = fixtures
cdist_type = core.CdistType(base_path, '__not_install')
self.assertFalse(cdist_type.is_install)
def test_with_explorers(self):
base_path = fixtures
cdist_type = core.CdistType(base_path, '__with_explorers')

2
cdist/test/config_install/__init__.py → cdist/test/config/__init__.py
View file

@ -38,7 +38,7 @@ object_base_path = op.join(fixtures, 'object')
type_base_path = op.join(fixtures, 'type')
add_conf_dir = op.join(fixtures, 'conf')
class ConfigInstallRunTestCase(test.CdistTestCase):
class ConfigRunTestCase(test.CdistTestCase):
def setUp(self):

0
cdist/test/config_install/fixtures/object/__first/.keep → cdist/test/config/fixtures/object/__first/.keep
View file

0
cdist/test/config_install/fixtures/object/__first/man/.cdist/.keep → cdist/test/config/fixtures/object/__first/man/.cdist/.keep
View file

0
cdist/test/config_install/fixtures/object/__second/.keep → cdist/test/config/fixtures/object/__second/.keep
View file

0
cdist/test/config_install/fixtures/object/__second/on-the/.cdist/.keep → cdist/test/config/fixtures/object/__second/on-the/.cdist/.keep
View file

0
cdist/test/config_install/fixtures/object/__third/.keep → cdist/test/config/fixtures/object/__third/.keep
View file

0
cdist/test/config_install/fixtures/object/__third/moon/.cdist/.keep → cdist/test/config/fixtures/object/__third/moon/.cdist/.keep
View file

0
cdist/test/config_install/fixtures/object/__third/moon/.cdist/parameter/name → cdist/test/config/fixtures/object/__third/moon/.cdist/parameter/name
View file

0
cdist/test/config_install/fixtures/object/__third/moon/.cdist/parameter/planet → cdist/test/config/fixtures/object/__third/moon/.cdist/parameter/planet
View file

0
cdist/test/config_install/fixtures/type/__first/.keep → cdist/test/config/fixtures/type/__first/.keep
View file

0
cdist/test/config_install/fixtures/type/__second/.keep → cdist/test/config/fixtures/type/__second/.keep
View file

0
cdist/test/config_install/fixtures/type/__singleton_test/singleton → cdist/test/config/fixtures/type/__singleton_test/singleton
View file

0
cdist/test/config_install/fixtures/type/__third/.keep → cdist/test/config/fixtures/type/__third/.keep
View file

38
scripts/cdist
View file

@ -26,7 +26,6 @@ def commandline():
import cdist.banner
import cdist.config
# import cdist.install
import cdist.shell
# Construct parser others can reuse
@ -53,39 +52,35 @@ def commandline():
parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
# Config and install (common stuff)
parser['configinstall'] = argparse.ArgumentParser(add_help=False)
parser['configinstall'].add_argument('host', nargs='+',
# Config
parser['config'] = parser['sub'].add_parser('config',
parents=[parser['loglevel']])
parser['config'].add_argument('host', nargs='+',
help='one or more hosts to operate on')
parser['configinstall'].add_argument('-c', '--conf-dir',
parser['config'].add_argument('-c', '--conf-dir',
help='Add configuration directory (can be repeated, last one wins)',
action='append')
parser['configinstall'].add_argument('-i', '--initial-manifest',
parser['config'].add_argument('-i', '--initial-manifest',
help='Path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False)
parser['configinstall'].add_argument('-n', '--dry-run',
parser['config'].add_argument('-n', '--dry-run',
help='Do not execute code', action='store_true')
parser['configinstall'].add_argument('-o', '--out-dir',
parser['config'].add_argument('-o', '--out-dir',
help='Directory to save cdist output in', dest="out_path")
parser['configinstall'].add_argument('-p', '--parallel',
parser['config'].add_argument('-p', '--parallel',
help='Operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['configinstall'].add_argument('-s', '--sequential',
parser['config'].add_argument('-s', '--sequential',
help='Operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
parser['configinstall'].add_argument('--remote-copy',
parser['config'].add_argument('--remote-copy',
help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy',
default=cdist.REMOTE_COPY)
parser['configinstall'].add_argument('--remote-exec',
parser['config'].add_argument('--remote-exec',
help='Command to use for remote execution (should behave like ssh)',
action='store', dest='remote_exec',
default=cdist.REMOTE_EXEC)
# Config
parser['config'] = parser['sub'].add_parser('config',
parents=[parser['loglevel'], parser['configinstall']])
parser['config'].set_defaults(func=cdist.config.Config.commandline)
# Shell
@ -96,12 +91,6 @@ def commandline():
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
# Install
# 20120525/sar: commented until it actually does something
#parser['install'] = parser['sub'].add_parser('install',
# parents=[parser['loglevel'], parser['configinstall']])
#parser['install'].set_defaults(func=install)
for p in parser:
parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
@ -133,9 +122,6 @@ def commandline():
args.func(args)
#def install(args):
# configinstall(args, mode=cdist.install.Install)
def emulator():
"""Prepare and run emulator"""
import cdist.emulator