Merge branch 'cleanup/remove-install' of https://github.com/asteven/cdist

Also merged in base_path patch.

Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>

Conflicts:
	cdist/config_install.py

cdist/config.py

@@ -20,7 +20,252 @@
 #
 #
 
-import cdist.config_install
+import logging
+import os
+import shutil
+import sys
+import time
+import pprint
 
-class Config(cdist.config_install.ConfigInstall):
+import cdist
-    pass
+
+import cdist.exec.local
+import cdist.exec.remote
+
+from cdist import core
+
+class Config(object):
+    """Cdist main class to hold arbitrary data"""
+
+    def __init__(self, local, remote, dry_run=False):
+
+        self.local      = local
+        self.remote     = remote
+        self.log        = logging.getLogger(self.local.target_host)
+        self.dry_run    = dry_run
+
+        self.explorer = core.Explorer(self.local.target_host, self.local, self.remote)
+        self.manifest = core.Manifest(self.local.target_host, self.local)
+        self.code     = core.Code(self.local.target_host, self.local, self.remote)
+
+    def _init_files_dirs(self):
+        """Prepare files and directories for the run"""
+        self.local.create_files_dirs()
+        self.remote.create_files_dirs()
+
+    @classmethod
+    def commandline(cls, args):
+        """Configure remote system"""
+        import multiprocessing
+
+        # FIXME: Refactor relict - remove later
+        log = logging.getLogger("cdist")
+    
+        initial_manifest_tempfile = None
+        if args.manifest == '-':
+            # read initial manifest from stdin
+            import tempfile
+            try:
+                handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
+                with os.fdopen(handle, 'w') as fd:
+                    fd.write(sys.stdin.read())
+            except (IOError, OSError) as e:
+                raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
+    
+            args.manifest = initial_manifest_temp_path
+            import atexit
+            atexit.register(lambda: os.remove(initial_manifest_temp_path))
+    
+        process = {}
+        failed_hosts = []
+        time_start = time.time()
+    
+        for host in args.host:
+            if args.parallel:
+                log.debug("Creating child process for %s", host)
+                process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True))
+                process[host].start()
+            else:
+                try:
+                    cls.onehost(host, args, parallel=False)
+                except cdist.Error as e:
+                    failed_hosts.append(host)
+    
+        # Catch errors in parallel mode when joining
+        if args.parallel:
+            for host in process.keys():
+                log.debug("Joining process %s", host)
+                process[host].join()
+    
+                if not process[host].exitcode == 0:
+                    failed_hosts.append(host)
+    
+        time_end = time.time()
+        log.info("Total processing time for %s host(s): %s", len(args.host),
+                    (time_end - time_start))
+    
+        if len(failed_hosts) > 0:
+            raise cdist.Error("Failed to configure the following hosts: " + 
+                " ".join(failed_hosts))
+    
+    @classmethod
+    def onehost(cls, host, args, parallel):
+        """Configure ONE system"""
+
+        log = logging.getLogger(host)
+    
+        try:
+            local = cdist.exec.local.Local(
+                target_host=host,
+                initial_manifest=args.manifest,
+                out_path=args.out_path,
+                add_conf_dirs=args.conf_dir)
+
+            remote = cdist.exec.remote.Remote(
+                target_host=host,
+                remote_exec=args.remote_exec,
+                remote_copy=args.remote_copy)
+    
+            c = cls(local, remote)
+            c.run()
+    
+        except cdist.Error as e:
+            log.error(e)
+            if parallel:
+                # We are running in our own process here, need to sys.exit!
+                sys.exit(1)
+            else:
+                raise
+    
+        except KeyboardInterrupt:
+            # Ignore in parallel mode, we are existing anyway
+            if parallel:
+                sys.exit(0)
+            # Pass back to controlling code in sequential mode
+            else:
+                raise
+
+    def run(self):
+        """Do what is most often done: deploy & cleanup"""
+        start_time = time.time()
+
+        self._init_files_dirs()
+
+        self.explorer.run_global_explorers(self.local.global_explorer_out_path)
+        self.manifest.run_initial_manifest(self.local.initial_manifest)
+        self.iterate_until_finished()
+
+        self.local.save_cache()
+        self.log.info("Finished successful run in %s seconds", time.time() - start_time)
+
+
+    def object_list(self):
+        """Short name for object list retrieval"""
+        for cdist_object in core.CdistObject.list_objects(self.local.object_path,
+                                                         self.local.type_path):
+            yield cdist_object
+
+    def iterate_once(self):
+        """
+            Iterate over the objects once - helper method for 
+            iterate_until_finished
+        """
+        objects_changed  = False
+
+        for cdist_object in self.object_list():
+            if cdist_object.requirements_unfinished(cdist_object.requirements):
+                """We cannot do anything for this poor object"""
+                continue
+        
+            if cdist_object.state == core.CdistObject.STATE_UNDEF:
+                """Prepare the virgin object"""
+        
+                self.object_prepare(cdist_object)
+                objects_changed = True
+        
+            if cdist_object.requirements_unfinished(cdist_object.autorequire):
+                """The previous step created objects we depend on - wait for them"""
+                continue
+        
+            if cdist_object.state == core.CdistObject.STATE_PREPARED:
+                self.object_run(cdist_object)
+                objects_changed = True
+
+        return objects_changed
+
+
+    def iterate_until_finished(self):
+        """
+            Go through all objects and solve them
+            one after another
+        """
+
+        objects_changed = True
+
+        while objects_changed:
+            objects_changed = self.iterate_once()
+
+        # Check whether all objects have been finished
+        unfinished_objects = []
+        for cdist_object in self.object_list():
+            if not cdist_object.state == cdist_object.STATE_DONE:
+                unfinished_objects.append(cdist_object)
+
+        if unfinished_objects:
+            info_string = []
+
+            for cdist_object in unfinished_objects:
+
+                requirement_names = []
+                autorequire_names = []
+
+                for requirement in cdist_object.requirements_unfinished(cdist_object.requirements):
+                    requirement_names.append(requirement.name)
+
+                for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire):
+                    autorequire_names.append(requirement.name)
+
+                requirements = ", ".join(requirement_names)
+                autorequire  = ", ".join(autorequire_names)
+                info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire))
+
+            raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved: %s" %
+                ("; ".join(info_string)))
+
+    def object_prepare(self, cdist_object):
+        """Prepare object: Run type explorer + manifest"""
+        self.log.info("Running manifest and explorers for " + cdist_object.name)
+        self.explorer.run_type_explorers(cdist_object)
+        self.manifest.run_type_manifest(cdist_object)
+        cdist_object.state = core.CdistObject.STATE_PREPARED
+
+    def object_run(self, cdist_object):
+        """Run gencode and code for an object"""
+
+        self.log.debug("Trying to run object %s" % (cdist_object.name))
+        if cdist_object.state == core.CdistObject.STATE_DONE:
+            raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
+
+        cdist_type = cdist_object.cdist_type
+
+        # Generate
+        self.log.info("Generating and executing code for %s" % (cdist_object.name))
+        cdist_object.code_local = self.code.run_gencode_local(cdist_object)
+        cdist_object.code_remote = self.code.run_gencode_remote(cdist_object)
+        if cdist_object.code_local or cdist_object.code_remote:
+            cdist_object.changed = True
+
+        # Execute
+        if not self.dry_run:
+            if cdist_object.code_local:
+                self.code.run_code_local(cdist_object)
+            if cdist_object.code_remote:
+                self.code.transfer_code_remote(cdist_object)
+                self.code.run_code_remote(cdist_object)
+        else:
+            self.log.info("Skipping code execution due to DRY RUN")
+
+
+        # Mark this object as done
+        self.log.debug("Finishing run of " + cdist_object.name)
+        cdist_object.state = core.CdistObject.STATE_DONE

cdist/config_install.py

@@ -1,271 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import logging
-import os
-import shutil
-import sys
-import time
-import pprint
-
-import cdist
-
-import cdist.exec.local
-import cdist.exec.remote
-
-from cdist import core
-
-class ConfigInstall(object):
-    """Cdist main class to hold arbitrary data"""
-
-    def __init__(self, local, remote, dry_run=False):
-
-        self.local      = local
-        self.remote     = remote
-        self.log        = logging.getLogger(self.local.target_host)
-        self.dry_run    = dry_run
-
-        self.explorer = core.Explorer(self.local.target_host, self.local, self.remote)
-        self.manifest = core.Manifest(self.local.target_host, self.local)
-        self.code     = core.Code(self.local.target_host, self.local, self.remote)
-
-    def _init_files_dirs(self):
-        """Prepare files and directories for the run"""
-        self.local.create_files_dirs()
-        self.remote.create_files_dirs()
-
-    @classmethod
-    def commandline(cls, args):
-        """Configure or install remote system"""
-        import multiprocessing
-
-        # FIXME: Refactor relict - remove later
-        log = logging.getLogger("cdist")
-    
-        initial_manifest_tempfile = None
-        if args.manifest == '-':
-            # read initial manifest from stdin
-            import tempfile
-            try:
-                handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
-                with os.fdopen(handle, 'w') as fd:
-                    fd.write(sys.stdin.read())
-            except (IOError, OSError) as e:
-                raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
-    
-            args.manifest = initial_manifest_temp_path
-            import atexit
-            atexit.register(lambda: os.remove(initial_manifest_temp_path))
-    
-        process = {}
-        failed_hosts = []
-        time_start = time.time()
-    
-        for host in args.host:
-            if args.parallel:
-                log.debug("Creating child process for %s", host)
-                process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True))
-                process[host].start()
-            else:
-                try:
-                    cls.onehost(host, args, parallel=False)
-                except cdist.Error as e:
-                    failed_hosts.append(host)
-    
-        # Catch errors in parallel mode when joining
-        if args.parallel:
-            for host in process.keys():
-                log.debug("Joining process %s", host)
-                process[host].join()
-    
-                if not process[host].exitcode == 0:
-                    failed_hosts.append(host)
-    
-        time_end = time.time()
-        log.info("Total processing time for %s host(s): %s", len(args.host),
-                    (time_end - time_start))
-    
-        if len(failed_hosts) > 0:
-            raise cdist.Error("Failed to configure the following hosts: " + 
-                " ".join(failed_hosts))
-    
-    @classmethod
-    def onehost(cls, host, args, parallel):
-        """Configure or install ONE system"""
-
-        log = logging.getLogger(host)
-    
-        try:
-            local = cdist.exec.local.Local(
-                target_host=host,
-                initial_manifest=args.manifest,
-                base_path=args.out_path,
-                add_conf_dirs=args.conf_dir)
-
-            remote = cdist.exec.remote.Remote(
-                target_host=host,
-                remote_exec=args.remote_exec,
-                remote_copy=args.remote_copy)
-    
-            c = cls(local, remote)
-            c.run()
-    
-        except cdist.Error as e:
-            log.error(e)
-            if parallel:
-                # We are running in our own process here, need to sys.exit!
-                sys.exit(1)
-            else:
-                raise
-    
-        except KeyboardInterrupt:
-            # Ignore in parallel mode, we are existing anyway
-            if parallel:
-                sys.exit(0)
-            # Pass back to controlling code in sequential mode
-            else:
-                raise
-
-    def run(self):
-        """Do what is most often done: deploy & cleanup"""
-        start_time = time.time()
-
-        self._init_files_dirs()
-
-        self.explorer.run_global_explorers(self.local.global_explorer_out_path)
-        self.manifest.run_initial_manifest(self.local.initial_manifest)
-        self.iterate_until_finished()
-
-        self.local.save_cache()
-        self.log.info("Finished successful run in %s seconds", time.time() - start_time)
-
-
-    def object_list(self):
-        """Short name for object list retrieval"""
-        for cdist_object in core.CdistObject.list_objects(self.local.object_path,
-                                                         self.local.type_path):
-            yield cdist_object
-
-    def iterate_once(self):
-        """
-            Iterate over the objects once - helper method for 
-            iterate_until_finished
-        """
-        objects_changed  = False
-
-        for cdist_object in self.object_list():
-            if cdist_object.requirements_unfinished(cdist_object.requirements):
-                """We cannot do anything for this poor object"""
-                continue
-        
-            if cdist_object.state == core.CdistObject.STATE_UNDEF:
-                """Prepare the virgin object"""
-        
-                self.object_prepare(cdist_object)
-                objects_changed = True
-        
-            if cdist_object.requirements_unfinished(cdist_object.autorequire):
-                """The previous step created objects we depend on - wait for them"""
-                continue
-        
-            if cdist_object.state == core.CdistObject.STATE_PREPARED:
-                self.object_run(cdist_object)
-                objects_changed = True
-
-        return objects_changed
-
-
-    def iterate_until_finished(self):
-        """
-            Go through all objects and solve them
-            one after another
-        """
-
-        objects_changed = True
-
-        while objects_changed:
-            objects_changed = self.iterate_once()
-
-        # Check whether all objects have been finished
-        unfinished_objects = []
-        for cdist_object in self.object_list():
-            if not cdist_object.state == cdist_object.STATE_DONE:
-                unfinished_objects.append(cdist_object)
-
-        if unfinished_objects:
-            info_string = []
-
-            for cdist_object in unfinished_objects:
-
-                requirement_names = []
-                autorequire_names = []
-
-                for requirement in cdist_object.requirements_unfinished(cdist_object.requirements):
-                    requirement_names.append(requirement.name)
-
-                for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire):
-                    autorequire_names.append(requirement.name)
-
-                requirements = ", ".join(requirement_names)
-                autorequire  = ", ".join(autorequire_names)
-                info_string.append("%s requires: %s autorequires: %s" % (cdist_object.name, requirements, autorequire))
-
-            raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved: %s" %
-                ("; ".join(info_string)))
-
-    def object_prepare(self, cdist_object):
-        """Prepare object: Run type explorer + manifest"""
-        self.log.info("Running manifest and explorers for " + cdist_object.name)
-        self.explorer.run_type_explorers(cdist_object)
-        self.manifest.run_type_manifest(cdist_object)
-        cdist_object.state = core.CdistObject.STATE_PREPARED
-
-    def object_run(self, cdist_object):
-        """Run gencode and code for an object"""
-
-        self.log.debug("Trying to run object %s" % (cdist_object.name))
-        if cdist_object.state == core.CdistObject.STATE_DONE:
-            raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
-
-        cdist_type = cdist_object.cdist_type
-
-        # Generate
-        self.log.info("Generating and executing code for %s" % (cdist_object.name))
-        cdist_object.code_local = self.code.run_gencode_local(cdist_object)
-        cdist_object.code_remote = self.code.run_gencode_remote(cdist_object)
-        if cdist_object.code_local or cdist_object.code_remote:
-            cdist_object.changed = True
-
-        # Execute
-        if not self.dry_run:
-            if cdist_object.code_local:
-                self.code.run_code_local(cdist_object)
-            if cdist_object.code_remote:
-                self.code.transfer_code_remote(cdist_object)
-                self.code.run_code_remote(cdist_object)
-        else:
-            self.log.info("Skipping code execution due to DRY RUN")
-
-
-        # Mark this object as done
-        self.log.debug("Finishing run of " + cdist_object.name)
-        cdist_object.state = core.CdistObject.STATE_DONE

cdist/core/cdist_type.py

@@ -99,11 +99,6 @@ class CdistType(object):
         """Check whether a type is a singleton."""
         return os.path.isfile(os.path.join(self.absolute_path, "singleton"))
 
-    @property
-    def is_install(self):
-        """Check whether a type is used for installation (if not: for configuration)"""
-        return os.path.isfile(os.path.join(self.absolute_path, "install"))
-
     @property
     def explorers(self):
         """Return a list of available explorers"""

cdist/emulator.py

@@ -78,11 +78,6 @@ class Emulator(object):
     def run(self):
         """Emulate type commands (i.e. __file and co)"""
 
-        if '__install' in self.env:
-            if not self.cdist_type.is_install:
-                self.log.debug("Running in install mode, ignoring non install type")
-                return True
-
         self.commandline()
         self.setup_object()
         self.save_stdin()

cdist/install.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import os
-import cdist.config_install
-
-class Install(cdist.config_install.ConfigInstall):
-    def __init__(self, *args, **kargs):
-        """Enhance config install with install support"""
-
-        # Setup environ to be used in emulator
-        os.environ['__install'] = "yes"
-
-        super().__init__(*args, **kargs)

cdist/shell.py

@@ -27,9 +27,6 @@ import subprocess
 import cdist.exec.local
 
 
-# FIXME: only considering config here - enable
-# command line switch for using install object
-# when it is available
 import cdist.config
 
 log = logging.getLogger(__name__)

cdist/test/cdist_type/__init__.py

@@ -106,16 +106,6 @@ class TypeTestCase(test.CdistTestCase):
         cdist_type = core.CdistType(base_path, '__not_singleton')
         self.assertFalse(cdist_type.is_singleton)
 
-    def test_install_is_install(self):
-        base_path = fixtures
-        cdist_type = core.CdistType(base_path, '__install')
-        self.assertTrue(cdist_type.is_install)
-
-    def test_not_install_is_install(self):
-        base_path = fixtures
-        cdist_type = core.CdistType(base_path, '__not_install')
-        self.assertFalse(cdist_type.is_install)
-
     def test_with_explorers(self):
         base_path = fixtures
         cdist_type = core.CdistType(base_path, '__with_explorers')

cdist/test/config/__init__.py

@@ -38,7 +38,7 @@ object_base_path = op.join(fixtures, 'object')
 type_base_path = op.join(fixtures, 'type')
 add_conf_dir = op.join(fixtures, 'conf')
 
-class ConfigInstallRunTestCase(test.CdistTestCase):
+class ConfigRunTestCase(test.CdistTestCase):
 
     def setUp(self):
 

scripts/cdist

@@ -26,7 +26,6 @@ def commandline():
 
     import cdist.banner
     import cdist.config
-    # import cdist.install
     import cdist.shell
 
     # Construct parser others can reuse
@@ -53,39 +52,35 @@ def commandline():
         parents=[parser['loglevel']])
     parser['banner'].set_defaults(func=cdist.banner.banner)
 
-    # Config and install (common stuff)
+    # Config
-    parser['configinstall'] = argparse.ArgumentParser(add_help=False)
+    parser['config'] = parser['sub'].add_parser('config',
-    parser['configinstall'].add_argument('host', nargs='+',
+        parents=[parser['loglevel']])
+    parser['config'].add_argument('host', nargs='+',
         help='one or more hosts to operate on')
-    parser['configinstall'].add_argument('-c', '--conf-dir',
+    parser['config'].add_argument('-c', '--conf-dir',
          help='Add configuration directory (can be repeated, last one wins)',
          action='append')
-    parser['configinstall'].add_argument('-i', '--initial-manifest', 
+    parser['config'].add_argument('-i', '--initial-manifest', 
          help='Path to a cdist manifest or \'-\' to read from stdin.',
          dest='manifest', required=False)
-    parser['configinstall'].add_argument('-n', '--dry-run',
+    parser['config'].add_argument('-n', '--dry-run',
          help='Do not execute code', action='store_true')
-    parser['configinstall'].add_argument('-o', '--out-dir',
+    parser['config'].add_argument('-o', '--out-dir',
          help='Directory to save cdist output in', dest="out_path")
-    parser['configinstall'].add_argument('-p', '--parallel',
+    parser['config'].add_argument('-p', '--parallel',
          help='Operate on multiple hosts in parallel',
          action='store_true', dest='parallel')
-    parser['configinstall'].add_argument('-s', '--sequential',
+    parser['config'].add_argument('-s', '--sequential',
          help='Operate on multiple hosts sequentially (default)',
          action='store_false', dest='parallel')
-
+    parser['config'].add_argument('--remote-copy',
-    parser['configinstall'].add_argument('--remote-copy',
          help='Command to use for remote copy (should behave like scp)',
          action='store', dest='remote_copy',
          default=cdist.REMOTE_COPY)
-    parser['configinstall'].add_argument('--remote-exec',
+    parser['config'].add_argument('--remote-exec',
          help='Command to use for remote execution (should behave like ssh)',
          action='store', dest='remote_exec',
          default=cdist.REMOTE_EXEC)
-
-    # Config
-    parser['config'] = parser['sub'].add_parser('config',
-        parents=[parser['loglevel'], parser['configinstall']])
     parser['config'].set_defaults(func=cdist.config.Config.commandline)
 
     # Shell
@@ -96,12 +91,6 @@ def commandline():
     parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
 
 
-    # Install
-    # 20120525/sar: commented until it actually does something
-    #parser['install'] = parser['sub'].add_parser('install',
-    #    parents=[parser['loglevel'], parser['configinstall']])
-    #parser['install'].set_defaults(func=install)
-
     for p in parser:
         parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
 
@@ -133,9 +122,6 @@ def commandline():
 
     args.func(args)
 
-#def install(args):
-#    configinstall(args, mode=cdist.install.Install)
-
 def emulator():
     """Prepare and run emulator"""
     import cdist.emulator