forked from ungleich-public/cdist
		
	Merge remote-tracking branch 'telmich/master'
Conflicts: lib/cdist/emulator.py
This commit is contained in:
		
				commit
				
					
						405ebbccb7
					
				
			
		
					 12 changed files with 221 additions and 234 deletions
				
			
		|  | @ -114,11 +114,15 @@ def configinstall(args, mode): | ||||||
| 
 | 
 | ||||||
|     time_start = time.time() |     time_start = time.time() | ||||||
| 
 | 
 | ||||||
|  |     # FIXME: do not overwrite, if set! | ||||||
|     os.environ['__remote_exec'] = "ssh -o User=root -q" |     os.environ['__remote_exec'] = "ssh -o User=root -q" | ||||||
|     os.environ['__remote_copy'] = "scp -o User=root -q" |     os.environ['__remote_copy'] = "scp -o User=root -q" | ||||||
| 
 | 
 | ||||||
|     for host in args.host: |     for host in args.host: | ||||||
|         c = mode(host, initial_manifest=args.manifest, base_path=args.cdist_home, debug=args.debug) |         c = mode(host, | ||||||
|  |                 initial_manifest=args.manifest, | ||||||
|  |                 base_path=args.cdist_home, | ||||||
|  |                 debug=args.debug) | ||||||
|         if args.parallel: |         if args.parallel: | ||||||
|             log.debug("Creating child process for %s", host) |             log.debug("Creating child process for %s", host) | ||||||
|             process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) |             process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) | ||||||
|  |  | ||||||
							
								
								
									
										20
									
								
								build.sh
									
										
									
									
									
								
							
							
						
						
									
										20
									
								
								build.sh
									
										
									
									
									
								
							|  | @ -126,25 +126,15 @@ case "$1" in | ||||||
|       | xargs rm -f |       | xargs rm -f | ||||||
|    ;; |    ;; | ||||||
| 
 | 
 | ||||||
|    t) |    test) | ||||||
|       shift # skip t |       shift # skip t | ||||||
|  |       set -x | ||||||
|  |       if [ $# -lt 1 ]; then | ||||||
|  |          set -- cdist.test | ||||||
|  |       fi | ||||||
|       PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ |       PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ | ||||||
|          python3 -m unittest "$@" |          python3 -m unittest "$@" | ||||||
|    ;; |  | ||||||
|        |        | ||||||
|    test) |  | ||||||
|       PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ |  | ||||||
|          python3 -m cdist.test |  | ||||||
|    ;; |  | ||||||
| 
 |  | ||||||
|    test-install) |  | ||||||
|       PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ |  | ||||||
|          python3 -m unittest cdist.test.test_install |  | ||||||
|    ;; |  | ||||||
| 
 |  | ||||||
|    test-all) |  | ||||||
|       PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ |  | ||||||
|          python3 -m unittest discover lib/cdist/test '*.py'  |  | ||||||
|    ;; |    ;; | ||||||
| 
 | 
 | ||||||
|    *) |    *) | ||||||
|  |  | ||||||
|  | @ -1,7 +1,10 @@ | ||||||
| 2.0.3: | 2.0.3: | ||||||
| 	* Improved logging, added --verbose, by more quiet by default | 	* Improved logging, added --verbose, by more quiet by default | ||||||
| 	* Bugfix __user: Correct quoting (Steven Armstrong) | 	* Bugfix __user: Correct quoting (Steven Armstrong) | ||||||
| 	* FIXME: Support for __remote_exec and __remote_copy | 	* Bugfix requirements: Restore original require="" behaviour | ||||||
|  | 	* Feature: Initial undocumented support for replacing | ||||||
|  | 		the remote exec and remote copy commands | ||||||
|  | 	* Feature: Initial undocumented support for installations in core | ||||||
| 
 | 
 | ||||||
| 2.0.2: 2011-09-27 | 2.0.2: 2011-09-27 | ||||||
| 	* Add support for detection of OpenWall Linux (Matthias Teege) | 	* Add support for detection of OpenWall Linux (Matthias Teege) | ||||||
|  |  | ||||||
							
								
								
									
										8
									
								
								doc/dev/logs/2011-10-11.emulator-output
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										8
									
								
								doc/dev/logs/2011-10-11.emulator-output
									
										
									
									
									
										Normal file
									
								
							|  | @ -0,0 +1,8 @@ | ||||||
|  | Debug: | ||||||
|  | DEBUG: Namespace(name=None, state='installed') | ||||||
|  | DEBUG: Object output dir = /home/users/nico/.tmp/tmpsdaonx/out/object/__package_pacman/zsh/.cdist | ||||||
|  | DEBUG: Object param dir = /home/users/nico/.tmp/tmpsdaonx/out/object/__package_pacman/zsh/.cdist/parameter | ||||||
|  | DEBUG: /home/users/nico/.tmp/tmpsdaonx/out/object/__package_pacman/zsh/.cdist/parameter/state<-state = installed | ||||||
|  | DEBUG: zsh:Writing requirements:  | ||||||
|  | DEBUG: Finished __package_pacman/zsh{'state': 'installed', 'name': None} | ||||||
|  | 
 | ||||||
|  | @ -26,3 +26,4 @@ Tests needed: | ||||||
|       run_type_explorer(): ensure environment is setup correctly |       run_type_explorer(): ensure environment is setup correctly | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  |       all: check that messages of all functions contain target_host in string | ||||||
|  |  | ||||||
|  | @ -23,12 +23,13 @@ | ||||||
| import logging | import logging | ||||||
| import os | import os | ||||||
| import stat | import stat | ||||||
|  | import shutil | ||||||
| import sys | import sys | ||||||
|  | import tempfile | ||||||
| import time | import time | ||||||
| 
 | 
 | ||||||
| import cdist.context |  | ||||||
| import cdist.core | import cdist.core | ||||||
| import cdist.emulator | import cdist.exec | ||||||
| 
 | 
 | ||||||
| log = logging.getLogger(__name__) | log = logging.getLogger(__name__) | ||||||
| 
 | 
 | ||||||
|  | @ -37,44 +38,136 @@ CODE_HEADER = "#!/bin/sh -e\n" | ||||||
| class ConfigInstall: | class ConfigInstall: | ||||||
|     """Cdist main class to hold arbitrary data""" |     """Cdist main class to hold arbitrary data""" | ||||||
| 
 | 
 | ||||||
|     def __init__(self, target_host, initial_manifest=False, |     def __init__(self,  | ||||||
|  |         target_host, | ||||||
|  |         initial_manifest=False, | ||||||
|         base_path=False, |         base_path=False, | ||||||
|         exec_path=sys.argv[0], |         exec_path=sys.argv[0], | ||||||
|         debug=False): |         debug=False): | ||||||
| 
 | 
 | ||||||
|         self.target_host    = target_host |         self.target_host    = target_host | ||||||
|         os.environ['target_host'] = target_host |  | ||||||
| 
 | 
 | ||||||
|         self.debug          = debug |         self.debug          = debug | ||||||
|  | 
 | ||||||
|  |         # Required for testing | ||||||
|         self.exec_path      = exec_path |         self.exec_path      = exec_path | ||||||
| 
 | 
 | ||||||
|         self.context = cdist.context.Context(self.target_host, |         # Configure logging | ||||||
|             initial_manifest=initial_manifest, |         log.addFilter(self) | ||||||
|             base_path=base_path, | 
 | ||||||
|             debug=debug) |         # Base and Temp Base  | ||||||
|  |         if base_path: | ||||||
|  |             self.base_path = base_path | ||||||
|  |         else: | ||||||
|  |             self.base_path = os.path.abspath( | ||||||
|  |                 os.path.join(os.path.dirname(__file__), | ||||||
|  |                     os.pardir, | ||||||
|  |                     os.pardir)) | ||||||
|  | 
 | ||||||
|  |         # Local input | ||||||
|  |         self.cache_path             = os.path.join(self.base_path, "cache",  | ||||||
|  |             self.target_host) | ||||||
|  |         self.conf_path              = os.path.join(self.base_path, "conf") | ||||||
|  | 
 | ||||||
|  |         self.global_explorer_path   = os.path.join(self.conf_path, "explorer") | ||||||
|  |         self.manifest_path          = os.path.join(self.conf_path, "manifest") | ||||||
|  |         self.type_base_path         = os.path.join(self.conf_path, "type") | ||||||
|  |         self.lib_path               = os.path.join(self.base_path, "lib") | ||||||
|  | 
 | ||||||
|  |         if initial_manifest: | ||||||
|  |             self.initial_manifest = initial_manifest | ||||||
|  |         else: | ||||||
|  |             self.initial_manifest = os.path.join(self.manifest_path, "init") | ||||||
|  | 
 | ||||||
|  |         # Local output | ||||||
|  |         if '__cdist_out_dir' in os.environ: | ||||||
|  |             self.out_path = os.environ['__cdist_out_dir'] | ||||||
|  |         else: | ||||||
|  |             self.out_path = os.path.join(tempfile.mkdtemp(), "out") | ||||||
|  |         self.bin_path                 = os.path.join(self.out_path, "bin") | ||||||
|  |         self.global_explorer_out_path = os.path.join(self.out_path, "explorer") | ||||||
|  |         self.object_base_path         = os.path.join(self.out_path, "object") | ||||||
|  | 
 | ||||||
|  |         # Remote directory base | ||||||
|  |         if '__cdist_remote_out_dir' in os.environ: | ||||||
|  |             self.remote_base_path = os.environ['__cdist_remote_out_dir'] | ||||||
|  |         else: | ||||||
|  |             self.remote_base_path = "/var/lib/cdist" | ||||||
|  | 
 | ||||||
|  |         self.remote_conf_path            = os.path.join(self.remote_base_path, "conf") | ||||||
|  |         self.remote_object_path          = os.path.join(self.remote_base_path, "object") | ||||||
|  | 
 | ||||||
|  |         self.remote_type_path            = os.path.join(self.remote_conf_path, "type") | ||||||
|  |         self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer") | ||||||
|  | 
 | ||||||
|  |         # Setup env to be used by others | ||||||
|  |         self.__init_env() | ||||||
|  | 
 | ||||||
|  |         # Create directories | ||||||
|  |         self.__init_local_paths() | ||||||
|  |         self.__init_remote_paths() | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  |     def __init_remote_paths(self): | ||||||
|  |         """Initialise remote directory structure""" | ||||||
|  |         self.remove_remote_path(self.remote_base_path) | ||||||
|  |         self.remote_mkdir(self.remote_base_path) | ||||||
|  |         self.remote_mkdir(self.remote_conf_path) | ||||||
|  | 
 | ||||||
|  |     def __init_local_paths(self): | ||||||
|  |         """Initialise local directory structure""" | ||||||
|  | 
 | ||||||
|  |         # Create base dir, if user supplied and not existing | ||||||
|  |         if not os.path.isdir(self.base_path): | ||||||
|  |             os.mkdir(self.base_path) | ||||||
|  |              | ||||||
|  |         os.mkdir(self.out_path) | ||||||
|  |         os.mkdir(self.global_explorer_out_path) | ||||||
|  |         os.mkdir(self.bin_path) | ||||||
|  | 
 | ||||||
|  |     def __init_env(self): | ||||||
|  |         """Environment usable for other stuff""" | ||||||
|  |         os.environ['__target_host'] = self.target_host | ||||||
| 
 | 
 | ||||||
|     def cleanup(self): |     def cleanup(self): | ||||||
|         self.context.cleanup() |         # Do not use in __del__: | ||||||
|  |         # http://docs.python.org/reference/datamodel.html#customization | ||||||
|  |         # "other globals referenced by the __del__() method may already have been deleted  | ||||||
|  |         # or in the process of being torn down (e.g. the import machinery shutting down)" | ||||||
|  |         # | ||||||
|  |         log.debug("Saving " + self.out_path + " to " + self.cache_path) | ||||||
|  |         # Remove previous cache | ||||||
|  |         if os.path.exists(self.cache_path): | ||||||
|  |             shutil.rmtree(self.cache_path) | ||||||
|  |         shutil.move(self.out_path, self.cache_path) | ||||||
|  | 
 | ||||||
|  |     def filter(self, record): | ||||||
|  |         """Add hostname to logs via logging Filter""" | ||||||
|  | 
 | ||||||
|  |         record.msg = self.target_host + ": " + record.msg | ||||||
|  | 
 | ||||||
|  |         return True | ||||||
| 
 | 
 | ||||||
|     def run_initial_manifest(self): |     def run_initial_manifest(self): | ||||||
|         """Run the initial manifest""" |         """Run the initial manifest""" | ||||||
|         log.info("Running initial manifest %s", self.context.initial_manifest) |         log.info("Running initial manifest %s", self.initial_manifest) | ||||||
|         env = {  "__manifest" : self.context.manifest_path } |         env = {  "__manifest" : self.manifest_path } | ||||||
|         self.run_manifest(self.context.initial_manifest, extra_env=env) |         self.run_manifest(self.initial_manifest, extra_env=env) | ||||||
| 
 | 
 | ||||||
|     def run_type_manifest(self, cdist_object): |     def run_type_manifest(self, cdist_object): | ||||||
|         """Run manifest for a specific object""" |         """Run manifest for a specific object""" | ||||||
|         cdist_type = cdist_object.type |         cdist_type = cdist_object.type | ||||||
|         manifest_path = os.path.join(self.context.type_base_path, |         manifest_path = os.path.join(self.type_base_path, | ||||||
|                             cdist_type.manifest_path) |                             cdist_type.manifest_path) | ||||||
|          |          | ||||||
|         log.debug("%s: Running %s", cdist_object.name, manifest_path) |         log.debug("%s: Running %s", cdist_object.name, manifest_path) | ||||||
|         if os.path.exists(manifest_path): |         if os.path.exists(manifest_path): | ||||||
|             env = { "__object" :    os.path.join(self.context.object_base_path, |             env = { "__object" :    os.path.join(self.object_base_path, | ||||||
|                                         cdist_object.path), |                                         cdist_object.path), | ||||||
|                     "__object_id":  cdist_object.object_id, |                     "__object_id":  cdist_object.object_id, | ||||||
|                     "__object_fq":  cdist_object.name, |                     "__object_fq":  cdist_object.name, | ||||||
|                     "__type":       os.path.join(self.context.type_base_path, |                     "__type":       os.path.join(self.type_base_path, | ||||||
|                                         cdist_type.path) |                                         cdist_type.path) | ||||||
|                     } |                     } | ||||||
|             self.run_manifest(manifest_path, extra_env=env) |             self.run_manifest(manifest_path, extra_env=env) | ||||||
|  | @ -83,11 +176,11 @@ class ConfigInstall: | ||||||
|         """Run a manifest""" |         """Run a manifest""" | ||||||
|         log.debug("Running manifest %s, env=%s", manifest_path, extra_env) |         log.debug("Running manifest %s, env=%s", manifest_path, extra_env) | ||||||
|         env = os.environ.copy() |         env = os.environ.copy() | ||||||
|         env['PATH'] = self.context.bin_path + ":" + env['PATH'] |         env['PATH'] = self.bin_path + ":" + env['PATH'] | ||||||
| 
 | 
 | ||||||
|         # Information required in every manifest |         # Information required in every manifest | ||||||
|         env['__target_host']            = self.target_host |         env['__target_host']            = self.target_host | ||||||
|         env['__global']                 = self.context.out_path |         env['__global']                 = self.out_path | ||||||
|          |          | ||||||
|         # Submit debug flag to manifest, can be used by emulator and types |         # Submit debug flag to manifest, can be used by emulator and types | ||||||
|         if self.debug: |         if self.debug: | ||||||
|  | @ -97,7 +190,7 @@ class ConfigInstall: | ||||||
|         env['__cdist_manifest']         = manifest_path |         env['__cdist_manifest']         = manifest_path | ||||||
| 
 | 
 | ||||||
|         # Required to find types in emulator |         # Required to find types in emulator | ||||||
|         env['__cdist_type_base_path']   = self.context.type_base_path |         env['__cdist_type_base_path']   = self.type_base_path | ||||||
| 
 | 
 | ||||||
|         # Other environment stuff |         # Other environment stuff | ||||||
|         if extra_env: |         if extra_env: | ||||||
|  | @ -124,19 +217,19 @@ class ConfigInstall: | ||||||
|         # |         # | ||||||
|         env = os.environ.copy() |         env = os.environ.copy() | ||||||
|         env['__target_host']    = self.target_host |         env['__target_host']    = self.target_host | ||||||
|         env['__global']         = self.context.out_path |         env['__global']         = self.out_path | ||||||
|         env["__object"]         = os.path.join(self.context.object_base_path, cdist_object.path) |         env["__object"]         = os.path.join(self.object_base_path, cdist_object.path) | ||||||
|         env["__object_id"]      = cdist_object.object_id |         env["__object_id"]      = cdist_object.object_id | ||||||
|         env["__object_fq"]      = cdist_object.name |         env["__object_fq"]      = cdist_object.name | ||||||
|         env["__type"]           = cdist_type.name |         env["__type"]           = cdist_type.name | ||||||
| 
 | 
 | ||||||
|         # gencode |         # gencode | ||||||
|         for cmd in ["local", "remote"]: |         for cmd in ["local", "remote"]: | ||||||
|             bin = os.path.join(self.context.type_base_path, |             bin = os.path.join(self.type_base_path, | ||||||
|                     getattr(cdist_type, "gencode_" + cmd + "_path")) |                     getattr(cdist_type, "gencode_" + cmd + "_path")) | ||||||
| 
 | 
 | ||||||
|             if os.path.isfile(bin): |             if os.path.isfile(bin): | ||||||
|                 outfile = os.path.join(self.context.object_base_path, |                 outfile = os.path.join(self.object_base_path, | ||||||
|                             getattr(cdist_object, "code_" + cmd + "_path")) |                             getattr(cdist_object, "code_" + cmd + "_path")) | ||||||
| 
 | 
 | ||||||
|                 outfile_fd = open(outfile, "w") |                 outfile_fd = open(outfile, "w") | ||||||
|  | @ -164,12 +257,12 @@ class ConfigInstall: | ||||||
|             cdist.exec.run_or_fail([code_local]) |             cdist.exec.run_or_fail([code_local]) | ||||||
| 
 | 
 | ||||||
|         # code remote |         # code remote | ||||||
|         local_remote_code   = os.path.join(self.context.object_base_path, |         local_remote_code   = os.path.join(self.object_base_path, | ||||||
|             cdist_object.code_remote_path) |             cdist_object.code_remote_path) | ||||||
|         remote_remote_code  = os.path.join(self.context.remote_object_path, |         remote_remote_code  = os.path.join(self.remote_object_path, | ||||||
|             cdist_object.code_remote_path) |             cdist_object.code_remote_path) | ||||||
|         if os.path.isfile(local_remote_code): |         if os.path.isfile(local_remote_code): | ||||||
|             self.context.transfer_path(local_remote_code, remote_remote_code) |             self.transfer_path(local_remote_code, remote_remote_code) | ||||||
|             cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) |             cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) | ||||||
| 
 | 
 | ||||||
|         cdist_object.ran = True |         cdist_object.ran = True | ||||||
|  | @ -181,12 +274,12 @@ class ConfigInstall: | ||||||
|         self.transfer_type_explorers(cdist_type) |         self.transfer_type_explorers(cdist_type) | ||||||
| 
 | 
 | ||||||
|         cmd = [] |         cmd = [] | ||||||
|         cmd.append("__explorer="        + self.context.remote_global_explorer_path) |         cmd.append("__explorer="        + self.remote_global_explorer_path) | ||||||
|         cmd.append("__type_explorer="   + os.path.join( |         cmd.append("__type_explorer="   + os.path.join( | ||||||
|                                             self.context.remote_type_path, |                                             self.remote_type_path, | ||||||
|                                             cdist_type.explorer_path)) |                                             cdist_type.explorer_path)) | ||||||
|         cmd.append("__object="          + os.path.join( |         cmd.append("__object="          + os.path.join( | ||||||
|                                             self.context.remote_object_path, |                                             self.remote_object_path, | ||||||
|                                             cdist_object.path)) |                                             cdist_object.path)) | ||||||
|         cmd.append("__object_id="       + cdist_object.object_id) |         cmd.append("__object_id="       + cdist_object.object_id) | ||||||
|         cmd.append("__object_fq="       + cdist_object.name) |         cmd.append("__object_fq="       + cdist_object.name) | ||||||
|  | @ -195,9 +288,9 @@ class ConfigInstall: | ||||||
|         self.transfer_object_parameter(cdist_object) |         self.transfer_object_parameter(cdist_object) | ||||||
| 
 | 
 | ||||||
|         for explorer in cdist_type.explorers: |         for explorer in cdist_type.explorers: | ||||||
|             remote_cmd = cmd + [os.path.join(self.context.remote_base_path, |             remote_cmd = cmd + [os.path.join(self.remote_type_path, | ||||||
|                 cdist_type.explorer_path, explorer)] |                 cdist_type.explorer_path, explorer)] | ||||||
|             output = os.path.join(self.context.object_base_path, |             output = os.path.join(self.object_base_path, | ||||||
|                         cdist_object.explorer_path, explorer) |                         cdist_object.explorer_path, explorer) | ||||||
|             output_fd = open(output, mode='w') |             output_fd = open(output, mode='w') | ||||||
|             log.debug("%s exploring %s using %s storing to %s",  |             log.debug("%s exploring %s using %s storing to %s",  | ||||||
|  | @ -210,8 +303,8 @@ class ConfigInstall: | ||||||
|     def link_emulator(self): |     def link_emulator(self): | ||||||
|         """Link emulator to types""" |         """Link emulator to types""" | ||||||
|         src = os.path.abspath(self.exec_path) |         src = os.path.abspath(self.exec_path) | ||||||
|         for cdist_type in cdist.core.Type.list_types(self.context.type_base_path): |         for cdist_type in cdist.core.Type.list_types(self.type_base_path): | ||||||
|             dst = os.path.join(self.context.bin_path, cdist_type.name) |             dst = os.path.join(self.bin_path, cdist_type.name) | ||||||
|             log.debug("Linking emulator: %s to %s", src, dst) |             log.debug("Linking emulator: %s to %s", src, dst) | ||||||
| 
 | 
 | ||||||
|             # FIXME: handle exception / make it more beautiful |             # FIXME: handle exception / make it more beautiful | ||||||
|  | @ -221,11 +314,11 @@ class ConfigInstall: | ||||||
|         """Run global explorers""" |         """Run global explorers""" | ||||||
|         log.info("Running global explorers") |         log.info("Running global explorers") | ||||||
| 
 | 
 | ||||||
|         src_path = self.context.global_explorer_path |         src_path = self.global_explorer_path | ||||||
|         dst_path = self.context.global_explorer_out_path |         dst_path = self.global_explorer_out_path | ||||||
|         remote_dst_path = self.context.remote_global_explorer_path |         remote_dst_path = self.remote_global_explorer_path | ||||||
| 
 | 
 | ||||||
|         self.context.transfer_path(src_path, remote_dst_path) |         self.transfer_path(src_path, remote_dst_path) | ||||||
| 
 | 
 | ||||||
|         for explorer in os.listdir(src_path): |         for explorer in os.listdir(src_path): | ||||||
|             output_fd = open(os.path.join(dst_path, explorer), mode='w') |             output_fd = open(os.path.join(dst_path, explorer), mode='w') | ||||||
|  | @ -240,8 +333,8 @@ class ConfigInstall: | ||||||
|     def stage_run(self): |     def stage_run(self): | ||||||
|         """The final (and real) step of deployment""" |         """The final (and real) step of deployment""" | ||||||
|         log.info("Generating and executing code") |         log.info("Generating and executing code") | ||||||
|         for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path, |         for cdist_object in cdist.core.Object.list_objects(self.object_base_path, | ||||||
|                                                            self.context.type_base_path): |                                                            self.type_base_path): | ||||||
|             log.debug("Run object: %s", cdist_object) |             log.debug("Run object: %s", cdist_object) | ||||||
|             self.object_run(cdist_object) |             self.object_run(cdist_object) | ||||||
| 
 | 
 | ||||||
|  | @ -271,8 +364,8 @@ class ConfigInstall: | ||||||
|         new_objects_created = True |         new_objects_created = True | ||||||
|         while new_objects_created: |         while new_objects_created: | ||||||
|             new_objects_created = False |             new_objects_created = False | ||||||
|             for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path, |             for cdist_object in cdist.core.Object.list_objects(self.object_base_path, | ||||||
|                                                                self.context.type_base_path): |                                                                self.type_base_path): | ||||||
|                 if cdist_object.prepared: |                 if cdist_object.prepared: | ||||||
|                     log.debug("Skipping rerun of object %s", cdist_object) |                     log.debug("Skipping rerun of object %s", cdist_object) | ||||||
|                     continue |                     continue | ||||||
|  | @ -285,19 +378,19 @@ class ConfigInstall: | ||||||
| 
 | 
 | ||||||
|     def transfer_object_parameter(self, cdist_object): |     def transfer_object_parameter(self, cdist_object): | ||||||
|         """Transfer the object parameter to the remote destination""" |         """Transfer the object parameter to the remote destination""" | ||||||
|         src  = os.path.join(self.context.object_base_path, |         src  = os.path.join(self.object_base_path, | ||||||
|             cdist_object.parameter_path) |             cdist_object.parameter_path) | ||||||
|         dst = os.path.join(self.context.remote_object_path, |         dst = os.path.join(self.remote_object_path, | ||||||
|             cdist_object.parameter_path) |             cdist_object.parameter_path) | ||||||
| 
 | 
 | ||||||
|         # Synchronise parameter dir afterwards |         # Synchronise parameter dir afterwards | ||||||
|         self.context.remote_mkdir(dst) |         self.remote_mkdir(dst) | ||||||
|         self.context.transfer_path(src, dst) |         self.transfer_path(src, dst) | ||||||
| 
 | 
 | ||||||
|     def transfer_global_explorers(self): |     def transfer_global_explorers(self): | ||||||
|         """Transfer the global explorers""" |         """Transfer the global explorers""" | ||||||
|         self.remote_mkdir(self.context.remote_global_explorer_path) |         self.remote_mkdir(self.remote_global_explorer_path) | ||||||
|         self.transfer_path(self.context.global_explorer_path,  |         self.transfer_path(self.global_explorer_path,  | ||||||
|             self.remote_global_explorer_path) |             self.remote_global_explorer_path) | ||||||
| 
 | 
 | ||||||
|     def transfer_type_explorers(self, cdist_type): |     def transfer_type_explorers(self, cdist_type): | ||||||
|  | @ -306,6 +399,7 @@ class ConfigInstall: | ||||||
|             log.debug("Skipping retransfer for explorers of %s", cdist_type) |             log.debug("Skipping retransfer for explorers of %s", cdist_type) | ||||||
|             return |             return | ||||||
|         else: |         else: | ||||||
|  |             log.debug("Ensure no retransfer for %s", cdist_type) | ||||||
|             # Do not retransfer |             # Do not retransfer | ||||||
|             cdist_type.transferred_explorers = True |             cdist_type.transferred_explorers = True | ||||||
| 
 | 
 | ||||||
|  | @ -313,13 +407,26 @@ class ConfigInstall: | ||||||
| 
 | 
 | ||||||
|         if len(explorers) > 0: |         if len(explorers) > 0: | ||||||
|             rel_path = cdist_type.explorer_path |             rel_path = cdist_type.explorer_path | ||||||
|             src = os.path.join(self.context.type_base_path, rel_path) |             src = os.path.join(self.type_base_path, rel_path) | ||||||
|             dst = os.path.join(self.context.remote_type_path, rel_path) |             dst = os.path.join(self.remote_type_path, rel_path) | ||||||
| 
 | 
 | ||||||
|             # Ensure full path until type exists: |             # Ensure full path until type exists: | ||||||
|             # /var/lib/cdist/conf/type/__directory/explorer |             # /var/lib/cdist/conf/type/__directory/explorer | ||||||
|             # /var/lib/cdist/conf/type/__directory may not exist, |             # /var/lib/cdist/conf/type/__directory may not exist, | ||||||
|             # but remote_mkdir uses -p to fix this |             # but remote_mkdir uses -p to fix this | ||||||
|             self.context.remote_mkdir(dst) |             self.remote_mkdir(dst) | ||||||
|             self.context.transfer_path(src, dst) |             self.transfer_path(src, dst) | ||||||
| 
 | 
 | ||||||
|  |     def remote_mkdir(self, directory): | ||||||
|  |         """Create directory on remote side""" | ||||||
|  |         cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) | ||||||
|  | 
 | ||||||
|  |     def remove_remote_path(self, destination): | ||||||
|  |         """Ensure path on remote side vanished""" | ||||||
|  |         cdist.exec.run_or_fail(["rm", "-rf",  destination], remote_prefix=True) | ||||||
|  | 
 | ||||||
|  |     def transfer_path(self, source, destination): | ||||||
|  |         """Transfer directory and previously delete the remote destination""" | ||||||
|  |         self.remove_remote_path(destination) | ||||||
|  |         cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + | ||||||
|  |             ["-r", source, self.target_host + ":" + destination]) | ||||||
|  |  | ||||||
|  | @ -1,137 +0,0 @@ | ||||||
| # -*- coding: utf-8 -*- |  | ||||||
| # |  | ||||||
| # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) |  | ||||||
| # |  | ||||||
| # This file is part of cdist. |  | ||||||
| # |  | ||||||
| # cdist is free software: you can redistribute it and/or modify |  | ||||||
| # it under the terms of the GNU General Public License as published by |  | ||||||
| # the Free Software Foundation, either version 3 of the License, or |  | ||||||
| # (at your option) any later version. |  | ||||||
| # |  | ||||||
| # cdist is distributed in the hope that it will be useful, |  | ||||||
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |  | ||||||
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the |  | ||||||
| # GNU General Public License for more details. |  | ||||||
| # |  | ||||||
| # You should have received a copy of the GNU General Public License |  | ||||||
| # along with cdist. If not, see <http://www.gnu.org/licenses/>. |  | ||||||
| # |  | ||||||
| # |  | ||||||
| 
 |  | ||||||
| import logging |  | ||||||
| import os |  | ||||||
| import shutil |  | ||||||
| import sys |  | ||||||
| import tempfile |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| log = logging.getLogger(__name__) |  | ||||||
| 
 |  | ||||||
| import cdist.exec |  | ||||||
| 
 |  | ||||||
| class Context: |  | ||||||
|     """Storing context information""" |  | ||||||
| 
 |  | ||||||
|     def __init__(self, |  | ||||||
|         target_host, |  | ||||||
|         initial_manifest=False, |  | ||||||
|         base_path=False, |  | ||||||
|         out_path=False, |  | ||||||
|         remote_base_path=False, |  | ||||||
|         debug=False): |  | ||||||
| 
 |  | ||||||
|         self.target_host = target_host |  | ||||||
| 
 |  | ||||||
|         # Base and Temp Base  |  | ||||||
|         if base_path: |  | ||||||
|             self.base_path = base_path |  | ||||||
|         else: |  | ||||||
|             self.base_path = os.path.abspath( |  | ||||||
|                 os.path.join(os.path.dirname(__file__), |  | ||||||
|                     os.pardir, |  | ||||||
|                     os.pardir)) |  | ||||||
|          |  | ||||||
| 
 |  | ||||||
|         # Local input directories |  | ||||||
|         self.cache_path              = os.path.join(self.base_path, "cache", target_host) |  | ||||||
|         self.conf_path               = os.path.join(self.base_path, "conf") |  | ||||||
| 
 |  | ||||||
|         self.global_explorer_path    = os.path.join(self.conf_path, "explorer") |  | ||||||
|         self.manifest_path           = os.path.join(self.conf_path, "manifest") |  | ||||||
|         self.type_base_path          = os.path.join(self.conf_path, "type") |  | ||||||
|         self.lib_path                = os.path.join(self.base_path, "lib") |  | ||||||
| 
 |  | ||||||
|         if initial_manifest: |  | ||||||
|             self.initial_manifest = initial_manifest |  | ||||||
|         else: |  | ||||||
|             self.initial_manifest = os.path.join(self.manifest_path, "init") |  | ||||||
| 
 |  | ||||||
|         # Local output directories |  | ||||||
|         if out_path: |  | ||||||
|             self.out_path = out_path |  | ||||||
|         else: |  | ||||||
|             self.out_path = os.path.join(tempfile.mkdtemp(), "out") |  | ||||||
| 
 |  | ||||||
|         self.bin_path                 = os.path.join(self.out_path, "bin") |  | ||||||
|         self.global_explorer_out_path = os.path.join(self.out_path, "explorer") |  | ||||||
|         self.object_base_path         = os.path.join(self.out_path, "object") |  | ||||||
| 
 |  | ||||||
|         # Remote directories |  | ||||||
|         if remote_base_path: |  | ||||||
|             self.remote_base_path = remote_base_path |  | ||||||
|         else: |  | ||||||
|             self.remote_base_path = "/var/lib/cdist" |  | ||||||
| 
 |  | ||||||
|         self.remote_conf_path            = os.path.join(self.remote_base_path, "conf") |  | ||||||
|         self.remote_object_path          = os.path.join(self.remote_base_path, "object") |  | ||||||
| 
 |  | ||||||
|         self.remote_type_path            = os.path.join(self.remote_conf_path, "type") |  | ||||||
|         self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer") |  | ||||||
| 
 |  | ||||||
|         # Create directories |  | ||||||
|         self.__init_out_paths() |  | ||||||
|         self.__init_remote_paths() |  | ||||||
| 
 |  | ||||||
|     def cleanup(self): |  | ||||||
|         # Do not use in __del__: |  | ||||||
|         # http://docs.python.org/reference/datamodel.html#customization |  | ||||||
|         # "other globals referenced by the __del__() method may already have been deleted  |  | ||||||
|         # or in the process of being torn down (e.g. the import machinery shutting down)" |  | ||||||
|         # |  | ||||||
|         log.debug("Saving " + self.out_path + " to " + self.cache_path) |  | ||||||
|         # Remove previous cache |  | ||||||
|         if os.path.exists(self.cache_path): |  | ||||||
|             shutil.rmtree(self.cache_path) |  | ||||||
|         shutil.move(self.out_path, self.cache_path) |  | ||||||
| 
 |  | ||||||
|     def __init_out_paths(self): |  | ||||||
|         """Initialise output directory structure""" |  | ||||||
| 
 |  | ||||||
|         # Create base dir, if user supplied and not existing |  | ||||||
|         if not os.path.isdir(self.base_path): |  | ||||||
|             os.mkdir(self.base_path) |  | ||||||
|              |  | ||||||
|         os.mkdir(self.out_path) |  | ||||||
|         os.mkdir(self.global_explorer_out_path) |  | ||||||
|         os.mkdir(self.bin_path) |  | ||||||
| 
 |  | ||||||
|     def __init_remote_paths(self): |  | ||||||
|         """Initialise remote directory structure""" |  | ||||||
| 
 |  | ||||||
|         self.remove_remote_path(self.remote_base_path) |  | ||||||
|         self.remote_mkdir(self.remote_base_path) |  | ||||||
|         self.remote_mkdir(self.remote_conf_path) |  | ||||||
| 
 |  | ||||||
|     def remote_mkdir(self, directory): |  | ||||||
|         """Create directory on remote side""" |  | ||||||
|         cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) |  | ||||||
| 
 |  | ||||||
|     def remove_remote_path(self, destination): |  | ||||||
|         cdist.exec.run_or_fail(["rm", "-rf",  destination], remote_prefix=True) |  | ||||||
| 
 |  | ||||||
|     def transfer_path(self, source, destination): |  | ||||||
|         """Transfer directory and previously delete the remote destination""" |  | ||||||
|         self.remove_remote_path(destination) |  | ||||||
|         cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + |  | ||||||
|             ["-r", source, self.target_host + ":" + destination]) |  | ||||||
|  | @ -30,19 +30,32 @@ log = logging.getLogger(__name__) | ||||||
| 
 | 
 | ||||||
| def run(argv): | def run(argv): | ||||||
|     """Emulate type commands (i.e. __file and co)""" |     """Emulate type commands (i.e. __file and co)""" | ||||||
|  |     global_path = os.environ['__global'] | ||||||
|  |     object_source = os.environ['__cdist_manifest'] | ||||||
|  |     target_host = os.environ['__target_host'] | ||||||
|  |     type_name = os.path.basename(argv[0]) | ||||||
|  | 
 | ||||||
|  |     # Logsetup - FIXME: add object_fq as soon as setup! | ||||||
|  |     #id = target_host + ": " + cdist_type + '/' + object_id  | ||||||
|  |     id = target_host + ": " | ||||||
|  |     # logformat = '%(levelname)s: ' + target_host + ": " + cdist_type + '/' + object_id + ': %(message)s' | ||||||
|  |     logformat = '%(levelname)s: ' + id + ': %(message)s' | ||||||
|  |     logging.basicConfig(format=logformat) | ||||||
|  | 
 | ||||||
|     if '__debug' in os.environ: |     if '__debug' in os.environ: | ||||||
|         logging.root.setLevel(logging.DEBUG) |         logging.root.setLevel(logging.DEBUG) | ||||||
|     else: |     else: | ||||||
|         logging.basicConfig(level=logging.INFO) |         logging.root.setLevel(logging.INFO) | ||||||
| 
 |  | ||||||
|     global_path = os.environ['__global'] |  | ||||||
|     object_source = os.environ['__cdist_manifest'] |  | ||||||
|     type_name = os.path.basename(argv[0]) |  | ||||||
| 
 | 
 | ||||||
|     object_base_path = os.path.join(global_path, "object") |     object_base_path = os.path.join(global_path, "object") | ||||||
|     type_base_path = os.environ['__cdist_type_base_path'] |     type_base_path = os.environ['__cdist_type_base_path'] | ||||||
|     cdist_type = core.Type(type_base_path, type_name) |     cdist_type = core.Type(type_base_path, type_name) | ||||||
| 
 | 
 | ||||||
|  |     if '__install' in os.environ: | ||||||
|  |         if not cdist_type.is_install: | ||||||
|  |             log.debug("Running in install mode, ignoring non install type") | ||||||
|  |             return True | ||||||
|  | 
 | ||||||
|     parser = argparse.ArgumentParser(add_help=False) |     parser = argparse.ArgumentParser(add_help=False) | ||||||
| 
 | 
 | ||||||
|     for parameter in cdist_type.optional_parameters: |     for parameter in cdist_type.optional_parameters: | ||||||
|  | @ -72,15 +85,15 @@ def run(argv): | ||||||
|     # Instantiate the cdist object whe are defining |     # Instantiate the cdist object whe are defining | ||||||
|     cdist_object = core.Object(cdist_type, object_base_path, object_id) |     cdist_object = core.Object(cdist_type, object_base_path, object_id) | ||||||
| 
 | 
 | ||||||
|     # Prefix output by object_self |  | ||||||
|     logformat = '%%(levelname)s: %s: %%(message)s' % cdist_object.path |  | ||||||
|     logging.basicConfig(format=logformat) |  | ||||||
| 
 |  | ||||||
|     # FIXME: verify object id |     # FIXME: verify object id | ||||||
|     log.debug(args) |     log.debug('#### emulator args: %s' % args) | ||||||
| 
 | 
 | ||||||
|     # Create object with given parameters |     # Create object with given parameters | ||||||
|     parameters = vars(args) |     parameters = {} | ||||||
|  |     for key,value in vars(args).items(): | ||||||
|  |         if value is not None: | ||||||
|  |             parameters[key] = value | ||||||
|  |      | ||||||
|     if cdist_object.exists: |     if cdist_object.exists: | ||||||
|         if cdist_object.parameters != parameters: |         if cdist_object.parameters != parameters: | ||||||
|             raise cdist.Error("Object %s already exists with conflicting parameters:\n%s: %s\n%s: %s" |             raise cdist.Error("Object %s already exists with conflicting parameters:\n%s: %s\n%s: %s" | ||||||
|  | @ -97,7 +110,6 @@ def run(argv): | ||||||
|         cdist_object.requirements.extend(requirements.split(" ")) |         cdist_object.requirements.extend(requirements.split(" ")) | ||||||
| 
 | 
 | ||||||
|     # Record / Append source |     # Record / Append source | ||||||
|     # FIXME: source should be list |  | ||||||
|     cdist_object.source.append(object_source) |     cdist_object.source.append(object_source) | ||||||
| 
 | 
 | ||||||
|     log.debug("Finished %s %s" % (cdist_object.path, parameters)) |     log.debug("Finished %s %s" % (cdist_object.path, parameters)) | ||||||
|  |  | ||||||
|  | @ -34,7 +34,7 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): | ||||||
| 
 | 
 | ||||||
|     if remote_prefix: |     if remote_prefix: | ||||||
|         remote_prefix = os.environ['__remote_exec'].split() |         remote_prefix = os.environ['__remote_exec'].split() | ||||||
|         remote_prefix.append(os.environ['target_host']) |         remote_prefix.append(os.environ['__target_host']) | ||||||
|         args[0][:0] = remote_prefix |         args[0][:0] = remote_prefix | ||||||
| 
 | 
 | ||||||
|     log.debug("Shell exec cmd: %s", args) |     log.debug("Shell exec cmd: %s", args) | ||||||
|  | @ -65,7 +65,7 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): | ||||||
| def run_or_fail(*args, remote_prefix=False, **kargs): | def run_or_fail(*args, remote_prefix=False, **kargs): | ||||||
|     if remote_prefix: |     if remote_prefix: | ||||||
|         remote_prefix = os.environ['__remote_exec'].split() |         remote_prefix = os.environ['__remote_exec'].split() | ||||||
|         remote_prefix.append(os.environ['target_host']) |         remote_prefix.append(os.environ['__target_host']) | ||||||
|         args[0][:0] = remote_prefix |         args[0][:0] = remote_prefix | ||||||
| 
 | 
 | ||||||
|     log.debug("Exec: " + " ".join(*args)) |     log.debug("Exec: " + " ".join(*args)) | ||||||
|  |  | ||||||
|  | @ -20,14 +20,14 @@ | ||||||
| # | # | ||||||
| # | # | ||||||
| 
 | 
 | ||||||
| import logging | import os | ||||||
| 
 |  | ||||||
| import cdist.config_install | import cdist.config_install | ||||||
| 
 | 
 | ||||||
| log = logging.getLogger(__name__) | class Install(cdist.config_install.ConfigInstall): | ||||||
|  |     def __init__(self, *args, **kargs): | ||||||
|  |         """Enhance config install with install support""" | ||||||
| 
 | 
 | ||||||
|  |         # Setup environ to be used in emulator | ||||||
|  |         os.environ['__install'] = "yes" | ||||||
| 
 | 
 | ||||||
| def install(args): |         super().__init__(*args, **kargs) | ||||||
|     """Install remote system""" |  | ||||||
|     process = {} |  | ||||||
| 
 |  | ||||||
|  |  | ||||||
|  | @ -30,9 +30,6 @@ cdist_commands=["banner", "config", "install"] | ||||||
| cdist_exec_path = os.path.abspath( | cdist_exec_path = os.path.abspath( | ||||||
|     os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../bin/cdist")) |     os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../bin/cdist")) | ||||||
| 
 | 
 | ||||||
| def exec(): |  | ||||||
|     print(cdist_exec_path) |  | ||||||
| 
 |  | ||||||
| #class UI(unittest.TestCase): | #class UI(unittest.TestCase): | ||||||
| #    def test_banner(self): | #    def test_banner(self): | ||||||
| #        self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) | #        self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) | ||||||
|  |  | ||||||
|  | @ -29,20 +29,22 @@ sys.path.insert(0, os.path.abspath( | ||||||
|         os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) |         os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) | ||||||
| 
 | 
 | ||||||
| import cdist.config | import cdist.config | ||||||
| 
 | import cdist.test | ||||||
| cdist_exec_path = os.path.abspath( |  | ||||||
|     os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) |  | ||||||
| 
 |  | ||||||
| 
 | 
 | ||||||
| class Install(unittest.TestCase): | class Install(unittest.TestCase): | ||||||
|     def setUp(self): |     def setUp(self): | ||||||
|         self.temp_dir = tempfile.mkdtemp() |         self.temp_dir = tempfile.mkdtemp() | ||||||
|         self.init_manifest = os.path.join(self.temp_dir, "manifest") |         self.init_manifest = os.path.join(self.temp_dir, "manifest") | ||||||
|  | 
 | ||||||
|  |         os.environ['__remote_exec'] = "ssh -o User=root -q" | ||||||
|  |         os.environ['__remote_copy'] = "scp -o User=root -q" | ||||||
|  | 
 | ||||||
|         self.config = cdist.config.Config("localhost", |         self.config = cdist.config.Config("localhost", | ||||||
|                             initial_manifest=self.init_manifest, |                             initial_manifest=self.init_manifest, | ||||||
|                             exec_path=cdist_exec_path) |                             exec_path=cdist.test.cdist_exec_path) | ||||||
|         self.config.link_emulator() |         self.config.link_emulator() | ||||||
| 
 | 
 | ||||||
|  | 
 | ||||||
| ### NEW FOR INSTALL ############################################################ | ### NEW FOR INSTALL ############################################################ | ||||||
| 
 | 
 | ||||||
|     def test_explorer_ran(self): |     def test_explorer_ran(self): | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue