From 6d430524f1a0ee661dbd1c4af1aac6d721a0d064 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 19 Nov 2012 12:04:07 +0100 Subject: [PATCH 1/6] binary all the way, fixes issue #138 Signed-off-by: Steven Armstrong --- cdist/emulator.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cdist/emulator.py b/cdist/emulator.py index 42744fab..a9c760cb 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -29,7 +29,7 @@ import cdist from cdist import core class Emulator(object): - def __init__(self, argv, stdin=sys.stdin, env=os.environ): + def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ): self.argv = argv self.stdin = stdin self.env = env @@ -151,6 +151,10 @@ class Emulator(object): # Record / Append source self.cdist_object.source.append(self.object_source) + chunk_size = 65536 + def _read_stdin(self): + return self.stdin.read(self.chunk_size) + def save_stdin(self): """If something is written to stdin, save it in the object as $__object/stdin so it can be accessed in manifest and gencode-* @@ -160,10 +164,12 @@ class Emulator(object): try: # go directly to file instead of using CdistObject's api # as that does not support streaming - # FIXME: no streaming needed anymore - use a raw file (not yet there?) path = os.path.join(self.cdist_object.absolute_path, 'stdin') - with open(path, 'w') as fd: - fd.write(self.stdin.read()) + with open(path, 'wb') as fd: + chunk = self._read_stdin() + while chunk: + fd.write(chunk) + chunk = self._read_stdin() except EnvironmentError as e: raise cdist.Error('Failed to read from stdin: %s' % e) From 84770b9ef2afbfa1232845ce6538b56df99cee3a Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 19 Nov 2012 11:25:56 +0100 Subject: [PATCH 2/6] implement before/after to declare dependencies and deprecate require Signed-off-by: Steven Armstrong --whitespace Signed-off-by: Steven Armstrong --- cdist/core/cdist_object.py | 2 ++ cdist/emulator.py | 25 +++++++++++++++++++++---- cdist/resolver.py | 17 ++++++++++++++--- cdist/test/object/__init__.py | 1 + 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 90a21e59..7e587bf3 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -186,6 +186,8 @@ class CdistObject(object): return os.path.join(self.path, "explorer") requirements = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'require')) + before = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'before')) + after = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'after')) autorequire = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'autorequire')) parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.parameter_path)) explorers = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.explorer_path)) diff --git a/cdist/emulator.py b/cdist/emulator.py index a9c760cb..bd3661d3 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -23,6 +23,7 @@ import argparse import logging import os +import warnings import sys import cdist @@ -92,8 +93,12 @@ class Emulator(object): def commandline(self): """Parse command line""" + self.meta_parameters = dict.fromkeys(('after', 'before')) + meta_parser = argparse.ArgumentParser(add_help=False) + for meta_parameter in self.meta_parameters.keys(): + meta_parser.add_argument('--%s' % meta_parameter, action='append', required=False) - parser = argparse.ArgumentParser(add_help=False, argument_default=argparse.SUPPRESS) + parser = argparse.ArgumentParser(add_help=False, parents=[meta_parser], argument_default=argparse.SUPPRESS) for parameter in self.cdist_type.required_parameters: argument = "--" + parameter @@ -119,6 +124,11 @@ class Emulator(object): self.args = parser.parse_args(self.argv[1:]) self.log.debug('Args: %s' % self.args) + # Handle meta parameters + for meta_parameter in self.meta_parameters.keys(): + if meta_parameter in self.args: + self.meta_parameters[meta_parameter] = getattr(self.args, meta_parameter) + delattr(self.args, meta_parameter) def setup_object(self): # Setup object_id - FIXME: unset / do not setup anymore! @@ -175,10 +185,18 @@ class Emulator(object): def record_requirements(self): """record requirements""" + for key in ('before', 'after'): + if key in self.meta_parameters and self.meta_parameters[key]: + for value in self.meta_parameters[key]: + self.log.debug("Recording requirement: %s %s %s", self.cdist_object.name, key, value) + dependency_list = getattr(self.cdist_object, key) + # append to the object.after or object.before lists + dependency_list.append(value) if "require" in self.env: + warnings.warn("The 'require' envrionment variable is deprecated. Use the --before and --after meta parameters to define dependencies.", category=PendingDeprecationWarning, stacklevel=2) + requirements = self.env['require'] - self.log.debug("reqs = " + requirements) for requirement in requirements.split(" "): # Ignore empty fields - probably the only field anyway if len(requirement) == 0: continue @@ -187,11 +205,10 @@ class Emulator(object): cdist_object = self.cdist_object.object_from_name(requirement) self.log.debug("Recording requirement: " + requirement) - # Save the sanitised version, not the user supplied one # (__file//bar => __file/bar) # This ensures pattern matching is done against sanitised list - self.cdist_object.requirements.append(cdist_object.name) + self.cdist_object.after.append(cdist_object.name) def record_auto_requirements(self): """An object shall automatically depend on all objects that it defined in it's type manifest. diff --git a/cdist/resolver.py b/cdist/resolver.py index 7e3a1a68..d181d066 100644 --- a/cdist/resolver.py +++ b/cdist/resolver.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2012 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -109,10 +109,21 @@ class DependencyResolver(object): raise RequirementNotFoundError(pattern) def _preprocess_requirements(self): - """Find all autorequire dependencies and merge them to be just requirements - for further processing. + """Find all before, after and autorequire dependencies and merge them + to be just requirements for further processing. """ for cdist_object in self.objects.values(): + if cdist_object.after: + cdist_object.requirements.extend(cdist_object.after) + # As we changed the object on disc, we have to ensure it is not + # preprocessed again if someone would call us multiple times. + cdist_object.after = [] + if cdist_object.before: + for other_object in self.find_requirements_by_name(cdist_object.before): + other_object.requirements.append(cdist_object.name) + # As we changed the object on disc, we have to ensure it is not + # preprocessed again if someone would call us multiple times. + cdist_object.before = [] if cdist_object.autorequire: # The objects (children) that this cdist_object (parent) defined # in it's type manifest shall inherit all explicit requirements diff --git a/cdist/test/object/__init__.py b/cdist/test/object/__init__.py index 3a91f709..7bdc037e 100644 --- a/cdist/test/object/__init__.py +++ b/cdist/test/object/__init__.py @@ -87,6 +87,7 @@ class ObjectTestCase(test.CdistTestCase): self.cdist_object.code_local = '' self.cdist_object.code_remote = '' self.cdist_object.state = '' + self.cdist_object.requirements = [] def test_name(self): self.assertEqual(self.cdist_object.name, '__third/moon') From 7f0ae7928a3e2bb37c7db20d23251fdd156c8f0a Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 19 Nov 2012 13:04:57 +0100 Subject: [PATCH 3/6] fix state explorer to properly detect already existing ppa Signed-off-by: Steven Armstrong --- cdist/conf/type/__apt_ppa/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_ppa/explorer/state b/cdist/conf/type/__apt_ppa/explorer/state index 2d8ca7c5..2bb4f65a 100755 --- a/cdist/conf/type/__apt_ppa/explorer/state +++ b/cdist/conf/type/__apt_ppa/explorer/state @@ -26,7 +26,7 @@ name="$__object_id" . /etc/lsb-release repo_name="${name#ppa:}" -repo_file_name="$(echo "$repo_name" | sed "s:\/:\-:")-${DISTRIB_CODENAME}.list" +repo_file_name="$(echo "$repo_name" | sed -e "s|[/:]|-|" -e "s|\.|_|")-${DISTRIB_CODENAME}.list" [ -s "/etc/apt/sources.list.d/${repo_file_name}" ] \ && echo present || echo absent From a8d4544c46bd54f920bff016ae372ccf56aa913f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Nov 2012 13:28:56 +0100 Subject: [PATCH 4/6] ++changes Signed-off-by: Nico Schottelius --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 68b35eac..d2c5f5fe 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Type __rvm_gemset: Change parameter "default" to be boolean * New Type: __user_groups (Steven Armstrong) * Type __user: Remove --groups support (now provided by __user_groups) + * Core: Support for --after and --before parameters 2.1.0pre8: 2012-11-15 * Type cleanup: __apt_ppa, __apt_ppa_update_index, __file, From 52368b0116106dd959bb5d90f88a6bb328ccf32b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Nov 2012 13:35:40 +0100 Subject: [PATCH 5/6] ++changes, ++update notes Signed-off-by: Nico Schottelius --- docs/changelog | 1 + docs/web/cdist/update.mdwn | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index d2c5f5fe..943d83e4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Type __rvm_gemset: Change parameter "default" to be boolean * New Type: __user_groups (Steven Armstrong) * Type __user: Remove --groups support (now provided by __user_groups) + * Type __apt_ppa: Bugfix: Installeded ppa detection (Steven Armstrong) * Core: Support for --after and --before parameters 2.1.0pre8: 2012-11-15 diff --git a/docs/web/cdist/update.mdwn b/docs/web/cdist/update.mdwn index 22b92769..3331ec85 100644 --- a/docs/web/cdist/update.mdwn +++ b/docs/web/cdist/update.mdwn @@ -42,6 +42,7 @@ To upgrade to the lastet version do * The types **\_\_autofs**, **\_\_autofs_map** and **\_\_autofs_reload** have been removed (no maintainer, no users) * Type **\_\_user**: Parameter --groups removed (use the new \_\_user_groups type) + * require="" is deprecated: Use --after and --before as parameters instead ### Updating from 1.7 to 2.0 From 4718b81a08c4b95ad76a654723d655a7ee62a324 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Nov 2012 13:40:35 +0100 Subject: [PATCH 6/6] Documentation cleanup (old pre-boolean variant) Signed-off-by: Nico Schottelius --- cdist/conf/type/__rvm_gem/man.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__rvm_gem/man.text b/cdist/conf/type/__rvm_gem/man.text index fbf9a622..2b72e7ae 100644 --- a/cdist/conf/type/__rvm_gem/man.text +++ b/cdist/conf/type/__rvm_gem/man.text @@ -36,7 +36,7 @@ __rvm_gemset rails --gemset ruby-1.9.3-p0@myset --user bill --state present # Do the same and also make ruby-1.9.3-p0@myset the default gemset __rvm_gemset rails --gemset ruby-1.9.3-p0@myset --user bill \ - --state present --default yes + --state present --default # Remove it __rvm_ruby rails --gemset ruby-1.9.3-p0@myset --user bill --state absent