From 7a41d6d8fa283fcd9faf3877f23dd3d18dcc1ea3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 20 Jan 2013 18:11:47 +0100 Subject: [PATCH 01/22] __file: notify when doing changes Signed-off-by: Nico Schottelius --- cdist/conf/type/__file/gencode-local | 1 + cdist/conf/type/__file/gencode-remote | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 087011c4..99456c5b 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -41,6 +41,7 @@ if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then if [ "$local_cksum" != "$remote_cksum" ]; then echo "$__remote_copy" "$source" "${__target_host}:${destination}" + echo "copy" >> "$__object/notifications" fi else echo "Source \"$source\" does not exist." >&2 diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 8b03e919..fa2adc6f 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -38,17 +38,20 @@ case "$state_should" in # Group if [ -f "$__object/parameter/group" ]; then echo chgrp \"$(cat "$__object/parameter/group")\" \"$destination\" + echo "chgrp" >> "$__object/notifications" fi # Owner if [ -f "$__object/parameter/owner" ]; then echo chown \"$(cat "$__object/parameter/owner")\" \"$destination\" + echo "chown" >> "$__object/notifications" fi # Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) if [ -f "$__object/parameter/mode" ]; then echo chmod \"$(cat "$__object/parameter/mode")\" \"$destination\" + echo "chmod" >> "$__object/notifications" fi ;; @@ -56,6 +59,7 @@ case "$state_should" in if [ "$exists" = "yes" ]; then echo rm -f \"$destination\" + echo "removal" >> "$__object/notifications" fi ;; From b06122f3a1314da4a82a2554898c70da6b4e3c59 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 20 Jan 2013 18:21:40 +0100 Subject: [PATCH 02/22] document notifications Signed-off-by: Nico Schottelius --- docs/dev/logs/2013-01-20.notifications | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docs/dev/logs/2013-01-20.notifications diff --git a/docs/dev/logs/2013-01-20.notifications b/docs/dev/logs/2013-01-20.notifications new file mode 100644 index 00000000..be326196 --- /dev/null +++ b/docs/dev/logs/2013-01-20.notifications @@ -0,0 +1,20 @@ +Allow cross-type communication + +Sending notifications is possible from + + - manifest + - gencode-local + - gencode-remote + +Sending a notification from an object means writing to the file "notifications" into +its object: + + echo mytest >> "$__object/notifications" # a type reports something + +Reading / Reacting on notifications works by accessing the file +referred to be "$__notifications". All notifications are prefixed with +the object name ($__object_name) and are appended into this file. + +To find out, whether a file was copied, run: + + grep __file/etc/passwd:copy "$__notifications" From fb630437dbd73900e209d8c970c1d58df7aa8292 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 20 Jan 2013 21:31:08 +0100 Subject: [PATCH 03/22] ++triggers Signed-off-by: Nico Schottelius --- docs/dev/logs/2013-01-20.triggers | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/dev/logs/2013-01-20.triggers diff --git a/docs/dev/logs/2013-01-20.triggers b/docs/dev/logs/2013-01-20.triggers new file mode 100644 index 00000000..dc13c746 --- /dev/null +++ b/docs/dev/logs/2013-01-20.triggers @@ -0,0 +1,49 @@ +An alternative / complementary approach to notifications: triggers (or actions?) + +A type may support various actions by creating files in its subdirectory +"actions". Other types can trigger an action of a different type or object +by calling them (indirectly?): + + +if grep "__file/etc/nginx/conf.d/.*:copy" "$__notifications"; then + + # Call action from a type + cdist trigger __nginx/reload +fi + + +Not sure whether this approach (calling "actions" of other types) is sane, +as nginx should probably better know if it should be restarted "itself". + + +-------------------------------------------------------------------------------- + +Alternate approach: + +__nginx_vhost www.some-domain.ch --custom << eof +some custom code for __nginx_vhost inclusion +eof + +__nginx_vhost: + + manifest: + # __nginx_vhost requires __nginx: creates directories + + require"$__object_name" __nginx --require-only + + # Do WE or __file ... depend on nginx? + cdist require __nginx + + # Create file that contains the giving code + __file /etc/nginx/conf.d/www.some-domain.ch + + require="__nginx" __file /etc/nginx/conf.d/www.some-domain.ch + +__nginx: + manifest: + __package nginx --state present + + __file some-custom-files + + gencode-remote: + if first_install or file changed: From d1a569fecd0b48ef7bac35ed5e01621c58d2ad28 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 25 Nov 2013 23:18:19 +0100 Subject: [PATCH 04/22] remove bug comments -> no bug here Signed-off-by: Nico Schottelius --- cdist/core/code.py | 5 +---- cdist/core/manifest.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index d5f59094..910d1c47 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -119,9 +119,6 @@ class Code(object): source = os.path.join(self.local.object_path, cdist_object.code_remote_path) destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path) # FIXME: BUG: do not create destination, but top level of destination! - # FIXME: BUG2: we are called AFTER the code-remote has been transferred already: - # mkdir: cannot create directory `/var/lib/cdist/object/__directory/etc/acpi/actions/.cdist/code-remote': File exists - # OR: this is from previous run -> cleanup missing! self.remote.mkdir(destination) self.remote.transfer(source, destination) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 97121474..6bb33bb3 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # From ca1c5ff71360459c90147383a229c1873c912ca7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Nov 2013 00:25:00 +0100 Subject: [PATCH 05/22] add another log for notifications Signed-off-by: Nico Schottelius --- docs/dev/logs/2013-11-25.notifications | 50 ++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/dev/logs/2013-11-25.notifications diff --git a/docs/dev/logs/2013-11-25.notifications b/docs/dev/logs/2013-11-25.notifications new file mode 100644 index 00000000..33c6f31b --- /dev/null +++ b/docs/dev/logs/2013-11-25.notifications @@ -0,0 +1,50 @@ +Follow up from 2013-01-20: + + - (re-)create message file per object? + - yes, but do not necessarily save in object space + - save $anywhere + + - object_run + - current notifications are imported into a file available at $__messages_in + - after object run, everything that has been written to $__messages_out is merged into the $__messages file + + - functions: + self.explorer.run_global_explorers(self.local.global_explorer_out_path) + self.manifest.run_initial_manifest(self.local.initial_manifest) + self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest)) + self.explorer.run_type_explorers(cdist_object) + self.manifest.run_type_manifest(cdist_object) + self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object)) + self.code.run_gencode_local(cdist_object) + self.local.run_script(script, env=env, return_output=True) + self.code.run_gencode_remote(cdist_object) + self.local.run_script(script, env=env, return_output=True) + + + - message support in ... + - initialmanifest - yes + - explorer - no + - only locally - yes + + - how to use notification / messaging in cdist + - can be used in all local scripts: + - initial manifest + - type manifest + - type gencode-* + - order of object exeution is random or as you requested using require="" + + - example use: + +__file/gencode-local: + if [ "$local_cksum" != "$remote_cksum" ]; then + echo "$__remote_copy" "$source" "${__target_host}:${destination}" + echo "copy" >> "$__messages_out" + fi + +__nginx/manifest: + __file /etc/nginx/sites-enabled/myfile --source "$__type/files/nginx-config" + +__nginx/gencode-remote: + if grep -q "__file/etc/nginx/sites-enabled/myfile:copy" "$__messages_in"; then + echo /etc/init.d/nginx restart + fi From 22a83d2c9365ce757d6d01fd70bf97b7e7a52f83 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Nov 2013 00:27:08 +0100 Subject: [PATCH 06/22] add new message object Signed-off-by: Nico Schottelius --- cdist/core/message.py | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 cdist/core/message.py diff --git a/cdist/core/message.py b/cdist/core/message.py new file mode 100644 index 00000000..50f039f4 --- /dev/null +++ b/cdist/core/message.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import shutil +import tempfile + +import cdist + +log = logging.getLogger(__name__) + + +class Message(object): + """Support messaging between types + + """ + def __init__(self, prefix, global_messages): + self.prefix = prefix + self.global_messages = global_messages + + self.messages_in = tempfile.mkstemp(suffix='.cdist_message_in') + self.messages_out = tempfile.mkstemp(suffix='.cdist_message_out') + + shutil.copyfile(self.global_messages, self.messages_in) + + @property + def env(self, env): + env = {} + env['__messages_in'] = self.messages_in + env['__messages_out'] = self.messages_out + + return env + + def _cleanup(self): + os.remove(self.messages_in) + os.remove(self.messages_out) + + def _merge_messages(self): + with open(self.messages_in) as fd: + content = fd.readlines() + + with open(self.global_messages, 'a') as fd: + for line in content: + fd.write("%s:%s" % (self.prefix, line)) + + def merge_messages(self): + self._merge_messages() + self._cleanup() From ac5fa7cd6461717e6f82fc10dece2b16405b38b8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Nov 2013 01:01:30 +0100 Subject: [PATCH 07/22] integrate messaging into cdist Signed-off-by: Nico Schottelius --- cdist/exec/local.py | 11 ++++- cdist/{core => }/message.py | 26 +++++++---- cdist/test/message/__init__.py | 82 ++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 10 deletions(-) rename cdist/{core => }/message.py (77%) create mode 100644 cdist/test/message/__init__.py diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 3a3ac706..b0109a4e 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -44,9 +44,11 @@ class Local(object): exec_path=sys.argv[0], initial_manifest=None, base_path=None, - add_conf_dirs=None): + add_conf_dirs=None, + message_prefix=None): self.target_host = target_host + self.message_prefix=message_prefix # FIXME: stopped: create base that does not require moving later if base_path: @@ -92,6 +94,7 @@ class Local(object): self.conf_path = os.path.join(self.base_path, "conf") self.global_explorer_out_path = os.path.join(self.base_path, "explorer") self.object_path = os.path.join(self.base_path, "object") + self.messages_path = os.path.join(self.base_path, "messages") # Depending on conf_path self.global_explorer_path = os.path.join(self.conf_path, "explorer") @@ -163,6 +166,9 @@ class Local(object): # Export __target_host for use in __remote_{copy,exec} scripts env['__target_host'] = self.target_host + if self.message_prefix: + message = cdist.Message(self.message_prefix, self.messages_path) + try: if return_output: return subprocess.check_output(command, env=env).decode() @@ -172,6 +178,9 @@ class Local(object): raise cdist.Error("Command failed: " + " ".join(command)) except OSError as error: raise cdist.Error(" ".join(*args) + ": " + error.args[1]) + finally: + if self.message_prefix: + message.merge_messages() def run_script(self, script, env=None, return_output=False): """Run the given script with the given environment. diff --git a/cdist/core/message.py b/cdist/message.py similarity index 77% rename from cdist/core/message.py rename to cdist/message.py index 50f039f4..057c43d3 100644 --- a/cdist/core/message.py +++ b/cdist/message.py @@ -20,6 +20,7 @@ # import logging +import os import shutil import tempfile @@ -32,29 +33,36 @@ class Message(object): """Support messaging between types """ - def __init__(self, prefix, global_messages): + def __init__(self, prefix, messages): self.prefix = prefix - self.global_messages = global_messages + self.global_messages = messages - self.messages_in = tempfile.mkstemp(suffix='.cdist_message_in') - self.messages_out = tempfile.mkstemp(suffix='.cdist_message_out') + self.messages_in = tempfile.mkstemp(suffix='.cdist_message_in')[1] + self.messages_out = tempfile.mkstemp(suffix='.cdist_message_out')[1] + + self._copy_messages() - shutil.copyfile(self.global_messages, self.messages_in) @property - def env(self, env): + def env(self): env = {} env['__messages_in'] = self.messages_in env['__messages_out'] = self.messages_out return env + def _copy_messages(self): + """Copy global contents into our copy""" + shutil.copyfile(self.global_messages, self.messages_in) + def _cleanup(self): - os.remove(self.messages_in) - os.remove(self.messages_out) + if os.path.exists(self.messages_in): + os.remove(self.messages_in) + if os.path.exists(self.messages_out): + os.remove(self.messages_out) def _merge_messages(self): - with open(self.messages_in) as fd: + with open(self.messages_out) as fd: content = fd.readlines() with open(self.global_messages, 'a') as fd: diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py new file mode 100644 index 00000000..653847f1 --- /dev/null +++ b/cdist/test/message/__init__.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import tempfile + +from cdist import test +import cdist.message + +class MessageTestCase(test.CdistTestCase): + + def setUp(self): + self.prefix="cdist-test" + self.content = "A very short story" + self.tempfile = tempfile.mkstemp()[1] + self.message = cdist.message.Message(prefix=self.prefix, + messages=self.tempfile) + + def tearDown(self): + os.remove(self.tempfile) + self.message._cleanup() + + def test_env(self): + """ + Ensure environment is correct + """ + + env = self.message.env + + self.assertIn('__messages_in', env) + self.assertIn('__messages_out', env) + + + def test_copy_content(self): + """ + Ensure content copying is working + """ + + with open(self.tempfile, "w") as fd: + fd.write(self.content) + + self.message._copy_messages() + + with open(self.tempfile, "r") as fd: + testcontent = fd.read() + + self.assertEqual(self.content, testcontent) + + def test_message_merge_prefix(self): + """Ensure messages are merged and are prefixed""" + + expectedcontent = "%s:%s" % (self.prefix, self.content) + + out = self.message.env['__messages_out'] + + with open(out, "w") as fd: + fd.write(self.content) + + self.message._merge_messages() + + with open(self.tempfile, "r") as fd: + testcontent = fd.read() + + self.assertEqual(expectedcontent, testcontent) From edec2abb1d583a12fa31f4645f8bd6037d8edb27 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Nov 2013 01:01:44 +0100 Subject: [PATCH 08/22] adopt first type: __file for messaging Signed-off-by: Nico Schottelius --- cdist/conf/type/__file/gencode-local | 2 +- cdist/conf/type/__file/gencode-remote | 11 +++++++---- cdist/conf/type/__file/man.text | 11 ++++++++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 99456c5b..8cd9b24e 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -41,7 +41,7 @@ if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then if [ "$local_cksum" != "$remote_cksum" ]; then echo "$__remote_copy" "$source" "${__target_host}:${destination}" - echo "copy" >> "$__object/notifications" + echo "copy" >> "$__messages_out" fi else echo "Source \"$source\" does not exist." >&2 diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index fa2adc6f..6e1fa5be 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -38,20 +38,23 @@ case "$state_should" in # Group if [ -f "$__object/parameter/group" ]; then echo chgrp \"$(cat "$__object/parameter/group")\" \"$destination\" - echo "chgrp" >> "$__object/notifications" +# FIXME: only if necessary, not if parameter is present +# echo "chgrp" >> "$__object/notifications" fi # Owner if [ -f "$__object/parameter/owner" ]; then echo chown \"$(cat "$__object/parameter/owner")\" \"$destination\" - echo "chown" >> "$__object/notifications" +# FIXME: only if necessary, not if parameter is present +# echo "chown" >> "$__object/notifications" fi # Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) if [ -f "$__object/parameter/mode" ]; then echo chmod \"$(cat "$__object/parameter/mode")\" \"$destination\" - echo "chmod" >> "$__object/notifications" +# FIXME: only if necessary, not if parameter is present +# echo "chmod" >> "$__object/notifications" fi ;; @@ -59,7 +62,7 @@ case "$state_should" in if [ "$exists" = "yes" ]; then echo rm -f \"$destination\" - echo "removal" >> "$__object/notifications" + echo "remove" >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__file/man.text b/cdist/conf/type/__file/man.text index 1c61fd51..9ac82d0f 100644 --- a/cdist/conf/type/__file/man.text +++ b/cdist/conf/type/__file/man.text @@ -41,6 +41,15 @@ source:: If not supplied, an empty file or directory will be created. If source is '-' (dash), take what was written to stdin as the file content. +MESSAGES +-------- + +copy:: + File was copied because cksum was different from local version + +remove:: + File exists, but state is absent, file will be removed by generated code. + EXAMPLES -------- @@ -81,5 +90,5 @@ SEE ALSO COPYING ------- -Copyright \(C) 2011-2012 Nico Schottelius. Free use of this software is +Copyright \(C) 2011-2013 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). From 2f842d56eb1e55cf0fbbc6fbe35dc2faa7832f57 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 26 Nov 2013 01:17:37 +0100 Subject: [PATCH 09/22] integrate messaging into gencode, manifest Signed-off-by: Nico Schottelius --- cdist/core/code.py | 3 ++- cdist/core/manifest.py | 4 +++- cdist/exec/local.py | 23 ++++++++++++++--------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index 910d1c47..f128697f 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -104,7 +104,8 @@ class Code(object): '__object_id': cdist_object.object_id, '__object_name': cdist_object.name, }) - return self.local.run_script(script, env=env, return_output=True) + message_prefix=cdist_object.name + return self.local.run_script(script, env=env, return_output=True, message_prefix=message_prefix) def run_gencode_local(self, cdist_object): """Run the gencode-local script for the given cdist object.""" diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 6bb33bb3..95bf4c25 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -122,7 +122,8 @@ class Manifest(object): if not os.path.isfile(initial_manifest): raise NoInitialManifestError(initial_manifest, user_supplied) - self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest)) + message_prefix="initialmanifest" + self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix) def env_type_manifest(self, cdist_object): type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path) @@ -141,5 +142,6 @@ class Manifest(object): def run_type_manifest(self, cdist_object): type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path) + message_prefix = cdist_object.name if os.path.isfile(type_manifest): self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object)) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index b0109a4e..28c50eec 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -30,6 +30,7 @@ import logging import tempfile import cdist +import cdist.message from cdist import core class Local(object): @@ -44,11 +45,9 @@ class Local(object): exec_path=sys.argv[0], initial_manifest=None, base_path=None, - add_conf_dirs=None, - message_prefix=None): + add_conf_dirs=None): self.target_host = target_host - self.message_prefix=message_prefix # FIXME: stopped: create base that does not require moving later if base_path: @@ -131,6 +130,7 @@ class Local(object): def create_files_dirs(self): self._init_directories() self._create_conf_path_and_link_conf_dirs() + self._create_messages() self._link_types_for_emulator() @@ -153,7 +153,7 @@ class Local(object): self.log.debug("Local mkdir: %s", path) os.makedirs(path, exist_ok=True) - def run(self, command, env=None, return_output=False): + def run(self, command, env=None, return_output=False, message_prefix=None): """Run the given command with the given environment. Return the output as a string. @@ -166,8 +166,8 @@ class Local(object): # Export __target_host for use in __remote_{copy,exec} scripts env['__target_host'] = self.target_host - if self.message_prefix: - message = cdist.Message(self.message_prefix, self.messages_path) + if message_prefix: + message = cdist.message.Message(message_prefix, self.messages_path) try: if return_output: @@ -179,10 +179,10 @@ class Local(object): except OSError as error: raise cdist.Error(" ".join(*args) + ": " + error.args[1]) finally: - if self.message_prefix: + if message_prefix: message.merge_messages() - def run_script(self, script, env=None, return_output=False): + def run_script(self, script, env=None, return_output=False, message_prefix=None): """Run the given script with the given environment. Return the output as a string. @@ -190,7 +190,7 @@ class Local(object): command = ["/bin/sh", "-e"] command.append(script) - return self.run(command, env, return_output) + return self.run(command=command, env=env, return_output=return_output, message_prefix=message_prefix) def save_cache(self): destination = os.path.join(self.cache_path, self.target_host) @@ -199,6 +199,11 @@ class Local(object): shutil.rmtree(destination) shutil.move(self.base_path, destination) + def _create_messages(self): + """Create empty messages""" + with open(self.messages_path, "w"): + pass + def _create_conf_path_and_link_conf_dirs(self): # Link destination directories for sub_dir in [ "explorer", "manifest", "type" ]: From d074713b94f026e4de78bb3c07b777a3066e7d85 Mon Sep 17 00:00:00 2001 From: Daniel Heule Date: Mon, 2 Dec 2013 13:27:42 +0100 Subject: [PATCH 10/22] Fix explorer and globalopts issue in type __package_zypper Fixes #215. Signed-off-by: Nico Schottelius --- cdist/conf/type/__package_zypper/explorer/pkg_version | 2 +- cdist/conf/type/__package_zypper/gencode-remote | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) mode change 100755 => 100644 cdist/conf/type/__package_zypper/explorer/pkg_version mode change 100755 => 100644 cdist/conf/type/__package_zypper/gencode-remote diff --git a/cdist/conf/type/__package_zypper/explorer/pkg_version b/cdist/conf/type/__package_zypper/explorer/pkg_version old mode 100755 new mode 100644 index fb3b7753..655b464d --- a/cdist/conf/type/__package_zypper/explorer/pkg_version +++ b/cdist/conf/type/__package_zypper/explorer/pkg_version @@ -27,4 +27,4 @@ else name="$__object_id" fi -rpm -q --whatprovides "$name" 2>/dev/null || true +rpm -q --whatprovides "$name" | grep -v 'no package provides' || true diff --git a/cdist/conf/type/__package_zypper/gencode-remote b/cdist/conf/type/__package_zypper/gencode-remote old mode 100755 new mode 100644 index ca9aec33..d1766126 --- a/cdist/conf/type/__package_zypper/gencode-remote +++ b/cdist/conf/type/__package_zypper/gencode-remote @@ -39,15 +39,22 @@ else state_should="present" fi +pkg_version="$(cat "$__object/explorer/pkg_version")" +if [ -z "$pkg_version" ]; then + state_is="absent" +else + state_is="present" +fi + # Exit if nothing is needed to be done [ "$state_is" = "$state_should" ] && exit 0 case "$state_should" in present) - echo zypper "$globalopts" install --auto-agree-with-licenses \"$name\" + echo zypper $globalopts install --auto-agree-with-licenses \"$name\" ">/dev/null" ;; absent) - echo pacman "$globalopts" remove \"$name\" + echo zypper $globalopts remove \"$name\" ">/dev/null" ;; *) echo "Unknown state: $state_should" >&2 From 1b36ed88f0b8d6b10ff7ae9478b83d0603bd0398 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 Dec 2013 13:29:17 +0100 Subject: [PATCH 11/22] ++changes(2.3.7) Signed-off-by: Nico Schottelius --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 67eaca1a..25079179 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ Changelog * Type __file: Only remove file when state is absent (Steven Armstrong) * Type __link: Only remove link when state is absent (Steven Armstrong) * Type __directory: Only remove directory when state is absent (Steven Armstrong) + * Type __package_zypper: Fix explorer and parameter issue (Daniel Heule) * Core: Fix backtrace when cache cannot be deleted 2.3.6: 2013-11-25 From 9d86f8c9b7f379c90522ce99d5abde3c43474531 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 Dec 2013 15:16:31 +0100 Subject: [PATCH 12/22] 2.3.7 release Signed-off-by: Nico Schottelius --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 25079179..b6122bec 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,7 +4,7 @@ Changelog * Changes are always commented with their author in (braces) * Exception: No braces means author == Nico Schottelius -2.3.7: +2.3.7: 2013-12-02 * Type __file: Secure the file transfer by using mktemp (Steven Armstrong) * Type __file: Only remove file when state is absent (Steven Armstrong) * Type __link: Only remove link when state is absent (Steven Armstrong) From a95167b374e4de8dd16423017a477e6336f2d05d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 Dec 2013 15:49:02 +0100 Subject: [PATCH 13/22] remove quotes from mkdiropt Signed-off-by: Nico Schottelius --- cdist/conf/type/__directory/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index 23fa4ed3..800fc6e4 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -75,7 +75,7 @@ case "$state_should" in set_attributes=1 cat << DONE rm -f "$destination" -mkdir "$mkdiropt" "$destination" +mkdir $mkdiropt "$destination" DONE fi From 90896a9e06b10a5daa26235420d08b745d771f3f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 3 Dec 2013 14:36:38 +0100 Subject: [PATCH 14/22] update environment with messages environment Signed-off-by: Nico Schottelius --- cdist/exec/local.py | 1 + cdist/message.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 28c50eec..f1313eea 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -168,6 +168,7 @@ class Local(object): if message_prefix: message = cdist.message.Message(message_prefix, self.messages_path) + env.update(message.env) try: if return_output: diff --git a/cdist/message.py b/cdist/message.py index 057c43d3..b840a84d 100644 --- a/cdist/message.py +++ b/cdist/message.py @@ -56,12 +56,14 @@ class Message(object): shutil.copyfile(self.global_messages, self.messages_in) def _cleanup(self): + """remove temporary files""" if os.path.exists(self.messages_in): os.remove(self.messages_in) if os.path.exists(self.messages_out): os.remove(self.messages_out) def _merge_messages(self): + """merge newly written lines into global file""" with open(self.messages_out) as fd: content = fd.readlines() From 68b7392021b5e6311b3283ab22cbc46cf761b12e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 3 Dec 2013 15:41:43 +0100 Subject: [PATCH 15/22] add support for messaging to __file, document messaging in reference Signed-off-by: Nico Schottelius --- cdist/conf/type/__file/gencode-local | 2 ++ cdist/conf/type/__file/gencode-remote | 8 ++++++-- cdist/conf/type/__file/man.text | 14 ++++++++++---- docs/man/cdist-reference.text.sh | 8 +++++++- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index a9eb6b10..601705c8 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -30,6 +30,7 @@ create_file= if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then if [ ! -f "$__object/parameter/source" ]; then create_file=1 + echo create >> "$__messages_out" else source="$(cat "$__object/parameter/source")" if [ "$source" = "-" ]; then @@ -64,6 +65,7 @@ if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" DONE if [ "$upload_file" ]; then + echo upload >> "$__messages_out" cat << DONE $__remote_copy $source ${__target_host}:\$destination_upload DONE diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index e80d5fae..57bf3503 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -43,15 +43,18 @@ get_current_value() { } set_group() { - echo chgrp \"$1\" \"$destination\" + echo chgrp \"$1\" \"$destination\" + echo chgrp $1 >> "$__messages_out" } set_owner() { - echo chown \"$1\" \"$destination\" + echo chown \"$1\" \"$destination\" + echo chown $1 >> "$__messages_out" } set_mode() { echo chmod \"$1\" \"$destination\" + echo chmod $1 >> "$__messages_out" } set_attributes= @@ -73,6 +76,7 @@ case "$state_should" in absent) if [ "$type" = "file" ]; then echo rm -f \"$destination\" + echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__file/man.text b/cdist/conf/type/__file/man.text index b76573bb..a582b27b 100644 --- a/cdist/conf/type/__file/man.text +++ b/cdist/conf/type/__file/man.text @@ -52,12 +52,18 @@ source:: MESSAGES -------- - -copy:: - File was copied because cksum was different from local version - +chgrp :: + Changed group membership +chown :: + Changed owner +chmod :: + Changed mode +create:: + Empty file was created (no --source specified) remove:: File exists, but state is absent, file will be removed by generated code. +upload:: + File was uploaded EXAMPLES diff --git a/docs/man/cdist-reference.text.sh b/docs/man/cdist-reference.text.sh index b41be801..a72452eb 100755 --- a/docs/man/cdist-reference.text.sh +++ b/docs/man/cdist-reference.text.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -191,6 +191,12 @@ __manifest:: __global:: Directory that contains generic output like explorer. Available for: initial manifest, type manifest, type gencode, shell +__messages_in:: + File to read messages from + Available for: initial manifest, type manifest, type gencode +__messages_out:: + File to write messages + Available for: initial manifest, type manifest, type gencode __object:: Directory that contains the current object. Available for: type manifest, type explorer, type gencode From b9dcd01ea1cb1842641f75e575e9d346b6ca46e1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Dec 2013 13:19:04 +0100 Subject: [PATCH 16/22] only restart the firewall (iptables) if needed Signed-off-by: Nico Schottelius --- cdist/conf/type/__iptables_apply/gencode-remote | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__iptables_apply/gencode-remote b/cdist/conf/type/__iptables_apply/gencode-remote index 0773b452..9cdf28cf 100644 --- a/cdist/conf/type/__iptables_apply/gencode-remote +++ b/cdist/conf/type/__iptables_apply/gencode-remote @@ -1,2 +1,3 @@ -# Rebuild rules - FIXME: do conditionally as soon as cdist supports it -echo /etc/init.d/iptables restart +if grep -q "^__file/etc/iptables.d/" "$__messages_in"; then + echo /etc/init.d/iptables restart +fi From ed10f4e5b44e002903ed67236773300122be41c6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Dec 2013 13:19:25 +0100 Subject: [PATCH 17/22] use default parameter Signed-off-by: Nico Schottelius --- cdist/conf/type/__iptables_rule/manifest | 7 +------ cdist/conf/type/__iptables_rule/parameter/default/state | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) create mode 100644 cdist/conf/type/__iptables_rule/parameter/default/state diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest index a6abbd5e..f02ab18b 100644 --- a/cdist/conf/type/__iptables_rule/manifest +++ b/cdist/conf/type/__iptables_rule/manifest @@ -21,12 +21,7 @@ base_dir=/etc/iptables.d name="$__object_id" - -if [ -f "$__object/parameter/state" ]; then - state="$(cat "$__object/parameter/state")" -else - state="present" -fi +state="$(cat "$__object/parameter/state")" ################################################################################ # Basic setup diff --git a/cdist/conf/type/__iptables_rule/parameter/default/state b/cdist/conf/type/__iptables_rule/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__iptables_rule/parameter/default/state @@ -0,0 +1 @@ +present From acd42b259bd079a970cb4b2c9c2e1be31e020809 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Dec 2013 13:37:15 +0100 Subject: [PATCH 18/22] do not generate code when mode = 0xxx format Signed-off-by: Nico Schottelius --- cdist/conf/type/__file/gencode-remote | 75 +++++++++++++++------------ 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 57bf3503..dcf3857b 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. @@ -26,20 +26,20 @@ stat_file="$__object/explorer/stat" get_current_value() { - if [ -s "$stat_file" ]; then - _name="$1" - _value="$2" - case "$_value" in - [0-9]*) - _index=2 - ;; - *) - _index=3 - ;; - esac - awk '/'"$_name"':/ { print $'$_index' }' "$stat_file" - unset _name _value _index - fi + if [ -s "$stat_file" ]; then + _name="$1" + _value="$2" + case "$_value" in + [0-9]*) + _index=2 + ;; + *) + _index=3 + ;; + esac + awk '/'"$_name"':/ { print $'$_index' }' "$stat_file" + unset _name _value _index + fi } set_group() { @@ -59,29 +59,36 @@ set_mode() { set_attributes= case "$state_should" in - present|exists) - # Note: Mode - needs to happen last as a chown/chgrp can alter mode by - # clearing S_ISUID and S_ISGID bits (see chown(2)) - for attribute in group owner mode; do - if [ -f "$__object/parameter/$attribute" ]; then + present|exists) + # Note: Mode - needs to happen last as a chown/chgrp can alter mode by + # clearing S_ISUID and S_ISGID bits (see chown(2)) + for attribute in group owner mode; do + if [ -f "$__object/parameter/$attribute" ]; then value_should="$(cat "$__object/parameter/$attribute")" + + # change 0xxx format to xxx format => same as stat returns + if [ "$attribute" = mode ]; then + value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')" + fi + value_is="$(get_current_value "$attribute" "$value_should")" if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then - "set_$attribute" "$value_should" + "set_$attribute" "$value_should" fi - fi - done - ;; + fi + done - absent) - if [ "$type" = "file" ]; then - echo rm -f \"$destination\" - echo remove >> "$__messages_out" - fi - ;; + ;; - *) - echo "Unknown state: $state_should" >&2 - exit 1 - ;; + absent) + if [ "$type" = "file" ]; then + echo rm -f \"$destination\" + echo remove >> "$__messages_out" + fi + ;; + + *) + echo "Unknown state: $state_should" >&2 + exit 1 + ;; esac From ccf0f4311d8d1420e0c79a04ce95ae3db154221d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Dec 2013 13:37:32 +0100 Subject: [PATCH 19/22] ++changes(3.0.0) (really!) Signed-off-by: Nico Schottelius --- docs/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog b/docs/changelog index b6122bec..5fb0af36 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,12 @@ Changelog * Changes are always commented with their author in (braces) * Exception: No braces means author == Nico Schottelius + +3.0.0: + * Core: Messaging support added + * Type: __iptables_rule: Use default parameter + * Type __file: Do not generate code if mode is 0xxx + 2.3.7: 2013-12-02 * Type __file: Secure the file transfer by using mktemp (Steven Armstrong) * Type __file: Only remove file when state is absent (Steven Armstrong) From 77830609939f4bd0b4525ee5f72bfca1f970bd0e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Dec 2013 14:19:23 +0100 Subject: [PATCH 20/22] document messaging Signed-off-by: Nico Schottelius --- docs/man/man7/cdist-messaging.text | 72 ++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/man/man7/cdist-messaging.text diff --git a/docs/man/man7/cdist-messaging.text b/docs/man/man7/cdist-messaging.text new file mode 100644 index 00000000..0e53871e --- /dev/null +++ b/docs/man/man7/cdist-messaging.text @@ -0,0 +1,72 @@ +cdist-messaging(7) +================== +Nico Schottelius + +NAME +---- +cdist-messaging - How the initial manifest and types can communication + + +DESCRIPTION +----------- +cdist has a simple but powerful way of allowing communication between +the initial manifest and types as well as types and types. + +Whenever execution is passed from cdist to one of the +scripts described below, cdist generate 2 new temporary files +and exports the environment variables __messages_in and +__messages_out to point to them. + +Before handing over the control, the content of the global message +file is copied into the file referenced by $__messages_in. + +After cdist gained control back, the content of the file referenced +by $__messages_out is appended to the global message file. + +This way overwriting any of the two files by accident does not +interfere with other types. + +The order of execution is not defined unless you create dependencies +between the different objects (see cdist-manifest(7)) and thus you +can only react reliably on messages by objects that you depend on. + + +AVAILABILITY +------------ +Messaging is possible between all **local** scripts: + +- initial manifest +- type/manifest +- type/gencode-local +- type/gencode-remote + + +EXAMPLES +-------- +When you want to emit a message use: + +-------------------------------------------------------------------------------- +echo "something" >> "$__messages_out" +-------------------------------------------------------------------------------- + +When you want to react on a message use: + +-------------------------------------------------------------------------------- +if grep -q "^__your_type/object/id:something" "$__messages_in"; then + echo "I do something else" +fi +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist(1) +- cdist-manifest(7) +- cdist-reference(7) +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2013 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). From e50b54273a524cf938328e8b90faa904ccc1861f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Dec 2013 14:21:23 +0100 Subject: [PATCH 21/22] docs not, doc Signed-off-by: Nico Schottelius --- {doc => docs}/dev/logs/2012-05-24.preos | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {doc => docs}/dev/logs/2012-05-24.preos (100%) diff --git a/doc/dev/logs/2012-05-24.preos b/docs/dev/logs/2012-05-24.preos similarity index 100% rename from doc/dev/logs/2012-05-24.preos rename to docs/dev/logs/2012-05-24.preos From 7adbc6f91308450f6a5d361f5b5fb5740031338f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Dec 2013 14:44:27 +0100 Subject: [PATCH 22/22] add changes from preos Signed-off-by: Nico Schottelius --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5fb0af36..577194bf 100644 --- a/docs/changelog +++ b/docs/changelog @@ -15,6 +15,7 @@ Changelog * Type __file: Only remove file when state is absent (Steven Armstrong) * Type __link: Only remove link when state is absent (Steven Armstrong) * Type __directory: Only remove directory when state is absent (Steven Armstrong) + * Type __directory: Fix newly introduced quoting issue * Type __package_zypper: Fix explorer and parameter issue (Daniel Heule) * Core: Fix backtrace when cache cannot be deleted