Merge branch 'master' into preos

Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>

Conflicts:
	docs/changelog
This commit is contained in:
Nico Schottelius 2013-12-04 14:44:54 +01:00
commit 84eb05aed6
19 changed files with 468 additions and 59 deletions

View file

@ -30,6 +30,7 @@ create_file=
if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then
if [ ! -f "$__object/parameter/source" ]; then
create_file=1
echo create >> "$__messages_out"
else
source="$(cat "$__object/parameter/source")"
if [ "$source" = "-" ]; then
@ -64,6 +65,7 @@ if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then
destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")"
DONE
if [ "$upload_file" ]; then
echo upload >> "$__messages_out"
cat << DONE
$__remote_copy $source ${__target_host}:\$destination_upload
DONE

View file

@ -1,6 +1,6 @@
#!/bin/sh
#
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# 2013 Steven Armstrong (steven-cdist armstrong.cc)
#
# This file is part of cdist.
@ -44,14 +44,17 @@ get_current_value() {
set_group() {
echo chgrp \"$1\" \"$destination\"
echo chgrp $1 >> "$__messages_out"
}
set_owner() {
echo chown \"$1\" \"$destination\"
echo chown $1 >> "$__messages_out"
}
set_mode() {
echo chmod \"$1\" \"$destination\"
echo chmod $1 >> "$__messages_out"
}
set_attributes=
@ -62,17 +65,25 @@ case "$state_should" in
for attribute in group owner mode; do
if [ -f "$__object/parameter/$attribute" ]; then
value_should="$(cat "$__object/parameter/$attribute")"
# change 0xxx format to xxx format => same as stat returns
if [ "$attribute" = mode ]; then
value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')"
fi
value_is="$(get_current_value "$attribute" "$value_should")"
if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then
"set_$attribute" "$value_should"
fi
fi
done
;;
absent)
if [ "$type" = "file" ]; then
echo rm -f \"$destination\"
echo remove >> "$__messages_out"
fi
;;

View file

@ -50,6 +50,21 @@ source::
If not supplied, an empty file or directory will be created.
If source is '-' (dash), take what was written to stdin as the file content.
MESSAGES
--------
chgrp <group>::
Changed group membership
chown <owner>::
Changed owner
chmod <mode>::
Changed mode
create::
Empty file was created (no --source specified)
remove::
File exists, but state is absent, file will be removed by generated code.
upload::
File was uploaded
EXAMPLES
--------
@ -90,5 +105,5 @@ SEE ALSO
COPYING
-------
Copyright \(C) 2011-2012 Nico Schottelius. Free use of this software is
Copyright \(C) 2011-2013 Nico Schottelius. Free use of this software is
granted under the terms of the GNU General Public License version 3 (GPLv3).

View file

@ -1,2 +1,3 @@
# Rebuild rules - FIXME: do conditionally as soon as cdist supports it
if grep -q "^__file/etc/iptables.d/" "$__messages_in"; then
echo /etc/init.d/iptables restart
fi

View file

@ -21,12 +21,7 @@
base_dir=/etc/iptables.d
name="$__object_id"
if [ -f "$__object/parameter/state" ]; then
state="$(cat "$__object/parameter/state")"
else
state="present"
fi
################################################################################
# Basic setup

View file

@ -0,0 +1 @@
present

2
cdist/conf/type/__package_zypper/explorer/pkg_version Executable file → Normal file
View file

@ -27,4 +27,4 @@ else
name="$__object_id"
fi
rpm -q --whatprovides "$name" 2>/dev/null || true
rpm -q --whatprovides "$name" | grep -v 'no package provides' || true

11
cdist/conf/type/__package_zypper/gencode-remote Executable file → Normal file
View file

@ -39,15 +39,22 @@ else
state_should="present"
fi
pkg_version="$(cat "$__object/explorer/pkg_version")"
if [ -z "$pkg_version" ]; then
state_is="absent"
else
state_is="present"
fi
# Exit if nothing is needed to be done
[ "$state_is" = "$state_should" ] && exit 0
case "$state_should" in
present)
echo zypper "$globalopts" install --auto-agree-with-licenses \"$name\"
echo zypper $globalopts install --auto-agree-with-licenses \"$name\" ">/dev/null"
;;
absent)
echo pacman "$globalopts" remove \"$name\"
echo zypper $globalopts remove \"$name\" ">/dev/null"
;;
*)
echo "Unknown state: $state_should" >&2

View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -104,7 +104,8 @@ class Code(object):
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
})
return self.local.run_script(script, env=env, return_output=True)
message_prefix=cdist_object.name
return self.local.run_script(script, env=env, return_output=True, message_prefix=message_prefix)
def run_gencode_local(self, cdist_object):
"""Run the gencode-local script for the given cdist object."""
@ -119,9 +120,6 @@ class Code(object):
source = os.path.join(self.local.object_path, cdist_object.code_remote_path)
destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path)
# FIXME: BUG: do not create destination, but top level of destination!
# FIXME: BUG2: we are called AFTER the code-remote has been transferred already:
# mkdir: cannot create directory `/var/lib/cdist/object/__directory/etc/acpi/actions/.cdist/code-remote': File exists
# OR: this is from previous run -> cleanup missing!
self.remote.mkdir(destination)
self.remote.transfer(source, destination)

View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -122,7 +122,8 @@ class Manifest(object):
if not os.path.isfile(initial_manifest):
raise NoInitialManifestError(initial_manifest, user_supplied)
self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest))
message_prefix="initialmanifest"
self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix)
def env_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
@ -141,5 +142,6 @@ class Manifest(object):
def run_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
message_prefix = cdist_object.name
if os.path.isfile(type_manifest):
self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object))

View file

@ -30,6 +30,7 @@ import logging
import tempfile
import cdist
import cdist.message
from cdist import core
class Local(object):
@ -92,6 +93,7 @@ class Local(object):
self.conf_path = os.path.join(self.base_path, "conf")
self.global_explorer_out_path = os.path.join(self.base_path, "explorer")
self.object_path = os.path.join(self.base_path, "object")
self.messages_path = os.path.join(self.base_path, "messages")
# Depending on conf_path
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
@ -128,6 +130,7 @@ class Local(object):
def create_files_dirs(self):
self._init_directories()
self._create_conf_path_and_link_conf_dirs()
self._create_messages()
self._link_types_for_emulator()
@ -150,7 +153,7 @@ class Local(object):
self.log.debug("Local mkdir: %s", path)
os.makedirs(path, exist_ok=True)
def run(self, command, env=None, return_output=False):
def run(self, command, env=None, return_output=False, message_prefix=None):
"""Run the given command with the given environment.
Return the output as a string.
@ -163,6 +166,10 @@ class Local(object):
# Export __target_host for use in __remote_{copy,exec} scripts
env['__target_host'] = self.target_host
if message_prefix:
message = cdist.message.Message(message_prefix, self.messages_path)
env.update(message.env)
try:
if return_output:
return subprocess.check_output(command, env=env).decode()
@ -172,8 +179,11 @@ class Local(object):
raise cdist.Error("Command failed: " + " ".join(command))
except OSError as error:
raise cdist.Error(" ".join(*args) + ": " + error.args[1])
finally:
if message_prefix:
message.merge_messages()
def run_script(self, script, env=None, return_output=False):
def run_script(self, script, env=None, return_output=False, message_prefix=None):
"""Run the given script with the given environment.
Return the output as a string.
@ -181,7 +191,7 @@ class Local(object):
command = ["/bin/sh", "-e"]
command.append(script)
return self.run(command, env, return_output)
return self.run(command=command, env=env, return_output=return_output, message_prefix=message_prefix)
def save_cache(self):
destination = os.path.join(self.cache_path, self.target_host)
@ -195,6 +205,11 @@ class Local(object):
shutil.move(self.base_path, destination)
def _create_messages(self):
"""Create empty messages"""
with open(self.messages_path, "w"):
pass
def _create_conf_path_and_link_conf_dirs(self):
# Link destination directories
for sub_dir in [ "explorer", "manifest", "type" ]:

76
cdist/message.py Normal file
View file

@ -0,0 +1,76 @@
# -*- coding: utf-8 -*-
#
# 2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import os
import shutil
import tempfile
import cdist
log = logging.getLogger(__name__)
class Message(object):
"""Support messaging between types
"""
def __init__(self, prefix, messages):
self.prefix = prefix
self.global_messages = messages
self.messages_in = tempfile.mkstemp(suffix='.cdist_message_in')[1]
self.messages_out = tempfile.mkstemp(suffix='.cdist_message_out')[1]
self._copy_messages()
@property
def env(self):
env = {}
env['__messages_in'] = self.messages_in
env['__messages_out'] = self.messages_out
return env
def _copy_messages(self):
"""Copy global contents into our copy"""
shutil.copyfile(self.global_messages, self.messages_in)
def _cleanup(self):
"""remove temporary files"""
if os.path.exists(self.messages_in):
os.remove(self.messages_in)
if os.path.exists(self.messages_out):
os.remove(self.messages_out)
def _merge_messages(self):
"""merge newly written lines into global file"""
with open(self.messages_out) as fd:
content = fd.readlines()
with open(self.global_messages, 'a') as fd:
for line in content:
fd.write("%s:%s" % (self.prefix, line))
def merge_messages(self):
self._merge_messages()
self._cleanup()

View file

@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
#
# 2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import tempfile
from cdist import test
import cdist.message
class MessageTestCase(test.CdistTestCase):
def setUp(self):
self.prefix="cdist-test"
self.content = "A very short story"
self.tempfile = tempfile.mkstemp()[1]
self.message = cdist.message.Message(prefix=self.prefix,
messages=self.tempfile)
def tearDown(self):
os.remove(self.tempfile)
self.message._cleanup()
def test_env(self):
"""
Ensure environment is correct
"""
env = self.message.env
self.assertIn('__messages_in', env)
self.assertIn('__messages_out', env)
def test_copy_content(self):
"""
Ensure content copying is working
"""
with open(self.tempfile, "w") as fd:
fd.write(self.content)
self.message._copy_messages()
with open(self.tempfile, "r") as fd:
testcontent = fd.read()
self.assertEqual(self.content, testcontent)
def test_message_merge_prefix(self):
"""Ensure messages are merged and are prefixed"""
expectedcontent = "%s:%s" % (self.prefix, self.content)
out = self.message.env['__messages_out']
with open(out, "w") as fd:
fd.write(self.content)
self.message._merge_messages()
with open(self.tempfile, "r") as fd:
testcontent = fd.read()
self.assertEqual(expectedcontent, testcontent)

View file

@ -4,12 +4,19 @@ Changelog
* Changes are always commented with their author in (braces)
* Exception: No braces means author == Nico Schottelius
2.3.7:
3.0.0:
* Core: Messaging support added
* Type: __iptables_rule: Use default parameter
* Type __file: Do not generate code if mode is 0xxx
2.3.7: 2013-12-02
* Type __file: Secure the file transfer by using mktemp (Steven Armstrong)
* Type __file: Only remove file when state is absent (Steven Armstrong)
* Type __link: Only remove link when state is absent (Steven Armstrong)
* Type __directory: Only remove directory when state is absent (Steven Armstrong)
* Type __directory: Fix newly introduced quoting issue
* Type __package_zypper: Fix explorer and parameter issue (Daniel Heule)
* Core: Fix backtrace when cache cannot be deleted
2.3.6: 2013-11-25

View file

@ -0,0 +1,20 @@
Allow cross-type communication
Sending notifications is possible from
- manifest
- gencode-local
- gencode-remote
Sending a notification from an object means writing to the file "notifications" into
its object:
echo mytest >> "$__object/notifications" # a type reports something
Reading / Reacting on notifications works by accessing the file
referred to be "$__notifications". All notifications are prefixed with
the object name ($__object_name) and are appended into this file.
To find out, whether a file was copied, run:
grep __file/etc/passwd:copy "$__notifications"

View file

@ -0,0 +1,49 @@
An alternative / complementary approach to notifications: triggers (or actions?)
A type may support various actions by creating files in its subdirectory
"actions". Other types can trigger an action of a different type or object
by calling them (indirectly?):
if grep "__file/etc/nginx/conf.d/.*:copy" "$__notifications"; then
# Call action from a type
cdist trigger __nginx/reload
fi
Not sure whether this approach (calling "actions" of other types) is sane,
as nginx should probably better know if it should be restarted "itself".
--------------------------------------------------------------------------------
Alternate approach:
__nginx_vhost www.some-domain.ch --custom << eof
some custom code for __nginx_vhost inclusion
eof
__nginx_vhost:
manifest:
# __nginx_vhost requires __nginx: creates directories
require"$__object_name" __nginx --require-only
# Do WE or __file ... depend on nginx?
cdist require __nginx
# Create file that contains the giving code
__file /etc/nginx/conf.d/www.some-domain.ch
require="__nginx" __file /etc/nginx/conf.d/www.some-domain.ch
__nginx:
manifest:
__package nginx --state present
__file some-custom-files
gencode-remote:
if first_install or file changed:

View file

@ -0,0 +1,50 @@
Follow up from 2013-01-20:
- (re-)create message file per object?
- yes, but do not necessarily save in object space
- save $anywhere
- object_run
- current notifications are imported into a file available at $__messages_in
- after object run, everything that has been written to $__messages_out is merged into the $__messages file
- functions:
self.explorer.run_global_explorers(self.local.global_explorer_out_path)
self.manifest.run_initial_manifest(self.local.initial_manifest)
self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest))
self.explorer.run_type_explorers(cdist_object)
self.manifest.run_type_manifest(cdist_object)
self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object))
self.code.run_gencode_local(cdist_object)
self.local.run_script(script, env=env, return_output=True)
self.code.run_gencode_remote(cdist_object)
self.local.run_script(script, env=env, return_output=True)
- message support in ...
- initialmanifest - yes
- explorer - no
- only locally - yes
- how to use notification / messaging in cdist
- can be used in all local scripts:
- initial manifest
- type manifest
- type gencode-*
- order of object exeution is random or as you requested using require=""
- example use:
__file/gencode-local:
if [ "$local_cksum" != "$remote_cksum" ]; then
echo "$__remote_copy" "$source" "${__target_host}:${destination}"
echo "copy" >> "$__messages_out"
fi
__nginx/manifest:
__file /etc/nginx/sites-enabled/myfile --source "$__type/files/nginx-config"
__nginx/gencode-remote:
if grep -q "__file/etc/nginx/sites-enabled/myfile:copy" "$__messages_in"; then
echo /etc/init.d/nginx restart
fi

View file

@ -1,6 +1,6 @@
#!/bin/sh
#
# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org)
# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -191,6 +191,12 @@ __manifest::
__global::
Directory that contains generic output like explorer.
Available for: initial manifest, type manifest, type gencode, shell
__messages_in::
File to read messages from
Available for: initial manifest, type manifest, type gencode
__messages_out::
File to write messages
Available for: initial manifest, type manifest, type gencode
__object::
Directory that contains the current object.
Available for: type manifest, type explorer, type gencode

View file

@ -0,0 +1,72 @@
cdist-messaging(7)
==================
Nico Schottelius <nico-cdist--@--schottelius.org>
NAME
----
cdist-messaging - How the initial manifest and types can communication
DESCRIPTION
-----------
cdist has a simple but powerful way of allowing communication between
the initial manifest and types as well as types and types.
Whenever execution is passed from cdist to one of the
scripts described below, cdist generate 2 new temporary files
and exports the environment variables __messages_in and
__messages_out to point to them.
Before handing over the control, the content of the global message
file is copied into the file referenced by $__messages_in.
After cdist gained control back, the content of the file referenced
by $__messages_out is appended to the global message file.
This way overwriting any of the two files by accident does not
interfere with other types.
The order of execution is not defined unless you create dependencies
between the different objects (see cdist-manifest(7)) and thus you
can only react reliably on messages by objects that you depend on.
AVAILABILITY
------------
Messaging is possible between all **local** scripts:
- initial manifest
- type/manifest
- type/gencode-local
- type/gencode-remote
EXAMPLES
--------
When you want to emit a message use:
--------------------------------------------------------------------------------
echo "something" >> "$__messages_out"
--------------------------------------------------------------------------------
When you want to react on a message use:
--------------------------------------------------------------------------------
if grep -q "^__your_type/object/id:something" "$__messages_in"; then
echo "I do something else"
fi
--------------------------------------------------------------------------------
SEE ALSO
--------
- cdist(1)
- cdist-manifest(7)
- cdist-reference(7)
- cdist-type(7)
COPYING
-------
Copyright \(C) 2013 Nico Schottelius. Free use of this software is
granted under the terms of the GNU General Public License version 3 (GPLv3).