Compare commits

...

31 Commits
master ... beta

Author SHA1 Message Date
Darko Poljak ed6d26245b Merge branch 'master' into beta 2021-08-24 20:38:23 +02:00
Darko Poljak 94a7975e32 Merge branch 'master' into beta 2021-07-10 20:52:06 +02:00
Darko Poljak 5b9c652221 Fix merging mistake 2021-04-20 07:47:29 +02:00
Darko Poljak deaff9c8e7 Merge branch 'master' into beta 2021-04-20 07:43:33 +02:00
Darko Poljak 340df8305c Merge branch 'master' into beta 2021-02-28 13:50:52 +01:00
Darko Poljak f39a2ba975 Merge branch 'master' into beta 2020-12-21 19:29:24 +01:00
Darko Poljak 42b8a005be Merge branch 'master' into beta 2020-12-04 15:38:51 +01:00
Darko Poljak 1e2440161c Merge branch 'master' into beta 2020-11-20 20:01:54 +01:00
Darko Poljak 55baa0278d Merge branch 'master' into beta 2020-11-08 13:42:24 +01:00
Darko Poljak b17bf193a9 Merge branch 'master' into beta 2020-11-07 12:21:01 +01:00
Darko Poljak 9e89088e61 Merge branch 'master' into beta 2020-09-11 14:33:27 +02:00
Darko Poljak cde6a91ef2 Merge branch 'master' into beta 2020-07-28 07:27:32 +02:00
Darko Poljak bf6ce46c95 Merge branch 'master' into beta 2020-06-17 12:19:32 +02:00
Darko Poljak b47b9b1ba0 Merge branch 'master' into beta 2020-05-25 11:24:01 +02:00
Darko Poljak 1da9dcce4a Merge branch 'master' into beta 2020-05-01 13:09:45 +02:00
Darko Poljak 6802cd990e Merge branch 'master' into beta 2020-04-11 20:47:27 +02:00
Darko Poljak 17c39ebef5 Merge branch 'master' into beta 2020-04-03 19:54:46 +02:00
Darko Poljak 225f75ca8f Merge branch 'master' into beta 2020-02-27 20:37:19 +01:00
Darko Poljak 77d704113c Update file stat from master 2020-02-15 14:52:10 +01:00
Darko Poljak 595cfec572 Adapt order dependency impl to python types impl 2020-02-15 14:47:15 +01:00
Darko Poljak b7af305636 Align with shell type: implement onchange 2020-02-15 14:47:15 +01:00
Darko Poljak 1588476a91 Remove unnecessary files 2020-02-15 14:47:15 +01:00
Darko Poljak 420aaaee16 Make python file type default, mv old to __file_old 2020-02-15 14:46:03 +01:00
Darko Poljak a2243cf59e Add support for python type defined argument parser 2020-02-15 14:43:44 +01:00
Darko Poljak 3dcd270158 Implement python types 2020-02-15 14:43:44 +01:00
Dominique Roux 4a5a475e3e Updated the man pages for the cdist trigger and preos 2020-02-15 14:43:44 +01:00
Darko Poljak 1a5cc67cc8 Add missing configuration arg 2020-02-15 14:43:44 +01:00
Darko Poljak a094679173 Update trigger to config 2020-02-15 14:43:44 +01:00
Darko Poljak 912c675903 Log trigger server error 2020-02-15 14:43:44 +01:00
Darko Poljak 9d030bddc1 ++ 2020-02-15 14:43:44 +01:00
Darko Poljak 9a2e5758f5 Implement triggering functionality 2020-02-15 14:43:44 +01:00
45 changed files with 1941 additions and 43 deletions

View File

@ -63,6 +63,9 @@ REMOTE_COPY = "scp -o User=root -q"
REMOTE_EXEC = "ssh -o User=root"
REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
ORDER_DEP_STATE_NAME = 'order_dep_state'
TYPEORDER_DEP_NAME = 'typeorder_dep'
MIN_SUPPORTED_PYTHON_VERSION = (3, 5)

View File

@ -5,6 +5,7 @@ import logging
import collections
import functools
import cdist.configuration
import cdist.trigger
import cdist.log
import cdist.preos
import cdist.info
@ -12,7 +13,7 @@ import cdist.scan.commandline
# set of beta sub-commands
BETA_COMMANDS = set(('install', 'inventory', 'scan', ))
BETA_COMMANDS = set(('install', 'inventory', 'scan', 'trigger', ))
# set of beta arguments for sub-commands
BETA_ARGS = {
'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@ -512,6 +513,28 @@ def get_parsers():
help='How long (seconds) to wait between ICMPv6 echo requests')
parser['scan'].set_defaults(func=cdist.scan.commandline.commandline)
# Trigger
parser['trigger'] = parser['sub'].add_parser(
'trigger', parents=[parser['loglevel'],
parser['beta'],
parser['common'],
parser['config_main']])
parser['trigger'].add_argument(
'-D', '--directory', action='store', required=False,
help=('Where to create local files'))
parser['trigger'].add_argument(
'-H', '--http-port', action='store', default=3000, required=False,
help=('Create trigger listener via http on specified port'))
parser['trigger'].add_argument(
'--ipv6', default=False,
help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
action='store_true')
parser['trigger'].add_argument(
'-O', '--source', action='store', required=False,
help=('Which file to copy for creation'))
parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
for p in parser:
parser[p].epilog = EPILOG

View File

@ -0,0 +1,12 @@
#!/bin/sh
os=$(cat "$__global/explorer/os")
case "$os" in
devuan)
echo "update-rc.d cdist-preos-trigger defaults > /dev/null"
;;
*)
;;
esac

View File

@ -0,0 +1,45 @@
cdist-type__cdist_preos_trigger(7)
==================================
NAME
----
cdist-type__cdist_preos_trigger - configure cdist preos trigger
DESCRIPTION
-----------
Create cdist PreOS trigger by creating systemd unit file that will be started
at boot and will execute trigger command - connect to specified host and port.
REQUIRED PARAMETERS
-------------------
trigger-command
Command that will be executed as a PreOS cdist trigger.
OPTIONAL PARAMETERS
-------------------
None
EXAMPLES
--------
.. code-block:: sh
# Configure default curl trigger for host cdist.ungleich.ch at port 80.
__cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80'
AUTHORS
-------
Darko Poljak <darko.poljak--@--ungleich.ch>
COPYING
-------
Copyright \(C) 2016 Darko Poljak. You can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

View File

@ -0,0 +1,67 @@
#!/bin/sh
os="$(cat "$__global/explorer/os")"
trigger_command=$(cat "$__object/parameter/trigger-command")
case "$os" in
devuan)
__file /etc/init.d/cdist-preos-trigger --owner root \
--group root \
--mode 755 \
--source - << EOF
#!/bin/sh
# /etc/init.d/cdist-preos-trigger
### BEGIN INIT INFO
# Provides: cdist-preos-trigger
# Required-Start: \$all
# Required-Stop:
# Default-Start: 2 3 4 5 S
# Default-Stop: 0 1 6
# Short-Description: Execute cdist preos trigger command
# Description: Execute cdist preos trigger commnad.
### END INIT INFO
case "\$1" in
start)
echo "Starting cdist-preos-trigger command"
${trigger_command} &
;;
stop)
# no-op
;;
*)
echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}"
exit 1
;;
esac
exit 0
EOF
;;
*)
__file /etc/systemd/system/cdist-preos-trigger.service --owner root \
--group root \
--mode 644 \
--source - << EOF
[Unit]
Description=preos trigger
Wants=network-online.target
After=network.target network-online.target
[Service]
Type=simple
Restart=no
# Broken systemd
ExecStartPre=/bin/sleep 5
ExecStart=${trigger_command}
[Install]
WantedBy=multi-user.target
EOF
require="__file/etc/systemd/system/cdist-preos-trigger.service" \
__start_on_boot cdist-preos-trigger
;;
esac

View File

@ -0,0 +1 @@
trigger-command

View File

@ -0,0 +1,124 @@
import os
import re
from cdist.core.pytypes import *
import argparse
class FileType(PythonType):
def get_attribute(self, stat_file, attribute, value_should):
if os.path.exists(stat_file):
if re.match('[0-9]', value_should):
index = 1
else:
index = 2
with open(stat_file, 'r') as f:
for line in f:
if re.match(attribute + ":", line):
fields = line.split()
return fields[index]
return None
def set_attribute(self, attribute, value_should, destination):
cmd = {
'group': 'chgrp',
'owner': 'chown',
'mode': 'chmod',
}
self.send_message("{} '{}'".format(cmd[attribute], value_should))
return "{} '{}' '{}'".format(cmd[attribute], value_should, destination)
def type_manifest(self):
yield from ()
def type_gencode(self):
typeis = self.get_explorer('type')
state_should = self.get_parameter('state')
if state_should == 'exists' and typeis == 'file':
return
source = self.get_parameter('source')
if source == '-':
source = self.stdin_path
destination = '/' + self.object_id
if state_should == 'pre-exists':
if source is not None:
self.die('--source cannot be used with --state pre-exists')
if typeis == 'file':
return None
else:
self.die('File {} does not exist'.format(destination))
create_file = False
upload_file = False
set_attributes = False
fire_onchange = False
code = []
if state_should == 'present' or state_should == 'exists':
if source is None:
remote_stat = self.get_explorer('stat')
if not remote_stat:
create_file = True
else:
if os.path.exists(source):
if typeis == 'file':
local_cksum = self.run_local(['cksum', source, ])
local_cksum = local_cksum.split()[0]
remote_cksum = self.get_explorer('cksum')
remote_cksum = remote_cksum.split()[0]
upload_file = local_cksum != remote_cksum
else:
upload_file = True
else:
self.die('Source {} does not exist'.format(source))
if create_file or upload_file:
set_attributes = True
fire_onchange = True
tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination)
destination_upload = self.run_remote(
["mktemp", tempfile_template, ])
if upload_file:
self.transfer(source, destination_upload)
code.append('rm -rf {}'.format(destination))
code.append('mv {} {}'.format(destination_upload, destination))
if state_should in ('present', 'exists', 'pre-exists', ):
for attribute in ('group', 'owner', 'mode', ):
if attribute in self.parameters:
value_should = self.get_parameter(attribute)
if attribute == 'mode':
value_should = re.sub('^0', '', value_should)
stat_file = self.get_explorer_file('stat')
value_is = self.get_attribute(stat_file, attribute,
value_should)
if set_attributes or value_should != value_is:
fire_onchange = True
code.append(self.set_attribute(attribute,
value_should,
destination))
elif state_should == 'absent':
if typeis == 'file':
code.append('rm -f {}'.format(destination))
self.send_message('remove')
fire_onchange = True
else:
self.die('Unknown state {}'.format(state_should))
if fire_onchange:
onchange = self.get_parameter('onchange')
if onchange:
code.append(onchange)
return "\n".join(code)
def get_args_parser(self):
parser = argparse.ArgumentParser(add_help=False,
argument_default=argparse.SUPPRESS)
parser.add_argument('--state', dest='state', action='store',
required=False, default='present')
for param in ('group', 'mode', 'owner', 'source', 'onchange'):
parser.add_argument('--' + param, dest=param, action='store',
required=False, default=None)
parser.add_argument("object_id", nargs=1)
return parser

View File

@ -0,0 +1,34 @@
#!/bin/sh
#
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# Retrieve the md5sum of a file to be created, if it is already existing.
#
destination="/$__object_id"
if [ -e "$destination" ]; then
if [ -f "$destination" ]; then
cksum < "$destination"
else
echo "NO REGULAR FILE"
fi
else
echo "NO FILE FOUND, NO CHECKSUM CALCULATED."
fi

View File

@ -0,0 +1,116 @@
#!/bin/sh
#
# 2013 Steven Armstrong (steven-cdist armstrong.cc)
# 2019 Nico Schottelius (nico-cdist at schottelius.org)
# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
destination="/$__object_id"
fallback() {
# Fallback: Patch the output together, manually.
ls_line=$(ls -ldn "$destination")
uid=$(echo "$ls_line" | awk '{ print $3 }')
gid=$(echo "$ls_line" | awk '{ print $4 }')
owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd)
group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group)
mode_text=$(echo "$ls_line" | awk '{ print $1 }')
mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }')
size=$(echo "$ls_line" | awk '{ print $5 }')
links=$(echo "$ls_line" | awk '{ print $2 }')
printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\nsize: %d\nlinks: %d\n' \
"$("$__type_explorer/type")" \
"$uid" "$owner" \
"$gid" "$group" \
"$mode" "$mode_text" \
"$size" \
"$links"
}
# nothing to work with, nothing we could do
[ -e "$destination" ] || exit 0
if ! command -v stat >/dev/null
then
fallback
exit
fi
case $("$__explorer/os")
in
freebsd|netbsd|openbsd|macosx)
stat -f "type: %HT
owner: %Du %Su
group: %Dg %Sg
mode: %Lp %Sp
size: %Dz
links: %Dl
" "$destination" | awk '/^type/ { print tolower($0); next } { print }'
;;
solaris)
ls1="$( ls -ld "$destination" )"
ls2="$( ls -ldn "$destination" )"
if [ -f "$__object/parameter/mode" ]
then mode_should="$( cat "$__object/parameter/mode" )"
fi
# yes, it is ugly hack, but if you know better way...
if [ -z "$( find "$destination" -perm "$mode_should" )" ]
then octets=888
else octets="$( echo "$mode_should" | sed 's/^0//' )"
fi
case "$( echo "$ls1" | cut -c1-1 )" in
-) echo 'type: regular file' ;;
d) echo 'type: directory' ;;
esac
echo "owner: $( echo "$ls2" \
| awk '{print $3}' ) $( echo "$ls1" \
| awk '{print $3}' )"
echo "group: $( echo "$ls2" \
| awk '{print $4}' ) $( echo "$ls1" \
| awk '{print $4}' )"
echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )"
echo "size: $( echo "$ls1" | awk '{print $5}' )"
echo "links: $( echo "$ls1" | awk '{print $2}' )"
;;
*)
# NOTE: Do not use --printf here as it is not supported by BusyBox stat.
# NOTE: BusyBox's stat might not support the "-c" option, in which case
# we fall through to the shell fallback.
stat -c "type: %F
owner: %u %U
group: %g %G
mode: %a %A
size: %s
links: %h" "$destination" 2>/dev/null || fallback
;;
esac

View File

@ -0,0 +1,33 @@
#!/bin/sh
#
# 2013 Steven Armstrong (steven-cdist armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
destination="/$__object_id"
if [ ! -e "$destination" ]; then
echo none
elif [ -h "$destination" ]; then
echo symlink
elif [ -f "$destination" ]; then
echo file
elif [ -d "$destination" ]; then
echo directory
else
echo unknown
fi

1
cdist/conf/type/__file_py Symbolic link
View File

@ -0,0 +1 @@
__file

View File

@ -41,6 +41,7 @@ import cdist.hostsource
import cdist.exec.local
import cdist.exec.remote
import cdist.util.ipaddr as ipaddr
import cdist.util.python_type_util as pytype_util
import cdist.configuration
@ -91,13 +92,15 @@ class Config:
shutil.rmtree(path)
def __init__(self, local, remote, dry_run=False, jobs=None,
cleanup_cmds=None, remove_remote_files_dirs=False):
cleanup_cmds=None, remove_remote_files_dirs=False,
timestamp=False):
self.local = local
self.remote = remote
self._open_logger()
self.dry_run = dry_run
self.jobs = jobs
self.timestamp = timestamp
if cleanup_cmds:
self.cleanup_cmds = cleanup_cmds
else:
@ -437,7 +440,8 @@ class Config:
cleanup_cmds.append(cleanup_cmd)
c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs,
cleanup_cmds=cleanup_cmds,
remove_remote_files_dirs=remove_remote_files_dirs)
remove_remote_files_dirs=remove_remote_files_dirs,
timestamp=args.timestamp)
c.run()
cls._remove_paths()
@ -475,6 +479,7 @@ class Config:
'dry' if self.dry_run else 'configuration')
self._init_files_dirs()
self.local.collect_python_types()
self.explorer.run_global_explorers(self.local.global_explorer_out_path)
try:
@ -790,6 +795,21 @@ class Config:
args = [param, cdist_type.name]
self.log.warning(format, *args)
def _timeit(self, func, msg_prefix):
def wrapper_func(*args, **kwargs):
loglevel = self.log.getEffectiveLevel()
if loglevel >= logging.VERBOSE and self.timestamp:
start_time = time.time()
rv = func(*args, **kwargs)
end_time = time.time()
duration = end_time - start_time
self.log.verbose("%s duration: %.6f seconds",
msg_prefix, duration)
else:
rv = func(*args, **kwargs)
return rv
return wrapper_func
def object_prepare(self, cdist_object, transfer_type_explorers=True):
"""Prepare object: Run type explorer + manifest"""
self._handle_deprecation(cdist_object)
@ -798,7 +818,19 @@ class Config:
cdist_object.name)
self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
try:
self.manifest.run_type_manifest(cdist_object)
self.log.verbose("Preparing object {}".format(cdist_object.name))
self.log.verbose(
"Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object,
transfer_type_explorers)
if pytype_util.is_python_type(cdist_object.cdist_type):
self._timeit(self.manifest.run_py_type_manifest,
"Python type manifest for {}".format(
cdist_object.name))(cdist_object)
else:
self._timeit(self.manifest.run_type_manifest,
"Type manifest for {}".format(
cdist_object.name))(cdist_object)
self.log.trace("[ORDER_DEP] Removing order dep files for %s",
cdist_object)
cdist_object.cleanup()
@ -816,9 +848,21 @@ class Config:
# Generate
self.log.debug("Generating code for %s", cdist_object.name)
cdist_object.code_local = self.code.run_gencode_local(cdist_object)
cdist_object.code_remote = self.code.run_gencode_remote(
cdist_object)
if pytype_util.is_python_type(cdist_object.cdist_type):
cdist_object.code_local = ''
cdist_object.code_remote = self._timeit(
self.code.run_py,
"Python type generate code for {}".format(
cdist_object.name))(cdist_object)
else:
cdist_object.code_local = self._timeit(
self.code.run_gencode_local,
"Type generate code local for {}".format(
cdist_object.name))(cdist_object)
cdist_object.code_remote = self._timeit(
self.code.run_gencode_remote,
"Type generate code remote for {}".format(
cdist_object.name))(cdist_object)
if cdist_object.code_local or cdist_object.code_remote:
cdist_object.changed = True
@ -829,12 +873,16 @@ class Config:
if cdist_object.code_local:
self.log.trace("Executing local code for %s",
cdist_object.name)
self.code.run_code_local(cdist_object)
self._timeit(self.code.run_code_local,
"Type run code local for {}".format(
cdist_object.name))(cdist_object)
if cdist_object.code_remote:
self.log.trace("Executing remote code for %s",
cdist_object.name)
self.code.transfer_code_remote(cdist_object)
self.code.run_code_remote(cdist_object)
self._timeit(self.code.run_code_remote,
"Type run code remote for {}".format(
cdist_object.name))(cdist_object)
# Mark this object as done
self.log.trace("Finishing run of %s", cdist_object.name)

View File

@ -29,3 +29,4 @@ from cdist.core.manifest import Manifest
from cdist.core.code import Code
from cdist.core.util import listdir
from cdist.core.util import log_level_env_var_val, log_level_name_env_var_val
import cdist.core.pytypes

View File

@ -22,6 +22,8 @@
#
import os
import inspect
from cdist.core.pytypes import get_pytype_class
from . import util
@ -120,6 +122,27 @@ class Code:
self.env['__cdist_log_server_socket'] = os.environ[
'__cdist_log_server_socket_export']
def run_py(self, cdist_object):
cdist_type = cdist_object.cdist_type
type_class = get_pytype_class(cdist_type)
if type_class is not None:
env = os.environ.copy()
env.update(self.env)
message_prefix = cdist_object.name
type_obj = type_class(env=env, cdist_object=cdist_object,
local=self.local, remote=self.remote,
message_prefix=message_prefix)
if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run):
if self.local.save_output_streams:
which = 'gencode-py'
stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'a+') as stderr, \
open(stdout_path, 'a+') as stdout:
return type_obj.run(stdout=stdout, stderr=stderr)
else:
return type_obj.run()
def _run_gencode(self, cdist_object, which):
cdist_type = cdist_object.cdist_type
gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which))

View File

@ -22,9 +22,12 @@
import logging
import os
import inspect
import cdist
import cdist.emulator
from . import util
from cdist.core.pytypes import Command, get_pytype_class
'''
common:
@ -96,9 +99,6 @@ class Manifest:
"""
ORDER_DEP_STATE_NAME = 'order_dep_state'
TYPEORDER_DEP_NAME = 'typeorder_dep'
def __init__(self, target_host, local, dry_run=False):
self.target_host = target_host
self.local = local
@ -225,5 +225,58 @@ class Manifest:
os.remove(os.path.join(self.local.base_path, fname))
except FileNotFoundError:
pass
_rm_file(Manifest.ORDER_DEP_STATE_NAME)
_rm_file(Manifest.TYPEORDER_DEP_NAME)
_rm_file(cdist.ORDER_DEP_STATE_NAME)
_rm_file(cdist.TYPEORDER_DEP_NAME)
def env_py_type_manifest(self, cdist_object):
env = os.environ.copy()
env.update(self.env)
env.update({
'__cdist_object_marker': self.local.object_marker_name,
'__cdist_manifest': cdist_object.cdist_type,
'__manifest': self.local.manifest_path,
'__object': cdist_object.absolute_path,
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
'__type': cdist_object.cdist_type.absolute_path,
})
return env
def run_py_type_manifest(self, cdist_object):
cdist_type = cdist_object.cdist_type
type_class = get_pytype_class(cdist_type)
if type_class is not None:
self.log.verbose("Running python type manifest for object %s",
cdist_object.name)
message_prefix = cdist_object.name
env = self.env_py_type_manifest(cdist_object)
type_obj = type_class(env=env, cdist_object=cdist_object,
local=self.local, remote=None,
message_prefix=message_prefix)
if self.local.save_output_streams:
which = 'manifest'
stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'a+') as stderr, \
open(stdout_path, 'a+') as stdout:
self._process_py_type_manifest_entries(
type_obj, env, stdout=stdout, stderr=stderr)
else:
self._process_py_type_manifest_entries(type_obj, env)
def _process_py_type_manifest_entries(self, type_obj, env, stdout=None,
stderr=None):
if hasattr(type_obj, 'manifest') and \
inspect.ismethod(type_obj.manifest):
for cmd in type_obj.manifest(stdout=stdout, stderr=stderr):
if not isinstance(cmd, Command):
raise TypeError("Manifest command must be of type Command")
kwargs = {
'argv': cmd.cmd_line(),
'env': env,
}
if cmd.stdin:
kwargs['stdin'] = cmd.stdin
emulator = cdist.emulator.Emulator(**kwargs)
emulator.run()

190
cdist/core/pytypes.py Normal file
View File

@ -0,0 +1,190 @@
import logging
import os
import io
import sys
import re
from cdist import message, Error
import importlib.util
import inspect
import cdist
__all__ = ["PythonType", "Command", "command"]
class PythonType:
def __init__(self, env, cdist_object, local, remote, message_prefix=None):
self.env = env
self.cdist_object = cdist_object
self.local = local
self.remote = remote
if self.cdist_object:
self.object_id = cdist_object.object_id
self.object_name = cdist_object.name
self.cdist_type = cdist_object.cdist_type
self.object_path = cdist_object.absolute_path
self.explorer_path = os.path.join(self.object_path, 'explorer')
self.type_path = cdist_object.cdist_type.absolute_path
self.parameters = cdist_object.parameters
self.stdin_path = os.path.join(self.object_path, 'stdin')
if self.local:
self.log = logging.getLogger(
self.local.target_host[0] + ':' + self.object_name)
self.message_prefix = message_prefix
self.message = None
def get_parameter(self, name):
return self.parameters.get(name)
def get_explorer_file(self, name):
path = os.path.join(self.explorer_path, name)
return path
def get_explorer(self, name):
path = self.get_explorer_file(name)
with open(path, 'r') as f:
value = f.read()
if value:
value = value.strip()
return value
def run_local(self, command, env=None):
rv = self.local.run(command, env=env, return_output=True)
if rv:
rv = rv.rstrip('\n')
return rv
def run_remote(self, command, env=None):
rv = self.remote.run(command, env=env, return_output=True)
if rv:
rv = rv.rstrip('\n')
return rv
def transfer(self, source, destination):
self.remote.transfer(source, destination)
def die(self, msg):
raise Error("{}: {}".format(self.cdist_object, msg))
def manifest(self, stdout=None, stderr=None):
try:
if self.message_prefix:
self.message = message.Message(self.message_prefix,
self.local.messages_path)
self.env.update(self.message.env)
if stdout is not None:
stdout_save = sys.stdout
sys.stdout = stdout
if stderr is not None:
stderr_save = sys.stderr
sys.stderr = stderr
yield from self.type_manifest()
finally:
if self.message:
self.message.merge_messages()
if stdout is not None:
sys.stdout = stdout_save
if stderr is not None:
sys.stderr = stderr_save
def run(self, stdout=None, stderr=None):
try:
if self.message_prefix:
self.message = message.Message(self.message_prefix,
self.local.messages_path)
if stdout is not None:
stdout_save = sys.stdout
sys.stdout = stdout
if stderr is not None:
stderr_save = sys.stderr
sys.stderr = stderr
return self.type_gencode()
finally:
if self.message:
self.message.merge_messages()
if stdout is not None:
sys.stdout = stdout_save
if stderr is not None:
sys.stderr = stderr_save
def send_message(self, msg):
if self.message:
with open(self.message.messages_out, 'a') as f:
print(msg, file=f)
def receive_message(self, pattern):
if self.message:
with open(self.message.messages_in, 'r') as f:
for line in f:
match = re.search(pattern, line)
if match:
return match
return None
def get_args_parser(self):
pass
def type_manifest(self):
pass
def type_gencode(self):
pass
class Command:
def __init__(self, name, *args, **kwargs):
self.name = name
self.args = args
self.kwargs = kwargs
self.stdin = None
def feed_stdin(self, value):
# If file-like object then read its value.
if value is not None and isinstance(value, io.IOBase):
value = value.read()
# Convert to bytes file-like object.
if value is None:
self.stdin = None
elif isinstance(value, str):
self.stdin = io.BytesIO(value.encode('utf-8'))
elif isinstance(value, bytes) or isinstance(value, bytearray):
self.stdin = io.BytesIO(value)
else:
raise TypeError("value must be str, bytes, bytearray, file-like "
"object or None")
return self
def cmd_line(self):
argv = [self.name, ]
for param in self.args:
argv.append(param)
for key, value in self.kwargs.items():
argv.append("--{}".format(key))
argv.append(value)
return argv
def command(name, *args, **kwargs):
return Command(name, *args, **kwargs)
def get_pytype_class(cdist_type):
module_name = cdist_type.name
file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
type_class = None
if os.path.isfile(file_path):
spec = importlib.util.spec_from_file_location(module_name, file_path)
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
classes = inspect.getmembers(m, inspect.isclass)
for _, cl in classes:
if cl != PythonType and issubclass(cl, PythonType):
if type_class:
raise cdist.Error(
"Only one python type class is supported, but at least"
" two found: {}".format((type_class, cl, )))
else:
type_class = cl
return type_class

View File

@ -30,7 +30,9 @@ import re
import cdist
from cdist import core
from cdist import flock
from cdist.core.manifest import Manifest
import cdist.util.python_type_util as pytype_util
from cdist.core.pytypes import get_pytype_class
import inspect
class MissingRequiredEnvironmentVariableError(cdist.Error):
@ -85,9 +87,9 @@ class Emulator:
self.typeorder_path = os.path.join(self.global_path, "typeorder")
self.typeorder_dep_path = os.path.join(self.global_path,
Manifest.TYPEORDER_DEP_NAME)
cdist.TYPEORDER_DEP_NAME)
self.order_dep_state_path = os.path.join(self.global_path,
Manifest.ORDER_DEP_STATE_NAME)
cdist.ORDER_DEP_STATE_NAME)
self.type_name = os.path.basename(argv[0])
self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
@ -97,7 +99,28 @@ class Emulator:
def run(self):
"""Emulate type commands (i.e. __file and co)"""
self.commandline()
args_parser = None
if pytype_util.is_python_type(self.cdist_type):
type_class = get_pytype_class(self.cdist_type)
if type_class is not None:
# We only need to call parse_args so we need plain instance.
type_obj = type_class(env=None, cdist_object=None, local=None,
remote=None)
if (hasattr(type_obj, 'get_args_parser') and
inspect.ismethod(type_obj.get_args_parser)):
args_parser = type_obj.get_args_parser()
self.log.trace("Using python type argument parser")
print("Using python type argument parser")
if args_parser is None:
# Fallback to classic way.
args_parser = self.get_args_parser()
self.log.trace("Fallback to classic argument parser")
print("Fallback to classic argument parser")
else:
args_parser = self.get_args_parser()
self.log.trace("Using emulator classic argument parser")
print("Using emulator classic argument parser")
self.commandline(args_parser)
self.init_object()
# locking for parallel execution
@ -133,9 +156,7 @@ class Emulator:
colored_log = self.env.get('__cdist_colored_log', 'false')
cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
def commandline(self):
"""Parse command line"""
def get_args_parser(self):
parser = argparse.ArgumentParser(add_help=False,
argument_default=argparse.SUPPRESS)
@ -167,8 +188,10 @@ class Emulator:
# If not singleton support one positional parameter
if not self.cdist_type.is_singleton:
parser.add_argument("object_id", nargs=1)
return parser
# And finally parse/verify parameter
def commandline(self, parser):
"""Parse command line"""
self.args = parser.parse_args(self.argv[1:])
self.log.trace('Args: %s', self.args)

View File

@ -35,6 +35,9 @@ import cdist
import cdist.message
from cdist import core
import cdist.exec.util as util
import cdist.util.python_type_util as pytype_util
from cdist.core import pytypes
import functools
CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
@ -369,3 +372,16 @@ class Local:
raise cdist.Error(
"Linking emulator from {} to {} failed: {}".format(
src, dst, e.__str__()))
def collect_python_types(self):
for cdist_type in core.CdistType.list_types(self.type_path):
if pytype_util.is_python_type(cdist_type):
self.log.trace("Detected python type %s, collecting it".format(
cdist_type.name))
f = functools.partial(pytypes.command, cdist_type.name)
if cdist_type.name.startswith('__'):
attr_name = cdist_type.name.replace('__', '', 1)
else:
attr_name = cdist_type.name
setattr(pytypes, attr_name, f)
pytypes.__all__.append(attr_name)

View File

@ -126,6 +126,12 @@ class Debian:
help="suite used for debootstrap, "
"by default '{}'".format(defargs.suite),
dest='suite', default=defargs.suite)
parser.add_argument(
'-t', '--trigger-command',
help=("trigger command that will be added to cdist config; "
"'__cdist_preos_trigger http ...' type is appended to "
"initial manifest"),
dest='trigger_command')
parser.add_argument(
'-y', '--remote-copy',
help=("remote copy that cdist config will use, by default "

View File

@ -127,6 +127,13 @@ then
exit 1
fi
if [ "${trigger_command}" ]
then
trigger_line="__cdist_preos_trigger http --trigger-command '${trigger_command}'\n"
else
trigger_line=""
fi
if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ]
then
i="$((keyfile_cnt - 1))"
@ -174,7 +181,7 @@ then
fi
grub_lines="${grub_manifest_line}${grub_kern_params_line}"
printf "${ssh_auth_keys_line}${grub_lines}" \
printf "${trigger_line}${ssh_auth_keys_line}${grub_lines}" \
| cat "${manifest}" - |\
cdist config \
${cdist_params} -i - \

229
cdist/trigger.py Normal file
View File

@ -0,0 +1,229 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2016 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import ipaddress
import logging
import re
import socket
import http.server
import os
import socketserver
import shutil
import cdist.config
import cdist.log
import cdist.util.ipaddr as ipaddr
class Trigger():
"""cdist trigger handling"""
# Arguments that are only trigger specific
triggers_args = ["http_port", "ipv6", "directory", "source", ]
def __init__(self, http_port=None, dry_run=False, ipv6=False,
directory=None, source=None, cdistargs=None):
self.dry_run = dry_run
self.http_port = int(http_port)
self.ipv6 = ipv6
self.args = cdistargs
self.directory = directory
self.source = source
log.debug("IPv6: %s", self.ipv6)
def run_httpd(self):
server_address = ('', self.http_port)
if self.ipv6:
httpdcls = HTTPServerV6
else:
httpdcls = HTTPServerV4
httpd = httpdcls(self.args, self.directory, self.source,
server_address, TriggerHttp)
log.debug("Starting server at port %d", self.http_port)
if self.dry_run:
log.debug("Running in dry run mode")
httpd.serve_forever()
def run(self):
if self.http_port:
self.run_httpd()
@classmethod
def commandline(cls, args):
global log
# remove root logger default cdist handler and configure trigger's own
logging.getLogger().handlers = []
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s')
log = logging.getLogger("trigger")
ownargs = {}
for targ in cls.triggers_args:
arg = getattr(args, targ)
ownargs[targ] = arg
del arg