evilham/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Implement python types

Browse source
This commit is contained in:
Darko Poljak 2018-11-01 17:56:40 +01:00 committed by Darko Poljak
parent e447d1aa87
commit 00f85be81b
26 changed files with 1224 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
cdist/conf/type/__file_py/__init__.py Normal file
View file

@ -0,0 +1,103 @@
import os
import re
import sys
from cdist.core.pytypes import *
class FileType(PythonType):
def get_attribute(self, stat_file, attribute, value_should):
if os.path.exists(stat_file):
if re.match('[0-9]', value_should):
index = 1
else:
index = 2
with open(stat_file, 'r') as f:
for line in f:
if re.match(attribute + ":", line):
fields = line.split()
return fields[index]
return None
def set_attribute(self, attribute, value_should, destination):
cmd = {
'group': 'chgrp',
'owner': 'chown',
'mode': 'chmod',
}
self.send_message("{} '{}'".format(cmd[attribute], value_should))
return "{} '{}' '{}'".format(cmd[attribute], value_should, destination)
def type_manifest(self):
yield from ()
def type_gencode(self):
typeis = self.get_explorer('type')
state_should = self.get_parameter('state')
if state_should == 'exists' and typeis == 'file':
return
source = self.get_parameter('source')
if source == '-':
source = self.stdin_path
destination = '/' + self.object_id
if state_should == 'pre-exists':
if source is not None:
self.die('--source cannot be used with --state pre-exists')
if typeis == 'file':
return None
else:
self.die('File {} does not exist'.format(destination))
create_file = False
upload_file = False
set_attributes = False
code = []
if state_should == 'present' or state_should == 'exists':
if source is None:
remote_stat = self.get_explorer('stat')
if not remote_stat:
create_file = True
else:
if os.path.exists(source):
if typeis == 'file':
local_cksum = self.run_local(['cksum', source, ])
local_cksum = local_cksum.split()[0]
remote_cksum = self.get_explorer('cksum')
remote_cksum = remote_cksum.split()[0]
upload_file = local_cksum != remote_cksum
else:
upload_file = True
else:
self.die('Source {} does not exist'.format(source))
if create_file or upload_file:
set_attributes = True
tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination)
destination_upload = self.run_remote(
["mktemp", tempfile_template, ])
if upload_file:
self.transfer(source, destination_upload)
code.append('rm -rf {}'.format(destination))
code.append('mv {} {}'.format(destination_upload, destination))
if state_should in ('present', 'exists', 'pre-exists', ):
for attribute in ('group', 'owner', 'mode', ):
if attribute in self.parameters:
value_should = self.get_parameter(attribute)
if attribute == 'mode':
value_should = re.sub('^0', '', value_should)
stat_file = self.get_explorer_file('stat')
value_is = self.get_attribute(stat_file, attribute,
value_should)
if set_attributes or value_should != value_is:
code.append(self.set_attribute(attribute,
value_should,
destination))
elif state_should == 'absent':
if typeis == 'file':
code.append('rm -f {}'.format(destination))
self.send_message('remove')
else:
self.die('Unknown state {}'.format(state_should))
return "\n".join(code)

34
cdist/conf/type/__file_py/explorer/cksum Executable file
View file

@ -0,0 +1,34 @@
#!/bin/sh
#
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# Retrieve the md5sum of a file to be created, if it is already existing.
#
destination="/$__object_id"
if [ -e "$destination" ]; then
if [ -f "$destination" ]; then
cksum < "$destination"
else
echo "NO REGULAR FILE"
fi
else
echo "NO FILE FOUND, NO CHECKSUM CALCULATED."
fi

56
cdist/conf/type/__file_py/explorer/stat Executable file
View file

@ -0,0 +1,56 @@
#!/bin/sh
#
# 2013 Steven Armstrong (steven-cdist armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
destination="/$__object_id"
# nothing to work with, nothing we could do
[ -e "$destination" ] || exit 0
os=$("$__explorer/os")
case "$os" in
"freebsd"|"netbsd"|"openbsd")
# FIXME: should be something like this based on man page, but can not test
stat -f "type: %ST
owner: %Du %Su
group: %Dg %Sg
mode: %Op %Sp
size: %Dz
links: %Dl
" "$destination"
;;
"macosx")
stat -f "type: %HT
owner: %Du %Su
group: %Dg %Sg
mode: %Lp %Sp
size: %Dz
links: %Dl
" "$destination"
;;
*)
stat --printf="type: %F
owner: %u %U
group: %g %G
mode: %a %A
size: %s
links: %h
" "$destination"
;;
esac

33
cdist/conf/type/__file_py/explorer/type Executable file
View file

@ -0,0 +1,33 @@
#!/bin/sh
#
# 2013 Steven Armstrong (steven-cdist armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
destination="/$__object_id"
if [ ! -e "$destination" ]; then
echo none
elif [ -h "$destination" ]; then
echo symlink
elif [ -f "$destination" ]; then
echo file
elif [ -d "$destination" ]; then
echo directory
else
echo unknown
fi

1
cdist/conf/type/__file_py/parameter/default/state Normal file
View file

@ -0,0 +1 @@
present

5
cdist/conf/type/__file_py/parameter/optional Normal file
View file

@ -0,0 +1,5 @@
state
group
mode
owner
source

65
cdist/config.py
View file

@ -38,6 +38,7 @@ import cdist.hostsource
import cdist.exec.local import cdist.exec.local
import cdist.exec.remote import cdist.exec.remote
import cdist.util.ipaddr as ipaddr import cdist.util.ipaddr as ipaddr
import cdist.util.python_type_util as pytype_util
import cdist.configuration import cdist.configuration
from cdist import core, inventory from cdist import core, inventory
from cdist.util.remoteutil import inspect_ssh_mux_opts from cdist.util.remoteutil import inspect_ssh_mux_opts
@ -90,13 +91,15 @@ class Config(object):
shutil.rmtree(path) shutil.rmtree(path)
def __init__(self, local, remote, dry_run=False, jobs=None, def __init__(self, local, remote, dry_run=False, jobs=None,
cleanup_cmds=None, remove_remote_files_dirs=False): cleanup_cmds=None, remove_remote_files_dirs=False,
timestamp=False):
self.local = local self.local = local
self.remote = remote self.remote = remote
self._open_logger() self._open_logger()
self.dry_run = dry_run self.dry_run = dry_run
self.jobs = jobs self.jobs = jobs
self.timestamp = timestamp
if cleanup_cmds: if cleanup_cmds:
self.cleanup_cmds = cleanup_cmds self.cleanup_cmds = cleanup_cmds
else: else:
@ -428,7 +431,8 @@ class Config(object):
cleanup_cmds.append(cleanup_cmd) cleanup_cmds.append(cleanup_cmd)
c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs, c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs,
cleanup_cmds=cleanup_cmds, cleanup_cmds=cleanup_cmds,
remove_remote_files_dirs=remove_remote_files_dirs) remove_remote_files_dirs=remove_remote_files_dirs,
timestamp=args.timestamp)
c.run() c.run()
cls._remove_paths() cls._remove_paths()
@ -466,6 +470,7 @@ class Config(object):
'dry' if self.dry_run else 'configuration')) 'dry' if self.dry_run else 'configuration'))
self._init_files_dirs() self._init_files_dirs()
self.local.collect_python_types()
self.explorer.run_global_explorers(self.local.global_explorer_out_path) self.explorer.run_global_explorers(self.local.global_explorer_out_path)
try: try:
@ -779,15 +784,41 @@ class Config(object):
args = [param, cdist_type.name] args = [param, cdist_type.name]
self.log.warning(format, *args) self.log.warning(format, *args)
def _timeit(self, func, msg_prefix):
def wrapper_func(*args, **kwargs):
loglevel = self.log.getEffectiveLevel()
if loglevel >= logging.VERBOSE and self.timestamp:
start_time = time.time()
rv = func(*args, **kwargs)
end_time = time.time()
duration = end_time - start_time
self.log.verbose("%s duration: %.6f seconds",
msg_prefix, duration)
else:
rv = func(*args, **kwargs)
return rv
return wrapper_func
def object_prepare(self, cdist_object, transfer_type_explorers=True): def object_prepare(self, cdist_object, transfer_type_explorers=True):
"""Prepare object: Run type explorer + manifest""" """Prepare object: Run type explorer + manifest"""
self._handle_deprecation(cdist_object) self._handle_deprecation(cdist_object)
self.log.verbose("Preparing object {}".format(cdist_object.name)) self.log.verbose("Preparing object {}".format(cdist_object.name))
self.log.verbose( self.log.verbose(
"Running manifest and explorers for " + cdist_object.name) "Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
try: try:
self.manifest.run_type_manifest(cdist_object) self.log.verbose("Preparing object {}".format(cdist_object.name))
self.log.verbose(
"Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object,
transfer_type_explorers)
if pytype_util.is_python_type(cdist_object.cdist_type):
self._timeit(self.manifest.run_py_type_manifest,
"Python type manifest for {}".format(
cdist_object.name))(cdist_object)
else:
self._timeit(self.manifest.run_type_manifest,
"Type manifest for {}".format(
cdist_object.name))(cdist_object)
self.log.trace("[ORDER_DEP] Removing order dep files for %s", self.log.trace("[ORDER_DEP] Removing order dep files for %s",
cdist_object) cdist_object)
cdist_object.cleanup() cdist_object.cleanup()
@ -805,9 +836,21 @@ class Config(object):
# Generate # Generate
self.log.debug("Generating code for %s" % (cdist_object.name)) self.log.debug("Generating code for %s" % (cdist_object.name))
cdist_object.code_local = self.code.run_gencode_local(cdist_object) if pytype_util.is_python_type(cdist_object.cdist_type):
cdist_object.code_remote = self.code.run_gencode_remote( cdist_object.code_local = ''
cdist_object) cdist_object.code_remote = self._timeit(
self.code.run_py,
"Python type generate code for {}".format(
cdist_object.name))(cdist_object)
else:
cdist_object.code_local = self._timeit(
self.code.run_gencode_local,
"Type generate code local for {}".format(
cdist_object.name))(cdist_object)
cdist_object.code_remote = self._timeit(
self.code.run_gencode_remote,
"Type generate code remote for {}".format(
cdist_object.name))(cdist_object)
if cdist_object.code_local or cdist_object.code_remote: if cdist_object.code_local or cdist_object.code_remote:
cdist_object.changed = True cdist_object.changed = True
@ -818,12 +861,16 @@ class Config(object):
if cdist_object.code_local: if cdist_object.code_local:
self.log.trace("Executing local code for %s" self.log.trace("Executing local code for %s"
% (cdist_object.name)) % (cdist_object.name))
self.code.run_code_local(cdist_object) self._timeit(self.code.run_code_local,
"Type run code local for {}".format(
cdist_object.name))(cdist_object)
if cdist_object.code_remote: if cdist_object.code_remote:
self.log.trace("Executing remote code for %s" self.log.trace("Executing remote code for %s"
% (cdist_object.name)) % (cdist_object.name))
self.code.transfer_code_remote(cdist_object) self.code.transfer_code_remote(cdist_object)
self.code.run_code_remote(cdist_object) self._timeit(self.code.run_code_remote,
"Type run code remote for {}".format(
cdist_object.name))(cdist_object)
# Mark this object as done # Mark this object as done
self.log.trace("Finishing run of " + cdist_object.name) self.log.trace("Finishing run of " + cdist_object.name)

1
cdist/core/__init__.py
View file

@ -29,3 +29,4 @@ from cdist.core.manifest import Manifest
from cdist.core.code import Code from cdist.core.code import Code
from cdist.core.util import listdir from cdist.core.util import listdir
from cdist.core.util import log_level_env_var_val, log_level_name_env_var_val from cdist.core.util import log_level_env_var_val, log_level_name_env_var_val
import cdist.core.pytypes

42
cdist/core/code.py
View file

@ -22,6 +22,10 @@
# #
import os import os
import importlib.util
import inspect
import cdist
from cdist.core.pytypes import PythonType
from . import util from . import util
@ -116,6 +120,44 @@ class Code(object):
if dry_run: if dry_run:
self.env['__cdist_dry_run'] = '1' self.env['__cdist_dry_run'] = '1'
def run_py(self, cdist_object):
cdist_type = cdist_object.cdist_type
module_name = cdist_type.name
file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
if os.path.isfile(file_path):
spec = importlib.util.spec_from_file_location(module_name,
file_path)
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
classes = inspect.getmembers(m, inspect.isclass)
type_class = None
for _, cl in classes:
if cl != PythonType and issubclass(cl, PythonType):
if type_class:
raise cdist.Error("Only one python type class is "
"supported, but at least two "
"found: {}".format((type_class,
cl, )))
else:
type_class = cl
env = os.environ.copy()
env.update(self.env)
message_prefix = cdist_object.name
type_obj = type_class(env=env, cdist_object=cdist_object,
local=self.local, remote=self.remote,
message_prefix=message_prefix)
if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run):
if self.local.save_output_streams:
which = 'gencode-py'
stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'a+') as stderr, \
open(stdout_path, 'a+') as stdout:
return type_obj.run(stdout=stdout, stderr=stderr)
else:
return type_obj.run()
def _run_gencode(self, cdist_object, which): def _run_gencode(self, cdist_object, which):
cdist_type = cdist_object.cdist_type cdist_type = cdist_object.cdist_type
script = os.path.join(self.local.type_path, script = os.path.join(self.local.type_path,

75
cdist/core/manifest.py
View file

@ -22,9 +22,13 @@
import logging import logging
import os import os
import importlib.util
import inspect
import cdist import cdist
import cdist.emulator
from . import util from . import util
from cdist.core.pytypes import PythonType, Command
''' '''
common: common:
@ -226,3 +230,72 @@ class Manifest(object):
pass pass
_rm_file(Manifest.ORDER_DEP_STATE_NAME) _rm_file(Manifest.ORDER_DEP_STATE_NAME)
_rm_file(Manifest.TYPEORDER_DEP_NAME) _rm_file(Manifest.TYPEORDER_DEP_NAME)
def env_py_type_manifest(self, cdist_object):
env = os.environ.copy()
env.update(self.env)
env.update({
'__cdist_object_marker': self.local.object_marker_name,
'__cdist_manifest': cdist_object.cdist_type,
'__manifest': self.local.manifest_path,
'__object': cdist_object.absolute_path,
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
'__type': cdist_object.cdist_type.absolute_path,
})
return env
def run_py_type_manifest(self, cdist_object):
cdist_type = cdist_object.cdist_type
module_name = cdist_type.name
file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
message_prefix = cdist_object.name
if os.path.isfile(file_path):
self.log.verbose("Running python type manifest for object %s",
cdist_object.name)
spec = importlib.util.spec_from_file_location(module_name,
file_path)
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
classes = inspect.getmembers(m, inspect.isclass)
type_class = None
for _, cl in classes:
if cl != PythonType and issubclass(cl, PythonType):
if type_class:
raise cdist.Error("Only one python type class is "
"supported, but at least two "
"found: {}".format((type_class,
cl, )))
else:
type_class = cl
env = self.env_py_type_manifest(cdist_object)
type_obj = type_class(env=env, cdist_object=cdist_object,
local=self.local, remote=None,
message_prefix=message_prefix)
if self.local.save_output_streams:
which = 'manifest'
stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'a+') as stderr, \
open(stdout_path, 'a+') as stdout:
self._process_py_type_manifest_entries(
type_obj, env, stdout=stdout, stderr=stderr)
else:
self._process_py_type_manifest_entries(type_obj, env)
def _process_py_type_manifest_entries(self, type_obj, env, stdout=None,
stderr=None):
if hasattr(type_obj, 'manifest') and \
inspect.ismethod(type_obj.manifest):
for cmd in type_obj.manifest(stdout=stdout, stderr=stderr):
if not isinstance(cmd, Command):
raise TypeError("Manifest command must be of type Command")
kwargs = {
'argv': cmd.cmd_line(),
'env': env,
}
if cmd.stdin:
kwargs['stdin'] = cmd.stdin
emulator = cdist.emulator.Emulator(**kwargs)
emulator.run()

162
cdist/core/pytypes.py Normal file
View file

@ -0,0 +1,162 @@
import logging
import os
import io
import sys
import re
from cdist import message, Error
__all__ = ["PythonType", "Command", "command"]
class PythonType:
def __init__(self, env, cdist_object, local, remote, message_prefix=None):
self.env = env
self.cdist_object = cdist_object
self.object_id = cdist_object.object_id
self.object_name = cdist_object.name
self.cdist_type = cdist_object.cdist_type
self.local = local
self.remote = remote
self.object_path = cdist_object.absolute_path
self.type_path = cdist_object.cdist_type.absolute_path
self.explorer_path = os.path.join(self.object_path, 'explorer')
self.parameters = cdist_object.parameters
self.stdin_path = os.path.join(self.object_path, 'stdin')
self.log = logging.getLogger(
self.local.target_host[0] + ':' + self.object_name)
self.message_prefix = message_prefix
self.message = None
def get_parameter(self, name):
return self.parameters.get(name)
def get_explorer_file(self, name):
path = os.path.join(self.explorer_path, name)
return path
def get_explorer(self, name):
path = self.get_explorer_file(name)
with open(path, 'r') as f:
value = f.read()
if value:
value = value.strip()
return value
def run_local(self, command, env=None):
rv = self.local.run(command, env=env, return_output=True)
if rv:
rv = rv.rstrip('\n')
return rv
def run_remote(self, command, env=None):
rv = self.remote.run(command, env=env, return_output=True)
if rv:
rv = rv.rstrip('\n')
return rv
def transfer(self, source, destination):
self.remote.transfer(source, destination)
def die(self, msg):
raise Error("{}: {}".format(self.cdist_object, msg))
def type_manifest(self):
pass
def type_gencode(self):
pass
def manifest(self, stdout=None, stderr=None):
try:
if self.message_prefix:
self.message = message.Message(self.message_prefix,
self.local.messages_path)
self.env.update(self.message.env)
if stdout is not None:
stdout_save = sys.stdout
sys.stdout = stdout
if stderr is not None:
stderr_save = sys.stderr
sys.stderr = stderr
yield from self.type_manifest()
finally:
if self.message:
self.message.merge_messages()
if stdout is not None:
sys.stdout = stdout_save
if stderr is not None:
sys.stderr = stderr_save
def run(self, stdout=None, stderr=None):
try:
if self.message_prefix:
self.message = message.Message(self.message_prefix,
self.local.messages_path)
if stdout is not None:
stdout_save = sys.stdout
sys.stdout = stdout
if stderr is not None:
stderr_save = sys.stderr
sys.stderr = stderr
return self.type_gencode()
finally:
if self.message:
self.message.merge_messages()
if stdout is not None:
sys.stdout = stdout_save
if stderr is not None:
sys.stderr = stderr_save
def send_message(self, msg):
if self.message:
with open(self.message.messages_out, 'a') as f:
print(msg, file=f)
def receive_message(self, pattern):
if self.message:
with open(self.message.messages_in, 'r') as f:
for line in f:
match = re.search(pattern, line)
if match:
return match
return None
class Command:
def __init__(self, name, *args, **kwargs):
self.name = name
self.args = args
self.kwargs = kwargs
self.stdin = None
def feed_stdin(self, value):
# If file-like object then read its value.
if value is not None and isinstance(value, io.IOBase):
value = value.read()
# Convert to bytes file-like object.
if value is None:
self.stdin = None
elif isinstance(value, str):
self.stdin = io.BytesIO(value.encode('utf-8'))
elif isinstance(value, bytes) or isinstance(value, bytearray):
self.stdin = io.BytesIO(value)
else:
raise TypeError("value must be str, bytes, bytearray, file-like "
"object or None")
return self
def cmd_line(self):
argv = [self.name, ]
for param in self.args:
argv.append(param)
for key, value in self.kwargs.items():
argv.append("--{}".format(key))
argv.append(value)
return argv
def command(name, *args, **kwargs):
return Command(name, *args, **kwargs)

16
cdist/exec/local.py
View file

@ -35,6 +35,9 @@ import cdist
import cdist.message import cdist.message
from cdist import core from cdist import core
import cdist.exec.util as util import cdist.exec.util as util
import cdist.util.python_type_util as pytype_util
from cdist.core import pytypes
import functools
CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
@ -398,3 +401,16 @@ class Local(object):
raise cdist.Error( raise cdist.Error(
"Linking emulator from %s to %s failed: %s" % ( "Linking emulator from %s to %s failed: %s" % (
src, dst, e.__str__())) src, dst, e.__str__()))
def collect_python_types(self):
for cdist_type in core.CdistType.list_types(self.type_path):
if pytype_util.is_python_type(cdist_type):
self.log.trace("Detected python type %s, collecting it".format(
cdist_type.name))
f = functools.partial(pytypes.command, cdist_type.name)
if cdist_type.name.startswith('__'):
attr_name = cdist_type.name.replace('__', '', 1)
else:
attr_name = cdist_type.name
setattr(pytypes, attr_name, f)
pytypes.__all__.append(attr_name)

27
cdist/util/python_type_util.py Normal file
View file

@ -0,0 +1,27 @@
# -*- coding: utf-8 -*-
#
# 2019 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
def is_python_type(cdist_type):
init_path = os.path.join(cdist_type.absolute_path, '__init__.py')
return os.path.exists(init_path)

1
docs/changelog
View file

@ -3,6 +3,7 @@ Changelog
next: next:
* Core: Add trigger functionality (Nico Schottelius, Darko Poljak) * Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
* Core: Implement core support for python types (Darko Poljak)
6.2.0: 2019-11-30 6.2.0: 2019-11-30
* Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak)

376
docs/dev/python-types/benchmark Normal file
View file

@ -0,0 +1,376 @@
# sh type, no file at remote
echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
INFO: [20181121125154.045799] 185.203.112.26: Starting configuration run
INFO: [20181121125237.029892] 185.203.112.26: Processing __file/root/foo0.bin
INFO: [20181121125239.881529] 185.203.112.26: Processing __file/root/foo1.bin
INFO: [20181121125243.265672] 185.203.112.26: Processing __file/root/foo2.bin
INFO: [20181121125246.929903] 185.203.112.26: Processing __file/root/foo3.bin
INFO: [20181121125251.811258] 185.203.112.26: Processing __file/root/foo4.bin
INFO: [20181121125257.784416] 185.203.112.26: Processing __file/root/foo5.bin
INFO: [20181121125302.686275] 185.203.112.26: Processing __file/root/foo6.bin
INFO: [20181121125306.394030] 185.203.112.26: Processing __file/root/foo7.bin
INFO: [20181121125308.610023] 185.203.112.26: Processing __file/root/foo8.bin
INFO: [20181121125310.868538] 185.203.112.26: Processing __file/root/foo9.bin
INFO: [20181121125313.017972] 185.203.112.26: Processing __file/root/foo10.bin
INFO: [20181121125315.201342] 185.203.112.26: Processing __file/root/foo11.bin
INFO: [20181121125317.333055] 185.203.112.26: Processing __file/root/foo12.bin
INFO: [20181121125319.463929] 185.203.112.26: Processing __file/root/foo13.bin
INFO: [20181121125321.595410] 185.203.112.26: Processing __file/root/foo14.bin
INFO: [20181121125323.689697] 185.203.112.26: Processing __file/root/foo15.bin
INFO: [20181121125325.768283] 185.203.112.26: Processing __file/root/foo16.bin
INFO: [20181121125327.814793] 185.203.112.26: Processing __file/root/foo17.bin
INFO: [20181121125329.873073] 185.203.112.26: Processing __file/root/foo18.bin
INFO: [20181121125331.953886] 185.203.112.26: Processing __file/root/foo19.bin
INFO: [20181121125334.118290] 185.203.112.26: Processing __file/root/foo20.bin
INFO: [20181121125336.390849] 185.203.112.26: Processing __file/root/foo21.bin
INFO: [20181121125338.576698] 185.203.112.26: Processing __file/root/foo22.bin
INFO: [20181121125340.819044] 185.203.112.26: Processing __file/root/foo23.bin
INFO: [20181121125343.680419] 185.203.112.26: Processing __file/root/foo24.bin
INFO: [20181121125346.044907] 185.203.112.26: Processing __file/root/foo25.bin
INFO: [20181121125348.179574] 185.203.112.26: Processing __file/root/foo26.bin
INFO: [20181121125350.314970] 185.203.112.26: Processing __file/root/foo27.bin
INFO: [20181121125352.447394] 185.203.112.26: Processing __file/root/foo28.bin
INFO: [20181121125354.586637] 185.203.112.26: Processing __file/root/foo29.bin
INFO: [20181121125356.722699] 185.203.112.26: Processing __file/root/foo30.bin
INFO: [20181121125358.883538] 185.203.112.26: Processing __file/root/foo31.bin
INFO: [20181121125401.020967] 185.203.112.26: Processing __file/root/foo32.bin
INFO: [20181121125403.160146] 185.203.112.26: Processing __file/root/foo33.bin
INFO: [20181121125405.289048] 185.203.112.26: Processing __file/root/foo34.bin
INFO: [20181121125407.423994] 185.203.112.26: Processing __file/root/foo35.bin
INFO: [20181121125409.530135] 185.203.112.26: Processing __file/root/foo36.bin
INFO: [20181121125411.659683] 185.203.112.26: Processing __file/root/foo37.bin
INFO: [20181121125413.786177] 185.203.112.26: Processing __file/root/foo38.bin
INFO: [20181121125415.919152] 185.203.112.26: Processing __file/root/foo39.bin
INFO: [20181121125418.051496] 185.203.112.26: Processing __file/root/foo40.bin
INFO: [20181121125420.204577] 185.203.112.26: Processing __file/root/foo41.bin
INFO: [20181121125422.339697] 185.203.112.26: Processing __file/root/foo42.bin
INFO: [20181121125424.450966] 185.203.112.26: Processing __file/root/foo43.bin
INFO: [20181121125426.487831] 185.203.112.26: Processing __file/root/foo44.bin
INFO: [20181121125428.585516] 185.203.112.26: Processing __file/root/foo45.bin
INFO: [20181121125430.749002] 185.203.112.26: Processing __file/root/foo46.bin
INFO: [20181121125432.865290] 185.203.112.26: Processing __file/root/foo47.bin
INFO: [20181121125435.004009] 185.203.112.26: Processing __file/root/foo48.bin
INFO: [20181121125437.228566] 185.203.112.26: Processing __file/root/foo49.bin
INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds
# sh type, files exist at remote but content changes
echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
INFO: [20181121125529.952800] 185.203.112.26: Starting configuration run
INFO: [20181121125541.175180] 185.203.112.26: Processing __file/root/foo0.bin
INFO: [20181121125543.219561] 185.203.112.26: Processing __file/root/foo1.bin
INFO: [20181121125545.116073] 185.203.112.26: Processing __file/root/foo2.bin
INFO: [20181121125547.011359] 185.203.112.26: Processing __file/root/foo3.bin
INFO: [20181121125548.916288] 185.203.112.26: Processing __file/root/foo4.bin
INFO: [20181121125550.821351] 185.203.112.26: Processing __file/root/foo5.bin
INFO: [20181121125552.723887] 185.203.112.26: Processing __file/root/foo6.bin
INFO: [20181121125554.635662] 185.203.112.26: Processing __file/root/foo7.bin
INFO: [20181121125556.568639] 185.203.112.26: Processing __file/root/foo8.bin
INFO: [20181121125558.508852] 185.203.112.26: Processing __file/root/foo9.bin
INFO: [20181121125600.464475] 185.203.112.26: Processing __file/root/foo10.bin
INFO: [20181121125602.429261] 185.203.112.26: Processing __file/root/foo11.bin
INFO: [20181121125604.428942] 185.203.112.26: Processing __file/root/foo12.bin
INFO: [20181121125606.442193] 185.203.112.26: Processing __file/root/foo13.bin
INFO: [20181121125608.474473] 185.203.112.26: Processing __file/root/foo14.bin
INFO: [20181121125610.535252] 185.203.112.26: Processing __file/root/foo15.bin
INFO: [20181121125612.609560] 185.203.112.26: Processing __file/root/foo16.bin
INFO: [20181121125614.708507] 185.203.112.26: Processing __file/root/foo17.bin
INFO: [20181121125616.824721] 185.203.112.26: Processing __file/root/foo18.bin
INFO: [20181121125618.924521] 185.203.112.26: Processing __file/root/foo19.bin
INFO: [20181121125621.007543] 185.203.112.26: Processing __file/root/foo20.bin
INFO: [20181121125623.133204] 185.203.112.26: Processing __file/root/foo21.bin
INFO: [20181121125625.333471] 185.203.112.26: Processing __file/root/foo22.bin
INFO: [20181121125627.396334] 185.203.112.26: Processing __file/root/foo23.bin
INFO: [20181121125629.526492] 185.203.112.26: Processing __file/root/foo24.bin
INFO: [20181121125631.628454] 185.203.112.26: Processing __file/root/foo25.bin
INFO: [20181121125633.743142] 185.203.112.26: Processing __file/root/foo26.bin
INFO: [20181121125635.952547] 185.203.112.26: Processing __file/root/foo27.bin
INFO: [20181121125637.986746] 185.203.112.26: Processing __file/root/foo28.bin
INFO: [20181121125640.020415] 185.203.112.26: Processing __file/root/foo29.bin
INFO: [20181121125642.081373] 185.203.112.26: Processing __file/root/foo30.bin
INFO: [20181121125644.174744] 185.203.112.26: Processing __file/root/foo31.bin
INFO: [20181121125646.286532] 185.203.112.26: Processing __file/root/foo32.bin
INFO: [20181121125648.396447] 185.203.112.26: Processing __file/root/foo33.bin
INFO: [20181121125650.460107] 185.203.112.26: Processing __file/root/foo34.bin
INFO: [20181121125652.557125] 185.203.112.26: Processing __file/root/foo35.bin
INFO: [20181121125654.667456] 185.203.112.26: Processing __file/root/foo36.bin
INFO: [20181121125656.746960] 185.203.112.26: Processing __file/root/foo37.bin
INFO: [20181121125658.854229] 185.203.112.26: Processing __file/root/foo38.bin
INFO: [20181121125700.968145] 185.203.112.26: Processing __file/root/foo39.bin
INFO: [20181121125703.109376] 185.203.112.26: Processing __file/root/foo40.bin
INFO: [20181121125705.318163] 185.203.112.26: Processing __file/root/foo41.bin
INFO: [20181121125707.440575] 185.203.112.26: Processing __file/root/foo42.bin
INFO: [20181121125709.551261] 185.203.112.26: Processing __file/root/foo43.bin
INFO: [20181121125711.657753] 185.203.112.26: Processing __file/root/foo44.bin
INFO: [20181121125713.774819] 185.203.112.26: Processing __file/root/foo45.bin
INFO: [20181121125715.887428] 185.203.112.26: Processing __file/root/foo46.bin
INFO: [20181121125717.995104] 185.203.112.26: Processing __file/root/foo47.bin
INFO: [20181121125720.110196] 185.203.112.26: Processing __file/root/foo48.bin
INFO: [20181121125722.232932] 185.203.112.26: Processing __file/root/foo49.bin
INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds
# py type, no file at remote
echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
INFO: [20181121125812.034197] 185.203.112.26: Starting configuration run
INFO: [20181121125823.927353] 185.203.112.26: Processing __file_py/root/foo0.bin
INFO: [20181121125825.715361] 185.203.112.26: Processing __file_py/root/foo1.bin
INFO: [20181121125827.511296] 185.203.112.26: Processing __file_py/root/foo2.bin
INFO: [20181121125829.293455] 185.203.112.26: Processing __file_py/root/foo3.bin
INFO: [20181121125831.086696] 185.203.112.26: Processing __file_py/root/foo4.bin
INFO: [20181121125832.867582] 185.203.112.26: Processing __file_py/root/foo5.bin
INFO: [20181121125834.652511] 185.203.112.26: Processing __file_py/root/foo6.bin
INFO: [20181121125836.450393] 185.203.112.26: Processing __file_py/root/foo7.bin
INFO: [20181121125838.255152] 185.203.112.26: Processing __file_py/root/foo8.bin
INFO: [20181121125840.065808] 185.203.112.26: Processing __file_py/root/foo9.bin
INFO: [20181121125841.889049] 185.203.112.26: Processing __file_py/root/foo10.bin
INFO: [20181121125843.719280] 185.203.112.26: Processing __file_py/root/foo11.bin
INFO: [20181121125845.560165] 185.203.112.26: Processing __file_py/root/foo12.bin
INFO: [20181121125847.416138] 185.203.112.26: Processing __file_py/root/foo13.bin
INFO: [20181121125849.289851] 185.203.112.26: Processing __file_py/root/foo14.bin
INFO: [20181121125851.180203] 185.203.112.26: Processing __file_py/root/foo15.bin
INFO: [20181121125853.074978] 185.203.112.26: Processing __file_py/root/foo16.bin
INFO: [20181121125855.086107] 185.203.112.26: Processing __file_py/root/foo17.bin
INFO: [20181121125857.041100] 185.203.112.26: Processing __file_py/root/foo18.bin
INFO: [20181121125859.025581] 185.203.112.26: Processing __file_py/root/foo19.bin
INFO: [20181121125901.072067] 185.203.112.26: Processing __file_py/root/foo20.bin
INFO: [20181121125903.026711] 185.203.112.26: Processing __file_py/root/foo21.bin
INFO: [20181121125904.994824] 185.203.112.26: Processing __file_py/root/foo22.bin
INFO: [20181121125906.956296] 185.203.112.26: Processing __file_py/root/foo23.bin
INFO: [20181121125908.929231] 185.203.112.26: Processing __file_py/root/foo24.bin
INFO: [20181121125910.882672] 185.203.112.26: Processing __file_py/root/foo25.bin
INFO: [20181121125912.839834] 185.203.112.26: Processing __file_py/root/foo26.bin
INFO: [20181121125914.789904] 185.203.112.26: Processing __file_py/root/foo27.bin
INFO: [20181121125916.743930] 185.203.112.26: Processing __file_py/root/foo28.bin
INFO: [20181121125918.698258] 185.203.112.26: Processing __file_py/root/foo29.bin
INFO: [20181121125920.657118] 185.203.112.26: Processing __file_py/root/foo30.bin
INFO: [20181121125922.618898] 185.203.112.26: Processing __file_py/root/foo31.bin
INFO: [20181121125924.567847] 185.203.112.26: Processing __file_py/root/foo32.bin
INFO: [20181121125926.524617] 185.203.112.26: Processing __file_py/root/foo33.bin
INFO: [20181121125928.396400] 185.203.112.26: Processing __file_py/root/foo34.bin
INFO: [20181121125930.209237] 185.203.112.26: Processing __file_py/root/foo35.bin
INFO: [20181121125931.998377] 185.203.112.26: Processing __file_py/root/foo36.bin
INFO: [20181121125933.786883] 185.203.112.26: Processing __file_py/root/foo37.bin
INFO: [20181121125935.579348] 185.203.112.26: Processing __file_py/root/foo38.bin
INFO: [20181121125937.366197] 185.203.112.26: Processing __file_py/root/foo39.bin
INFO: [20181121125939.155643] 185.203.112.26: Processing __file_py/root/foo40.bin
INFO: [20181121125941.052837] 185.203.112.26: Processing __file_py/root/foo41.bin
INFO: [20181121125942.953670] 185.203.112.26: Processing __file_py/root/foo42.bin
INFO: [20181121125944.781567] 185.203.112.26: Processing __file_py/root/foo43.bin
INFO: [20181121125946.622485] 185.203.112.26: Processing __file_py/root/foo44.bin
INFO: [20181121125948.470701] 185.203.112.26: Processing __file_py/root/foo45.bin
INFO: [20181121125950.356949] 185.203.112.26: Processing __file_py/root/foo46.bin
INFO: [20181121125952.232014] 185.203.112.26: Processing __file_py/root/foo47.bin
INFO: [20181121125954.128887] 185.203.112.26: Processing __file_py/root/foo48.bin
INFO: [20181121125956.037541] 185.203.112.26: Processing __file_py/root/foo49.bin
INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds
# py type, files exist at remote but content changes
echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
INFO: [20181121130056.484643] 185.203.112.26: Starting configuration run
INFO: [20181121130108.545059] 185.203.112.26: Processing __file_py/root/foo0.bin
INFO: [20181121130110.339217] 185.203.112.26: Processing __file_py/root/foo1.bin
INFO: [20181121130112.136448] 185.203.112.26: Processing __file_py/root/foo2.bin
INFO: [20181121130113.923820] 185.203.112.26: Processing __file_py/root/foo3.bin
INFO: [20181121130115.715667] 185.203.112.26: Processing __file_py/root/foo4.bin
INFO: [20181121130117.508696] 185.203.112.26: Processing __file_py/root/foo5.bin
INFO: [20181121130119.300839] 185.203.112.26: Processing __file_py/root/foo6.bin
INFO: [20181121130124.296312] 185.203.112.26: Processing __file_py/root/foo7.bin
INFO: [20181121130131.109195] 185.203.112.26: Processing __file_py/root/foo8.bin
INFO: [20181121130133.303817] 185.203.112.26: Processing __file_py/root/foo9.bin
INFO: [20181121130136.396440] 185.203.112.26: Processing __file_py/root/foo10.bin
INFO: [20181121130138.443128] 185.203.112.26: Processing __file_py/root/foo11.bin
INFO: [20181121130140.462868] 185.203.112.26: Processing __file_py/root/foo12.bin
INFO: [20181121130142.476196] 185.203.112.26: Processing __file_py/root/foo13.bin
INFO: [20181121130145.937900] 185.203.112.26: Processing __file_py/root/foo14.bin
INFO: [20181121130148.013672] 185.203.112.26: Processing __file_py/root/foo15.bin
INFO: [20181121130150.042588] 185.203.112.26: Processing __file_py/root/foo16.bin
INFO: [20181121130152.050793] 185.203.112.26: Processing __file_py/root/foo17.bin
INFO: [20181121130154.083089] 185.203.112.26: Processing __file_py/root/foo18.bin
INFO: [20181121130156.100091] 185.203.112.26: Processing __file_py/root/foo19.bin
INFO: [20181121130158.103005] 185.203.112.26: Processing __file_py/root/foo20.bin
INFO: [20181121130200.188390] 185.203.112.26: Processing __file_py/root/foo21.bin
INFO: [20181121130202.197574] 185.203.112.26: Processing __file_py/root/foo22.bin
INFO: [20181121130205.269102] 185.203.112.26: Processing __file_py/root/foo23.bin
INFO: [20181121130208.457011] 185.203.112.26: Processing __file_py/root/foo24.bin
INFO: [20181121130211.574321] 185.203.112.26: Processing __file_py/root/foo25.bin
INFO: [20181121130213.719894] 185.203.112.26: Processing __file_py/root/foo26.bin
INFO: [20181121130215.762977] 185.203.112.26: Processing __file_py/root/foo27.bin
INFO: [20181121130217.778624] 185.203.112.26: Processing __file_py/root/foo28.bin
INFO: [20181121130219.840477] 185.203.112.26: Processing __file_py/root/foo29.bin
INFO: [20181121130221.852389] 185.203.112.26: Processing __file_py/root/foo30.bin
INFO: [20181121130223.850898] 185.203.112.26: Processing __file_py/root/foo31.bin
INFO: [20181121130225.858812] 185.203.112.26: Processing __file_py/root/foo32.bin
INFO: [20181121130227.855295] 185.203.112.26: Processing __file_py/root/foo33.bin
INFO: [20181121130229.952673] 185.203.112.26: Processing __file_py/root/foo34.bin
INFO: [20181121130231.956904] 185.203.112.26: Processing __file_py/root/foo35.bin
INFO: [20181121130233.961954] 185.203.112.26: Processing __file_py/root/foo36.bin
INFO: [20181121130236.012158] 185.203.112.26: Processing __file_py/root/foo37.bin
INFO: [20181121130238.024422] 185.203.112.26: Processing __file_py/root/foo38.bin
INFO: [20181121130241.238800] 185.203.112.26: Processing __file_py/root/foo39.bin
INFO: [20181121130243.463237] 185.203.112.26: Processing __file_py/root/foo40.bin
INFO: [20181121130245.610314] 185.203.112.26: Processing __file_py/root/foo41.bin
INFO: [20181121130247.661385] 185.203.112.26: Processing __file_py/root/foo42.bin
INFO: [20181121130250.399845] 185.203.112.26: Processing __file_py/root/foo43.bin
INFO: [20181121130252.832133] 185.203.112.26: Processing __file_py/root/foo44.bin
INFO: [20181121130254.955658] 185.203.112.26: Processing __file_py/root/foo45.bin
INFO: [20181121130257.039587] 185.203.112.26: Processing __file_py/root/foo46.bin
INFO: [20181121130259.178847] 185.203.112.26: Processing __file_py/root/foo47.bin
INFO: [20181121130301.357922] 185.203.112.26: Processing __file_py/root/foo48.bin
INFO: [20181121130303.356299] 185.203.112.26: Processing __file_py/root/foo49.bin
INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds
# init test file content
head -c 102400 /dev/random > /tmp/test.file
# sh type, no file at remote
echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
INFO: [20181121130612.519698] 185.203.112.26: Starting configuration run
INFO: [20181121130624.219344] 185.203.112.26: Processing __file/root/foo0.bin
INFO: [20181121130626.980052] 185.203.112.26: Processing __file/root/foo1.bin
INFO: [20181121130631.200669] 185.203.112.26: Processing __file/root/foo2.bin
INFO: [20181121130642.790229] 185.203.112.26: Processing __file/root/foo3.bin
INFO: [20181121130646.565599] 185.203.112.26: Processing __file/root/foo4.bin
INFO: [20181121130648.724875] 185.203.112.26: Processing __file/root/foo5.bin
INFO: [20181121130651.464686] 185.203.112.26: Processing __file/root/foo6.bin
INFO: [20181121130653.639581] 185.203.112.26: Processing __file/root/foo7.bin
INFO: [20181121130655.773987] 185.203.112.26: Processing __file/root/foo8.bin
INFO: [20181121130657.933136] 185.203.112.26: Processing __file/root/foo9.bin
INFO: [20181121130700.065158] 185.203.112.26: Processing __file/root/foo10.bin
INFO: [20181121130702.216456] 185.203.112.26: Processing __file/root/foo11.bin
INFO: [20181121130704.429030] 185.203.112.26: Processing __file/root/foo12.bin
INFO: [20181121130706.562114] 185.203.112.26: Processing __file/root/foo13.bin
INFO: [20181121130708.696584] 185.203.112.26: Processing __file/root/foo14.bin
INFO: [20181121130710.830002] 185.203.112.26: Processing __file/root/foo15.bin
INFO: [20181121130712.966631] 185.203.112.26: Processing __file/root/foo16.bin
INFO: [20181121130715.151833] 185.203.112.26: Processing __file/root/foo17.bin
INFO: [20181121130717.355196] 185.203.112.26: Processing __file/root/foo18.bin
INFO: [20181121130719.486316] 185.203.112.26: Processing __file/root/foo19.bin
INFO: [20181121130721.619933] 185.203.112.26: Processing __file/root/foo20.bin
INFO: [20181121130723.786670] 185.203.112.26: Processing __file/root/foo21.bin
INFO: [20181121130725.924736] 185.203.112.26: Processing __file/root/foo22.bin
INFO: [20181121130728.060224] 185.203.112.26: Processing __file/root/foo23.bin
INFO: [20181121130730.178729] 185.203.112.26: Processing __file/root/foo24.bin
INFO: [20181121130732.309264] 185.203.112.26: Processing __file/root/foo25.bin
INFO: [20181121130734.479895] 185.203.112.26: Processing __file/root/foo26.bin
INFO: [20181121130736.653085] 185.203.112.26: Processing __file/root/foo27.bin
INFO: [20181121130738.814291] 185.203.112.26: Processing __file/root/foo28.bin
INFO: [20181121130741.029646] 185.203.112.26: Processing __file/root/foo29.bin
INFO: [20181121130743.128717] 185.203.112.26: Processing __file/root/foo30.bin
INFO: [20181121130745.233272] 185.203.112.26: Processing __file/root/foo31.bin
INFO: [20181121130747.364681] 185.203.112.26: Processing __file/root/foo32.bin
INFO: [20181121130749.491793] 185.203.112.26: Processing __file/root/foo33.bin
INFO: [20181121130751.620492] 185.203.112.26: Processing __file/root/foo34.bin
INFO: [20181121130753.743519] 185.203.112.26: Processing __file/root/foo35.bin
INFO: [20181121130755.862169] 185.203.112.26: Processing __file/root/foo36.bin
INFO: [20181121130758.000172] 185.203.112.26: Processing __file/root/foo37.bin
INFO: [20181121130800.090405] 185.203.112.26: Processing __file/root/foo38.bin
INFO: [20181121130802.211849] 185.203.112.26: Processing __file/root/foo39.bin
INFO: [20181121130804.356363] 185.203.112.26: Processing __file/root/foo40.bin
INFO: [20181121130806.548412] 185.203.112.26: Processing __file/root/foo41.bin
INFO: [20181121130808.671279] 185.203.112.26: Processing __file/root/foo42.bin
INFO: [20181121130810.752813] 185.203.112.26: Processing __file/root/foo43.bin
INFO: [20181121130812.844502] 185.203.112.26: Processing __file/root/foo44.bin
INFO: [20181121130814.950501] 185.203.112.26: Processing __file/root/foo45.bin
INFO: [20181121130817.040587] 185.203.112.26: Processing __file/root/foo46.bin
INFO: [20181121130819.175850] 185.203.112.26: Processing __file/root/foo47.bin
INFO: [20181121130821.332900] 185.203.112.26: Processing __file/root/foo48.bin
INFO: [20181121130823.543119] 185.203.112.26: Processing __file/root/foo49.bin
INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds
# sh type, files exist at remote
echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
INFO: [20181121130854.980007] 185.203.112.26: Starting configuration run
INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds
# py type, no file at remote
echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
INFO: [20181121131110.179480] 185.203.112.26: Starting configuration run
INFO: [20181121131122.086849] 185.203.112.26: Processing __file_py/root/foo0.bin
INFO: [20181121131123.876029] 185.203.112.26: Processing __file_py/root/foo1.bin
INFO: [20181121131125.668664] 185.203.112.26: Processing __file_py/root/foo2.bin
INFO: [20181121131127.460721] 185.203.112.26: Processing __file_py/root/foo3.bin
INFO: [20181121131129.591229] 185.203.112.26: Processing __file_py/root/foo4.bin
INFO: [20181121131131.390379] 185.203.112.26: Processing __file_py/root/foo5.bin
INFO: [20181121131133.195275] 185.203.112.26: Processing __file_py/root/foo6.bin
INFO: [20181121131135.006282] 185.203.112.26: Processing __file_py/root/foo7.bin
INFO: [20181121131136.834448] 185.203.112.26: Processing __file_py/root/foo8.bin
INFO: [20181121131138.659301] 185.203.112.26: Processing __file_py/root/foo9.bin
INFO: [20181121131140.496856] 185.203.112.26: Processing __file_py/root/foo10.bin
INFO: [20181121131142.367813] 185.203.112.26: Processing __file_py/root/foo11.bin
INFO: [20181121131144.239817] 185.203.112.26: Processing __file_py/root/foo12.bin
INFO: [20181121131146.133314] 185.203.112.26: Processing __file_py/root/foo13.bin
INFO: [20181121131148.049380] 185.203.112.26: Processing __file_py/root/foo14.bin
INFO: [20181121131149.974696] 185.203.112.26: Processing __file_py/root/foo15.bin
INFO: [20181121131151.929083] 185.203.112.26: Processing __file_py/root/foo16.bin
INFO: [20181121131153.923590] 185.203.112.26: Processing __file_py/root/foo17.bin
INFO: [20181121131155.874910] 185.203.112.26: Processing __file_py/root/foo18.bin
INFO: [20181121131157.857904] 185.203.112.26: Processing __file_py/root/foo19.bin
INFO: [20181121131159.902006] 185.203.112.26: Processing __file_py/root/foo20.bin
INFO: [20181121131201.859840] 185.203.112.26: Processing __file_py/root/foo21.bin
INFO: [20181121131203.810875] 185.203.112.26: Processing __file_py/root/foo22.bin
INFO: [20181121131205.763291] 185.203.112.26: Processing __file_py/root/foo23.bin
INFO: [20181121131207.710932] 185.203.112.26: Processing __file_py/root/foo24.bin
INFO: [20181121131209.658154] 185.203.112.26: Processing __file_py/root/foo25.bin
INFO: [20181121131211.615374] 185.203.112.26: Processing __file_py/root/foo26.bin
INFO: [20181121131213.569721] 185.203.112.26: Processing __file_py/root/foo27.bin
INFO: [20181121131215.522624] 185.203.112.26: Processing __file_py/root/foo28.bin
INFO: [20181121131217.471128] 185.203.112.26: Processing __file_py/root/foo29.bin
INFO: [20181121131219.421712] 185.203.112.26: Processing __file_py/root/foo30.bin
INFO: [20181121131221.375699] 185.203.112.26: Processing __file_py/root/foo31.bin
INFO: [20181121131223.327672] 185.203.112.26: Processing __file_py/root/foo32.bin
INFO: [20181121131225.281373] 185.203.112.26: Processing __file_py/root/foo33.bin
INFO: [20181121131227.256711] 185.203.112.26: Processing __file_py/root/foo34.bin
INFO: [20181121131229.209255] 185.203.112.26: Processing __file_py/root/foo35.bin
INFO: [20181121131231.170170] 185.203.112.26: Processing __file_py/root/foo36.bin
INFO: [20181121131233.123407] 185.203.112.26: Processing __file_py/root/foo37.bin
INFO: [20181121131235.077713] 185.203.112.26: Processing __file_py/root/foo38.bin
INFO: [20181121131237.017138] 185.203.112.26: Processing __file_py/root/foo39.bin
INFO: [20181121131238.988189] 185.203.112.26: Processing __file_py/root/foo40.bin
INFO: [20181121131241.026849] 185.203.112.26: Processing __file_py/root/foo41.bin
INFO: [20181121131242.978335] 185.203.112.26: Processing __file_py/root/foo42.bin
INFO: [20181121131244.934562] 185.203.112.26: Processing __file_py/root/foo43.bin
INFO: [20181121131246.885320] 185.203.112.26: Processing __file_py/root/foo44.bin
INFO: [20181121131248.835008] 185.203.112.26: Processing __file_py/root/foo45.bin
INFO: [20181121131250.789727] 185.203.112.26: Processing __file_py/root/foo46.bin
INFO: [20181121131252.738686] 185.203.112.26: Processing __file_py/root/foo47.bin
INFO: [20181121131254.691465] 185.203.112.26: Processing __file_py/root/foo48.bin
INFO: [20181121131256.640896] 185.203.112.26: Processing __file_py/root/foo49.bin
INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds
# py type, files exist at remote
echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26
INFO: [20181121131327.054523] 185.203.112.26: Starting configuration run
INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds
# Summary
# sh type, no file at remote
INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds
# py type, no file at remote
INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds
# sh type, files exist at remote but content changes
INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds
# py type, files exist at remote but content changes
INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds
# sh type, no file at remote
INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds
# py type, no file at remote
INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds
# sh type, files exist at remote
INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds
# py type, files exist at remote
INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds

43
docs/dev/python-types/benchmark.sh Executable file
View file

@ -0,0 +1,43 @@
#!/bin/sh
# Addapt to your env.
CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
export CDIST_PATH
TARGET_HOST=185.203.112.26
if [ $# -eq 0 ]
then
N=1
else
N=$1
fi
manifest() {
bytes=$(echo "$1 * 1024" | bc)
echo "head -c ${bytes} /dev/random | __file$2 /root/foo$3.bin --source - --mode 0640 --owner root --group root"
}
verbosity="-vv" #"-vvv"
i=0
while [ "$i" -lt "$N" ]
do
if [ "$N" -ne 1 ]
then
printf "iteration %d\\n" "$i"
fi
printf "shinit clean state...\\n"
ssh root@${TARGET_HOST} 'rm foo$i.bin;'
manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
printf "pyinit clean state...\\n"
ssh root@${TARGET_HOST} 'rm foo$i.bin;'
manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
printf "shinit present state...\\n"
manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
printf "pyinit present state...\\n"
manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST}
i=$((i + 1))
done

7
docs/dev/python-types/conf/manifest/pyinit Normal file
View file

@ -0,0 +1,7 @@
#for x in 1; do
# echo xxx${x} | __file_py /root/foobar${x} --source - --mode 0640 --owner root --group root;
#done
#__dummy_config_py test1
echo xxx | __file_py /root/foobar --source - --mode 0640 --owner root --group root
__dummy_config_py test1

7
docs/dev/python-types/conf/manifest/shinit Normal file
View file

@ -0,0 +1,7 @@
#for x in 1; do
# echo xxx${x} | __file /root/foobar${x} --source - --mode 0640 --owner root --group root;
#done
#__dummy_config_sh test1
echo xxx | __file /root/foobar --source - --mode 0640 --owner root --group root
__dummy_config_sh test1

22
docs/dev/python-types/conf/type/__dummy_config_py/__init__.py Normal file
View file

@ -0,0 +1,22 @@
import os
import sys
from cdist.core.pytypes import *
class DummyConfig(PythonType):
def type_manifest(self):
print('dummy manifest stdout')
print('dummy manifest stderr\n', file=sys.stderr)
yield file_py('/root/dummy1.conf',
mode='0640',
owner='root',
group='root',
source='-').feed_stdin('dummy=1\n')
self_path = os.path.dirname(os.path.realpath(__file__))
conf_path = os.path.join(self_path, 'files', 'dummy.conf')
yield file_py('/root/dummy2.conf',
mode='0640',
owner='root',
group='root',
source=conf_path)

1
docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf Normal file
View file

@ -0,0 +1 @@
dummy=2

1
docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf Normal file
View file

@ -0,0 +1 @@
dummy=2

6
docs/dev/python-types/conf/type/__dummy_config_sh/manifest Normal file
View file

@ -0,0 +1,6 @@
printf 'dummy manifest stdout\n'
printf 'dummy manifest stderr\n' >&2
printf "dummy=1\\n" | __file /root/dummy1.conf --mode 0640 --owner root --group root --source -
__file /root/dummy2.conf --mode 0600 --owner root --group root --source "$__type/files/dummy.conf"

27
docs/dev/python-types/test.sh Executable file
View file

@ -0,0 +1,27 @@
#!/bin/sh
# Addapt to your env.
CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
export CDIST_PATH
TARGET_HOST=185.203.112.26
env | grep CDIST_PATH
for streams in ' ' '-S'
do
for x in sh py
do
printf "[%s] Removing old foobar* files\\n" "$x"
printf -- "----------------\\n"
ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*'
printf "[%s] Listing foobar* files\\n" "$x"
printf -- "----------------\\n"
ssh root@${TARGET_HOST} 'ls foobar* dummy*'
printf "[%s] Running cdist config, streams: %s\\n" "$x" "$streams"
printf -- "----------------\\n"
./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init -- ${TARGET_HOST}
printf "[%s] Listing foobar* files\\n" "$x"
printf -- "----------------\\n"
ssh root@${TARGET_HOST} 'ls foobar* dummy*'
./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init -- ${TARGET_HOST}
done
done

36
docs/dev/python-types/timeit.sh Executable file
View file

@ -0,0 +1,36 @@
#!/bin/sh
# Addapt to your env.
CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf"
export CDIST_PATH
TARGET_HOST=185.203.112.26
if [ $# -eq 0 ]
then
N=1
else
N=$1
fi
i=0
while [ "$i" -lt "$N" ]
do
if [ "$N" -ne 1 ]
then
printf "iteration %d\\n" "$i"
fi
printf "shinit clean state...\\n"
ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*;'
time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST}
printf "pyinit clean state...\\n"
ssh root@$${TARGET_HOST} 'rm foobar*; rm dummy*;'
time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST}
printf "shinit present state...\\n"
time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST}
printf "pyinit present state...\\n"
time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST}
i=$((i + 1))
done

86
docs/src/cdist-type.rst
View file

@ -522,3 +522,89 @@ How to include a type into upstream cdist
If you think your type may be useful for others, ensure it works with the If you think your type may be useful for others, ensure it works with the
current master branch of cdist and have a look at `cdist hacking <cdist-hacker.html>`_ on current master branch of cdist and have a look at `cdist hacking <cdist-hacker.html>`_ on
how to submit it. how to submit it.
Python types
------------
From version/branch **beta** cdist support python types, types that are written
in python language with cdist's core support. cdist detects such type if type is
detectable as a python package, i.e. if **__init__.py** file is present in type's
root directory. Upon that detection cdist will try to run such type as core python
type.
Note that this differs from plain cdist type where scripts are written in pure
python and have a proper shebang.
Core python types replace manifest and gencode scripts. Parameters, singleton,
nonparallel are still defined as for common types. Explorer code is also written
in shell, since this is the code that is directly executed at target host.
When writing python type you can extend **cdist.core.pytypes.PythonType** class.
You need to implement the following methods:
* **type_manifest**: implementation should yield **cdist.core.pytypes.<type-name>**
attribute function call result, or **yield from ()** if type does not use other types
* **type_gencode**: implementation should return a string consisting of lines
of shell code that will be executed at target host.
**cdist.core.pytypes.<type-name>** attributes correspond to detected python types.
**Note** that double underscore ('__') at the beginning of type name is removed.
Example:
.. code-block:: sh
import os
import sys
from cdist.core.pytypes import *
class DummyConfig(PythonType):
def type_manifest(self):
print('dummy manifest stdout')
print('dummy manifest stderr\n', file=sys.stderr)
yield file_py('/root/dummy1.conf',
mode='0640',
owner='root',
group='root',
source='-').feed_stdin('dummy=1\n')
self_path = os.path.dirname(os.path.realpath(__file__))
conf_path = os.path.join(self_path, 'files', 'dummy.conf')
yield file_py('/root/dummy2.conf',
mode='0640',
owner='root',
group='root',
source=conf_path)
**cdist.core.PythonType** class provides the following methods:
* **get_parameter**: get type parameter
* **get_explorer_file**: get path to file for specified explorer
* **get_explorer**: get value for specified explorer
* **run_local**: run specified command locally
* **run_remote**: run specified command remotely
* **transfer**: transfer specified source to the remote
* **die**: raise error
* **send_message**: send message
* **receive_message**: get message.
When running python type, cdist will save output streams to **gencode-py**,
stdout and stderr output files.
As a reference implementation you can take a look at **__file_py** type,
which is re-implementation of **__file** type.
Furthermore, under **docs/dev/python-types** there are sample cdist conf directory,
init manifests and scripts for running and measuring duration of samples.
There, under **conf/type/__dummy_config** you can find another example of
python type, which (unlike **__file_py** type) also uses new manifest implementation
that yields **cdist.core.pytypes.<type-name>** attribute function call results.
**NOTE** that python types implementation is under the beta, not directly controled by
the **-b/--beta** option. It is controled by the explicit usage of python types in
your config.
Also, this documenation is only an introduction, and not a complete guide to python
types. Currently, it is just a short introduction so one can start to write and use
python types.

2
scripts/cdist
View file

@ -60,7 +60,7 @@ def commandline():
if __name__ == "__main__": if __name__ == "__main__":
cdistpythonversion = '3.2' cdistpythonversion = '3.5'
if sys.version < cdistpythonversion: if sys.version < cdistpythonversion:
print('Python >= {} is required on the source host.'.format( print('Python >= {} is required on the source host.'.format(
cdistpythonversion), file=sys.stderr) cdistpythonversion), file=sys.stderr)