Compare commits

...

6 Commits

Author SHA1 Message Date
Darko Poljak a9067aa846 Implement python types 2019-05-13 10:27:49 +02:00
Dominique Roux 66db5acc32 Updated the man pages for the cdist trigger and preos 2019-05-09 19:36:43 +02:00
Darko Poljak eb78d9b034 Add missing configuration arg 2019-05-09 19:36:43 +02:00
Darko Poljak 0e92f5bb0a Update trigger to config 2019-05-09 19:36:43 +02:00
Darko Poljak a87a69e281 Log trigger server error 2019-05-09 19:36:43 +02:00
Darko Poljak 11974e5ed6 Implement preos and triggering 2019-05-09 19:36:43 +02:00
125 changed files with 11746 additions and 14 deletions

View File

@ -5,10 +5,12 @@ import logging
import collections
import functools
import cdist.configuration
import cdist.trigger
import cdist.preos
# set of beta sub-commands
BETA_COMMANDS = set(('install', 'inventory', ))
BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', ))
# set of beta arguments for sub-commands
BETA_ARGS = {
'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
@ -422,6 +424,9 @@ def get_parsers():
parser['inventory'].set_defaults(
func=cdist.inventory.Inventory.commandline)
# PreOs
parser['preos'] = parser['sub'].add_parser('preos', add_help=False)
# Shell
parser['shell'] = parser['sub'].add_parser(
'shell', parents=[parser['loglevel']])
@ -431,6 +436,28 @@ def get_parsers():
' should be POSIX compatible shell.'))
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
# Trigger
parser['trigger'] = parser['sub'].add_parser(
'trigger', parents=[parser['loglevel'],
parser['beta'],
parser['common'],
parser['config_main']])
parser['trigger'].add_argument(
'-D', '--directory', action='store', required=False,
help=('Where to create local files'))
parser['trigger'].add_argument(
'-H', '--http-port', action='store', default=3000, required=False,
help=('Create trigger listener via http on specified port'))
parser['trigger'].add_argument(
'--ipv6', default=False,
help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'),
action='store_true')
parser['trigger'].add_argument(
'-O', '--source', action='store', required=False,
help=('Which file to copy for creation'))
parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline)
for p in parser:
parser[p].epilog = EPILOG

View File

@ -0,0 +1,12 @@
#!/bin/sh
os=$(cat "$__global/explorer/os")
case "$os" in
devuan)
echo "update-rc.d cdist-preos-trigger defaults > /dev/null"
;;
*)
;;
esac

View File

@ -0,0 +1,45 @@
cdist-type__cdist_preos_trigger(7)
==================================
NAME
----
cdist-type__cdist_preos_trigger - configure cdist preos trigger
DESCRIPTION
-----------
Create cdist PreOS trigger by creating systemd unit file that will be started
at boot and will execute trigger command - connect to specified host and port.
REQUIRED PARAMETERS
-------------------
trigger-command
Command that will be executed as a PreOS cdist trigger.
OPTIONAL PARAMETERS
-------------------
None
EXAMPLES
--------
.. code-block:: sh
# Configure default curl trigger for host cdist.ungleich.ch at port 80.
__cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80'
AUTHORS
-------
Darko Poljak <darko.poljak--@--ungleich.ch>
COPYING
-------
Copyright \(C) 2016 Darko Poljak. You can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

View File

@ -0,0 +1,67 @@
#!/bin/sh
os="$(cat "$__global/explorer/os")"
trigger_command=$(cat "$__object/parameter/trigger-command")
case "$os" in
devuan)
__file /etc/init.d/cdist-preos-trigger --owner root \
--group root \
--mode 755 \
--source - << EOF
#!/bin/sh
# /etc/init.d/cdist-preos-trigger
### BEGIN INIT INFO
# Provides: cdist-preos-trigger
# Required-Start: \$all
# Required-Stop:
# Default-Start: 2 3 4 5 S
# Default-Stop: 0 1 6
# Short-Description: Execute cdist preos trigger command
# Description: Execute cdist preos trigger commnad.
### END INIT INFO
case "\$1" in
start)
echo "Starting cdist-preos-trigger command"
${trigger_command} &
;;
stop)
# no-op
;;
*)
echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}"
exit 1
;;
esac
exit 0
EOF
;;
*)
__file /etc/systemd/system/cdist-preos-trigger.service --owner root \
--group root \
--mode 644 \
--source - << EOF
[Unit]
Description=preos trigger
Wants=network-online.target
After=network.target network-online.target
[Service]
Type=simple
Restart=no
# Broken systemd
ExecStartPre=/bin/sleep 5
ExecStart=${trigger_command}
[Install]
WantedBy=multi-user.target
EOF
require="__file/etc/systemd/system/cdist-preos-trigger.service" \
__start_on_boot cdist-preos-trigger
;;
esac

View File

@ -0,0 +1 @@
trigger-command

View File

@ -0,0 +1,103 @@
import os
import re
import sys
from cdist.core import PythonType
class FileType(PythonType):
def get_attribute(self, stat_file, attribute, value_should):
if os.path.exists(stat_file):
if re.match('[0-9]', value_should):
index = 1
else:
index = 2
with open(stat_file, 'r') as f:
for line in f:
if re.match(attribute + ":", line):
fields = line.split()
return fields[index]
return None
def set_attribute(self, attribute, value_should, destination):
cmd = {
'group': 'chgrp',
'owner': 'chown',
'mode': 'chmod',
}
self.send_message("{} '{}'".format(cmd[attribute], value_should))
return "{} '{}' '{}'".format(cmd[attribute], value_should, destination)
def type_manifest(self):
yield from ()
def type_gencode(self):
typeis = self.get_explorer('type')
state_should = self.get_parameter('state')
if state_should == 'exists' and typeis == 'file':
return
source = self.get_parameter('source')
if source == '-':
source = self.stdin_path
destination = '/' + self.object_id
if state_should == 'pre-exists':
if source is not None:
self.die('--source cannot be used with --state pre-exists')
if typeis == 'file':
return None
else:
self.die('File {} does not exist'.format(destination))
create_file = False
upload_file = False
set_attributes = False
code = []
if state_should == 'present' or state_should == 'exists':
if source is None:
remote_stat = self.get_explorer('stat')
if not remote_stat:
create_file = True
else:
if os.path.exists(source):
if typeis == 'file':
local_cksum = self.run_local(['cksum', source, ])
local_cksum = local_cksum.split()[0]
remote_cksum = self.get_explorer('cksum')
remote_cksum = remote_cksum.split()[0]
upload_file = local_cksum != remote_cksum
else:
upload_file = True
else:
self.die('Source {} does not exist'.format(source))
if create_file or upload_file:
set_attributes = True
tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination)
destination_upload = self.run_remote(
["mktemp", tempfile_template, ])
if upload_file:
self.transfer(source, destination_upload)
code.append('rm -rf {}'.format(destination))
code.append('mv {} {}'.format(destination_upload, destination))
if state_should in ('present', 'exists', 'pre-exists', ):
for attribute in ('group', 'owner', 'mode', ):
if attribute in self.parameters:
value_should = self.get_parameter(attribute)
if attribute == 'mode':
value_should = re.sub('^0', '', value_should)
stat_file = self.get_explorer_file('stat')
value_is = self.get_attribute(stat_file, attribute,
value_should)
if set_attributes or value_should != value_is:
code.append(self.set_attribute(attribute,
value_should,
destination))
elif state_should == 'absent':
if typeis == 'file':
code.append('rm -f {}'.format(destination))
self.send_message('remove')
else:
self.die('Unknown state {}'.format(state_should))
return "\n".join(code)

View File

@ -0,0 +1,34 @@
#!/bin/sh
#
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# Retrieve the md5sum of a file to be created, if it is already existing.
#
destination="/$__object_id"
if [ -e "$destination" ]; then
if [ -f "$destination" ]; then
cksum < "$destination"
else
echo "NO REGULAR FILE"
fi
else
echo "NO FILE FOUND, NO CHECKSUM CALCULATED."
fi

View File

@ -0,0 +1,56 @@
#!/bin/sh
#
# 2013 Steven Armstrong (steven-cdist armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
destination="/$__object_id"
# nothing to work with, nothing we could do
[ -e "$destination" ] || exit 0
os=$("$__explorer/os")
case "$os" in
"freebsd"|"netbsd"|"openbsd")
# FIXME: should be something like this based on man page, but can not test
stat -f "type: %ST
owner: %Du %Su
group: %Dg %Sg
mode: %Op %Sp
size: %Dz
links: %Dl
" "$destination"
;;
"macosx")
stat -f "type: %HT
owner: %Du %Su
group: %Dg %Sg
mode: %Lp %Sp
size: %Dz
links: %Dl
" "$destination"
;;
*)
stat --printf="type: %F
owner: %u %U
group: %g %G
mode: %a %A
size: %s
links: %h
" "$destination"
;;
esac

View File

@ -0,0 +1,33 @@
#!/bin/sh
#
# 2013 Steven Armstrong (steven-cdist armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
destination="/$__object_id"
if [ ! -e "$destination" ]; then
echo none
elif [ -h "$destination" ]; then
echo symlink
elif [ -f "$destination" ]; then
echo file
elif [ -d "$destination" ]; then
echo directory
else
echo unknown
fi

View File

@ -0,0 +1 @@
present

View File

@ -0,0 +1,5 @@
state
group
mode
owner
source

View File

@ -90,13 +90,15 @@ class Config(object):
shutil.rmtree(path)
def __init__(self, local, remote, dry_run=False, jobs=None,
cleanup_cmds=None, remove_remote_files_dirs=False):
cleanup_cmds=None, remove_remote_files_dirs=False,
timestamp=False):
self.local = local
self.remote = remote
self._open_logger()
self.dry_run = dry_run
self.jobs = jobs
self.timestamp = timestamp
if cleanup_cmds:
self.cleanup_cmds = cleanup_cmds
else:
@ -424,7 +426,8 @@ class Config(object):
cleanup_cmds.append(cleanup_cmd)
c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs,
cleanup_cmds=cleanup_cmds,
remove_remote_files_dirs=remove_remote_files_dirs)
remove_remote_files_dirs=remove_remote_files_dirs,
timestamp=args.timestamp)
c.run()
cls._remove_paths()
@ -755,6 +758,21 @@ class Config(object):
("The requirements of the following objects could not be "
"resolved:\n%s") % ("\n".join(info_string)))
def _timeit(self, func, msg_prefix):
def wrapper_func(*args, **kwargs):
loglevel = self.log.getEffectiveLevel()
if loglevel >= logging.VERBOSE and self.timestamp:
start_time = time.time()
rv = func(*args, **kwargs)
end_time = time.time()
duration = end_time - start_time
self.log.verbose("%s duration: %.6f seconds",
msg_prefix, duration)
else:
rv = func(*args, **kwargs)
return rv
return wrapper_func
def object_prepare(self, cdist_object, transfer_type_explorers=True):
"""Prepare object: Run type explorer + manifest"""
self.log.verbose("Preparing object {}".format(cdist_object.name))
@ -762,11 +780,28 @@ class Config(object):
"Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
try:
self.manifest.run_type_manifest(cdist_object)
self.log.verbose("Preparing object {}".format(cdist_object.name))
self.log.verbose(
"Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object,
transfer_type_explorers)
if self.is_py_type(cdist_object):
self._timeit(self.manifest.run_py_type_manifest,
"Python type manifest for {}".format(
cdist_object.name))(cdist_object)
else:
self._timeit(self.manifest.run_type_manifest,
"Type manifest for {}".format(
cdist_object.name))(cdist_object)
cdist_object.state = core.CdistObject.STATE_PREPARED
except cdist.Error as e:
raise cdist.CdistObjectError(cdist_object, e)
def is_py_type(self, cdist_object):
cdist_type = cdist_object.cdist_type
init_path = os.path.join(cdist_type.absolute_path, '__init__.py')
return os.path.exists(init_path)
def object_run(self, cdist_object):
"""Run gencode and code for an object"""
try:
@ -777,9 +812,20 @@ class Config(object):
# Generate
self.log.debug("Generating code for %s" % (cdist_object.name))
cdist_object.code_local = self.code.run_gencode_local(cdist_object)
cdist_object.code_remote = self.code.run_gencode_remote(
cdist_object)
if self.is_py_type(cdist_object):
cdist_object.code_local = ''
cdist_object.code_remote = self._timeit(self.code.run_py,
"Python type generate code for {}".format(
cdist_object.name))(cdist_object)
else:
cdist_object.code_local = self._timeit(
self.code.run_gencode_local,
"Type generate code local for {}".format(
cdist_object.name))(cdist_object)
cdist_object.code_remote = self._timeit(
self.code.run_gencode_remote,
"Type generate code remote for {}".format(
cdist_object.name))(cdist_object)
if cdist_object.code_local or cdist_object.code_remote:
cdist_object.changed = True
@ -790,12 +836,16 @@ class Config(object):
if cdist_object.code_local:
self.log.trace("Executing local code for %s"
% (cdist_object.name))
self.code.run_code_local(cdist_object)
self._timeit(self.code.run_code_local,
"Type run code local for {}".format(
cdist_object.name))(cdist_object)
if cdist_object.code_remote:
self.log.trace("Executing remote code for %s"
% (cdist_object.name))
self.code.transfer_code_remote(cdist_object)
self.code.run_code_remote(cdist_object)
self._timeit(self.code.run_code_remote,
"Type run code remote for {}".format(
cdist_object.name))(cdist_object)
# Mark this object as done
self.log.trace("Finishing run of " + cdist_object.name)

View File

@ -21,6 +21,7 @@
#
from cdist.core.cdist_type import CdistType
from cdist.core.python_type import PythonType, ManifestEntry
from cdist.core.cdist_type import InvalidTypeError
from cdist.core.cdist_object import CdistObject
from cdist.core.cdist_object import IllegalObjectIdError

View File

@ -22,6 +22,10 @@
#
import os
import importlib.util
import inspect
import cdist
from cdist.core import PythonType
from . import util
@ -113,6 +117,44 @@ class Code(object):
local.log),
}
def run_py(self, cdist_object):
cdist_type = cdist_object.cdist_type
module_name = cdist_type.name
file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
if os.path.isfile(file_path):
spec = importlib.util.spec_from_file_location(module_name,
file_path)
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
classes = inspect.getmembers(m, inspect.isclass)
type_class = None
for _, cl in classes:
if cl != PythonType and issubclass(cl, PythonType):
if type_class:
raise cdist.Error("Only one python type class is "
"supported, but at least two "
"found: {}".format((type_class,
cl, )))
else:
type_class = cl
env = os.environ.copy()
env.update(self.env)
message_prefix = cdist_object.name
type_obj = type_class(env=env, cdist_object=cdist_object,
local=self.local, remote=self.remote,
message_prefix=message_prefix)
if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run):
if self.local.save_output_streams:
which = 'gencode-py'
stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'a+') as stderr, \
open(stdout_path, 'a+') as stdout:
return type_obj.run(stdout=stdout, stderr=stderr)
else:
return type_obj.run()
def _run_gencode(self, cdist_object, which):
cdist_type = cdist_object.cdist_type
script = os.path.join(self.local.type_path,

View File

@ -22,9 +22,13 @@
import logging
import os
import importlib.util
import inspect
import cdist
import cdist.emulator
from . import util
from cdist.core import PythonType, ManifestEntry
'''
common:
@ -209,3 +213,73 @@ class Manifest(object):
type_manifest,
env=self.env_type_manifest(cdist_object),
message_prefix=message_prefix)
def env_py_type_manifest(self, cdist_object):
env = os.environ.copy()
env.update(self.env)
env.update({
'__cdist_object_marker': self.local.object_marker_name,
'__cdist_manifest': cdist_object.cdist_type,
'__manifest': self.local.manifest_path,
'__object': cdist_object.absolute_path,
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
'__type': cdist_object.cdist_type.absolute_path,
})
return env
def run_py_type_manifest(self, cdist_object):
cdist_type = cdist_object.cdist_type
module_name = cdist_type.name
file_path = os.path.join(cdist_type.absolute_path, '__init__.py')
message_prefix = cdist_object.name
if os.path.isfile(file_path):
self.log.verbose("Running python type manifest for object %s",
cdist_object.name)
spec = importlib.util.spec_from_file_location(module_name,
file_path)
m = importlib.util.module_from_spec(spec)
spec.loader.exec_module(m)
classes = inspect.getmembers(m, inspect.isclass)
type_class = None
for _, cl in classes:
if cl != PythonType and issubclass(cl, PythonType):
if type_class:
raise cdist.Error("Only one python type class is "
"supported, but at least two "
"found: {}".format((type_class,
cl, )))
else:
type_class = cl
env = self.env_py_type_manifest(cdist_object)
type_obj = type_class(env=env, cdist_object=cdist_object,
local=self.local, remote=None,
message_prefix=message_prefix)
if self.local.save_output_streams:
which = 'manifest'
stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'a+') as stderr, \
open(stdout_path, 'a+') as stdout:
self._process_py_type_manifest_entries(
type_obj, env, stdout=stdout, stderr=stderr)
else:
self._process_py_type_manifest_entries(type_obj, env)
def _process_py_type_manifest_entries(self, type_obj, env, stdout=None,
stderr=None):
if hasattr(type_obj, 'manifest') and \
inspect.ismethod(type_obj.manifest):
for entry in type_obj.manifest(stdout=stdout, stderr=stderr):
if not isinstance(entry, ManifestEntry):
raise TypeError("Manifest entry must be of "
"type ManifestEntry")
kwargs = {
'argv': entry.cmd_line(),
'env': env,
}
if entry.stdin:
kwargs['stdin'] = entry.stdin
emulator = cdist.emulator.Emulator(**kwargs)
emulator.run()

160
cdist/core/python_type.py Normal file
View File

@ -0,0 +1,160 @@
import logging
import os
import io
import sys
import re
from cdist import message, Error
class PythonType:
def __init__(self, env, cdist_object, local, remote, message_prefix=None):
self.env = env
self.cdist_object = cdist_object
self.object_id = cdist_object.object_id
self.object_name = cdist_object.name
self.cdist_type = cdist_object.cdist_type
self.local = local
self.remote = remote
self.object_path = cdist_object.absolute_path
self.type_path = cdist_object.cdist_type.absolute_path
self.explorer_path = os.path.join(self.object_path, 'explorer')
self.parameters = cdist_object.parameters
self.stdin_path = os.path.join(self.object_path, 'stdin')
self.log = logging.getLogger(
self.local.target_host[0] + ':' + self.object_name)
self.message_prefix = message_prefix
self.message = None
def get_parameter(self, name):
return self.parameters.get(name)
def get_explorer_file(self, name):
path = os.path.join(self.explorer_path, name)
return path
def get_explorer(self, name):
path = self.get_explorer_file(name)
with open(path, 'r') as f:
value = f.read()
if value:
value = value.strip()
return value
def run_local(self, command, env=None):
rv = self.local.run(command, env=env, return_output=True)
if rv:
rv = rv.rstrip('\n')
return rv
def run_remote(self, command, env=None):
rv = self.remote.run(command, env=env, return_output=True)
if rv:
rv = rv.rstrip('\n')
return rv
def transfer(self, source, destination):
self.remote.transfer(source, destination)
def die(self, msg):
raise Error("{}: {}".format(self.cdist_object, msg))
def type_manifest(self):
pass
def type_gencode(self):
pass
def manifest(self, stdout=None, stderr=None):
try:
if self.message_prefix:
self.message = message.Message(self.message_prefix,
self.local.messages_path)
self.env.update(self.message.env)
if stdout is not None:
stdout_save = sys.stdout
sys.stdout = stdout
if stderr is not None:
stderr_save = sys.stderr
sys.stderr = stderr
yield from self.type_manifest()
finally:
if self.message:
self.message.merge_messages()
if stdout is not None:
sys.stdout = stdout_save
if stderr is not None:
sys.stderr = stderr_save
def run(self, stdout=None, stderr=None):
try:
if self.message_prefix:
self.message = message.Message(self.message_prefix,
self.local.messages_path)
if stdout is not None:
stdout_save = sys.stdout
sys.stdout = stdout
if stderr is not None:
stderr_save = sys.stderr
sys.stderr = stderr
return self.type_gencode()
finally:
if self.message:
self.message.merge_messages()
if stdout is not None:
sys.stdout = stdout_save
if stderr is not None:
sys.stderr = stderr_save
def send_message(self, msg):
if self.message:
with open(self.message.messages_out, 'a') as f:
print(msg, file=f)
def receive_message(self, pattern):
if self.message:
with open(self.message.messages_in, 'r') as f:
for line in f:
match = re.search(pattern, line)
if match:
return match
return None
class ManifestEntry:
def __init__(self, name, stdin=None, parameters=None):
self.name = name
if parameters is None:
self.parameters = {}
else:
self.parameters = parameters
self.set_stdin(stdin)
def set_stdin(self, value):
# If file-like object then read its value.
if value is not None and isinstance(value, io.IOBase):
value = value.read()
# Convert to bytes file-like object.
if value is None:
self.stdin = None
elif isinstance(value, str):
self.stdin = io.BytesIO(value.encode('utf-8'))
elif isinstance(value, bytes) or isinstance(value, bytearray):
self.stdin = io.BytesIO(value)
else:
raise TypeError("value must be str, bytes, bytearray, file-like "
"object or None")
def cmd_line(self):
argv = [self.name, ]
for param in self.parameters:
argv.append(param)
val = self.parameters[param]
if val:
argv.append(val)
return argv
def __repr__(self):
return '<ManifestEntry name={}, parameters={}, stdin={}>'.format(
self.name, self.parameters, self.stdin)

101
cdist/preos.py Normal file
View File

@ -0,0 +1,101 @@
import os
import os.path
import sys
import inspect
import argparse
import cdist
import logging
_PREOS_CALL = "commandline"
_PREOS_NAME = "_preos_name"
_PREOS_MARKER = "_cdist_preos"
_PLUGINS_DIR = "preos"
_PLUGINS_PATH = [os.path.join(os.path.dirname(__file__), _PLUGINS_DIR), ]
cdist_home = cdist.home_dir()
if cdist_home:
cdist_home_preos = os.path.join(cdist_home, "preos")
if os.path.isdir(cdist_home_preos):
_PLUGINS_PATH.append(cdist_home_preos)
sys.path.extend(_PLUGINS_PATH)
log = logging.getLogger("PreOS")
def preos_plugin(obj):
"""It is preos if _PREOS_MARKER is True and has _PREOS_CALL."""
if hasattr(obj, _PREOS_MARKER):
is_preos = getattr(obj, _PREOS_MARKER)
else:
is_preos = False
if is_preos and hasattr(obj, _PREOS_CALL):
yield obj
def scan_preos_dir_plugins(dir):
for fname in os.listdir(dir):
if os.path.isfile(os.path.join(dir, fname)):
fname = os.path.splitext(fname)[0]
module_name = fname
try:
module = __import__(module_name)
yield from preos_plugin(module)
clsmembers = inspect.getmembers(module, inspect.isclass)
for cm in clsmembers:
c = cm[1]
yield from preos_plugin(c)
except ImportError as e:
log.warning("Cannot import '{}': {}".format(module_name, e))
def find_preos_plugins():
for dir in _PLUGINS_PATH:
yield from scan_preos_dir_plugins(dir)
def find_preoses():
preoses = {}
for preos in find_preos_plugins():
if hasattr(preos, _PREOS_NAME):
preos_name = getattr(preos, _PREOS_NAME)
else:
preos_name = preos.__name__.lower()
preoses[preos_name] = preos
return preoses
def check_root():
if os.geteuid() != 0:
raise cdist.Error("Must be run with root privileges")
class PreOS(object):
preoses = None
@classmethod
def commandline(cls, argv):
if not cls.preoses:
cls.preoses = find_preoses()
parser = argparse.ArgumentParser(
description="Create PreOS", prog="cdist preos")
parser.add_argument('preos', help='PreOS to create, one of: {}'.format(
set(cls.preoses)))
args = parser.parse_args(argv[1:2])
preos_name = args.preos
if preos_name in cls.preoses:
preos = cls.preoses[preos_name]
func = getattr(preos, _PREOS_CALL)
if inspect.ismodule(preos):
func_args = [preos, argv[2:], ]
else:
func_args = [argv[2:], ]
log.info("Running preos : {}".format(preos_name))
func(*func_args)
else:
log.error("Unknown preos: {}, available preoses: {}".format(
preos_name, set(cls.preoses.keys())))

View File

@ -0,0 +1 @@
from debootstrap.debootstrap import Debian, Ubuntu, Devuan

View File

@ -0,0 +1,246 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at ungleich.ch)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import cdist
import cdist.config
import cdist.core
import cdist.preos
import argparse
import cdist.argparse
import logging
import os
import subprocess
class Debian(object):
_preos_name = 'debian'
_cdist_preos = True
_files_dir = os.path.join(os.path.dirname(__file__), "files")
@classmethod
def default_args(cls):
default_remote_exec = os.path.join(cls._files_dir, "remote-exec.sh")
default_remote_copy = os.path.join(cls._files_dir, "remote-copy.sh")
default_init_manifest = os.path.join(
cls._files_dir, "init-manifest-{}".format(cls._preos_name))
defargs = argparse.Namespace()
defargs.arch = 'amd64'
defargs.bootstrap = False
defargs.configure = False
defargs.cdist_params = '-v'
defargs.rm_bootstrap_dir = False
defargs.suite = 'stable'
defargs.remote_exec = default_remote_exec
defargs.remote_copy = default_remote_copy
defargs.manifest = default_init_manifest
return defargs
@classmethod
def get_parser(cls):
defargs = cls.default_args()
cdist_parser = cdist.argparse.get_parsers()
parser = argparse.ArgumentParser(
prog='cdist preos {}'.format(cls._preos_name),
parents=[cdist_parser['loglevel'], cdist_parser['beta']])
parser.add_argument('target_dir', nargs=1,
help=("target directory where PreOS will be "
"bootstrapped"))
parser.add_argument(
'-a', '--arch',
help="target debootstrap architecture, by default '{}'".format(
defargs.arch), dest='arch', default=defargs.arch)
parser.add_argument(
'-B', '--bootstrap',
help='do bootstrap step',
dest='bootstrap', action='store_true', default=defargs.bootstrap)
parser.add_argument(
'-C', '--configure',
help='do configure step',
dest='configure', action='store_true', default=defargs.configure)
parser.add_argument(
'-c', '--cdist-params',
help=("parameters that will be passed to cdist config, by default"
" '{}' is used".format(defargs.cdist_params)),
dest='cdist_params', default=defargs.cdist_params)
parser.add_argument(
'-D', '--drive-boot',
help='create bootable PreOS on specified drive',
dest='drive')
parser.add_argument(
'-e', '--remote-exec',
help=("remote exec that cdist config will use, by default "
"internal script is used"),
dest='remote_exec', default=defargs.remote_exec)
parser.add_argument(
'-i', '--init-manifest',
help=("init manifest that cdist config will use, by default "
"internal init manifest is used"),
dest='manifest', default=defargs.manifest)
parser.add_argument(
'-k', '--keyfile', action="append",
help=("ssh key files that will be added to cdist config; "
"'__ssh_authorized_keys root ...' type is appended to "
"initial manifest"),
dest='keyfile')
parser.add_argument(
'-m', '--mirror',
help='use specified mirror for debootstrap',
dest='mirror')
parser.add_argument(
'-P', '--root-password',
help='Set specified password for root, generated by default',
dest='root_password')
parser.add_argument('-p', '--pxe-boot-dir', help='PXE boot directory',
dest='pxe_boot_dir')
parser.add_argument(
'-r', '--rm-bootstrap-dir',
help='remove target directory after finishing',
dest='rm_bootstrap_dir', action='store_true',
default=defargs.rm_bootstrap_dir)
parser.add_argument(
'-S', '--script',
help='use specified script for debootstrap',
dest='script')
parser.add_argument('-s', '--suite',
help="suite used for debootstrap, "
"by default '{}'".format(defargs.suite),
dest='suite', default=defargs.suite)
parser.add_argument(
'-t', '--trigger-command',
help=("trigger command that will be added to cdist config; "
"'__cdist_preos_trigger http ...' type is appended to "
"initial manifest"),
dest='trigger_command')
parser.add_argument(
'-y', '--remote-copy',
help=("remote copy that cdist config will use, by default "
"internal script is used"),
dest='remote_copy', default=defargs.remote_copy)
parser.epilog = cdist.argparse.EPILOG
return parser
@classmethod
def update_env(cls, env):
pass
@classmethod
def commandline(cls, argv):
log = logging.getLogger(cls.__name__)
parser = cls.get_parser()
cdist.argparse.add_beta_command(cls._preos_name)
args = parser.parse_args(argv)
if args.script and not args.mirror:
raise cdist.Error("script option cannot be used without "
"mirror option")
args.command = cls._preos_name
cdist.argparse.check_beta(vars(args))
cdist.preos.check_root()
args.target_dir = os.path.realpath(args.target_dir[0])
args.os = cls._preos_name
args.remote_exec = os.path.realpath(args.remote_exec)
args.remote_copy = os.path.realpath(args.remote_copy)
args.manifest = os.path.realpath(args.manifest)
if args.keyfile:
new_keyfile = [os.path.realpath(x) for x in args.keyfile]
args.keyfile = new_keyfile
if args.pxe_boot_dir:
args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir)
cdist.argparse.handle_loglevel(args)
log.debug("preos: {}, args: {}".format(cls._preos_name, args))
try:
env = vars(args)
new_env = {}
for key in env:
if key == 'verbose':
if env[key] >= 3:
new_env['debug'] = "yes"
elif env[key] == 2:
new_env['verbose'] = "yes"
elif not env[key]:
new_env[key] = ''
elif isinstance(env[key], bool) and env[key]:
new_env[key] = "yes"
elif isinstance(env[key], list):
val = env[key]
new_env[key + "_cnt"] = str(len(val))
for i, v in enumerate(val):
new_env[key + "_" + str(i)] = v
else:
new_env[key] = str(env[key])
env = new_env
env.update(os.environ)
cls.update_env(env)
log.debug("preos: {} env: {}".format(cls._preos_name, env))
cmd = os.path.join(cls._files_dir, "code")
info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
cls._preos_name, args.suite, args.arch), ]
if args.mirror:
info_msg.append("mirror: {}".format(args.mirror))
if args.script:
info_msg.append("script: {}".format(args.script))
if args.bootstrap:
info_msg.append("bootstrapping")
if args.configure:
info_msg.append("configuring")
if args.pxe_boot_dir:
info_msg.append("creating PXE")
if args.drive:
info_msg.append("creating bootable drive")
log.info(info_msg)
log.debug("cmd={}".format(cmd))
subprocess.check_call(cmd, env=env, shell=True)
except subprocess.CalledProcessError as e:
log.error("preos {} failed: {}".format(cls._preos_name, e))
class Ubuntu(Debian):
_preos_name = "ubuntu"
@classmethod
def default_args(cls):
defargs = super().default_args()
defargs.suite = 'xenial'
return defargs
class Devuan(Debian):
_preos_name = "devuan"
@classmethod
def default_args(cls):
defargs = super().default_args()
defargs.suite = 'jessie'
return defargs
@classmethod
def update_env(cls, env):
env['DEBOOTSTRAP_DIR'] = os.path.join(cls._files_dir,
'devuan-debootstrap')

View File

@ -0,0 +1,281 @@
#!/bin/sh
##
## 2016 Darko Poljak (darko.poljak at ungleich.ch)
##
## This file is part of cdist.
##
## cdist is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## cdist is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with cdist. If not, see <http://www.gnu.org/licenses/>.
set -e
if [ "${debug}" ]
then
set -x
cdist_params="${cdist_params} -d"
fi
bootstrap_dir="${target_dir}"
case "${os}" in
ubuntu|debian|devuan)
# nothing, those are valid values
;;
*)
echo "ERROR: invalid os value: ${os}" >&2
exit 1
;;
esac
check_bootstrap_dir() {
if [ ! -e "$1" ]
then
echo "ERROR: bootstrap directory $1 does not exist" >&2
exit 1
fi
}
# bootstrap
if [ "${bootstrap}" ]
then
if [ "${DEBOOTSTRAP_DIR}" ]
then
debootstrap_cmd="${DEBOOTSTRAP_DIR}/debootstrap"
else
command -v debootstrap 2>&1 > /dev/null || {
echo "ERROR: debootstrap not found" >&2
exit 1
}
debootstrap_cmd="debootstrap"
fi
# If PreOS on drive then do not check for directory emptiness.
# Partition can at least contain 'lost+found' directory.
if [ ! "${drive}" ]
then
if [ -e "${bootstrap_dir}" ]
then
dir_content=$(ls -A "${bootstrap_dir}" | wc -l)
else
dir_content=0
fi
if [ "${dir_content}" -ne 0 ]
then
echo "ERROR: "${bootstrap_dir}" not empty " >&2
exit 1
fi
fi
if [ "${verbose}" -o "${debug}" ]
then
echo "bootstrapping..."
fi
mkdir -p "${bootstrap_dir}"
"${debootstrap_cmd}" --include=openssh-server --arch=${arch} ${suite} ${bootstrap_dir} \
${mirror} ${script}
if [ "${verbose}" -o "${debug}" ]
then
echo "bootstrap finished"
fi
fi
chroot_mount() {
mount -t proc none "${bootstrap_dir}/proc" || true
mount -t sysfs none "${bootstrap_dir}/sys" || true
mount -o bind /dev "${bootstrap_dir}/dev" || true
mount -t devpts none "${bootstrap_dir}/dev/pts" || true