Adapt; update docs and code style

This commit is contained in:
Darko Poljak 2020-06-09 12:47:50 +02:00
parent 57e352cd1e
commit 59b98091d7
8 changed files with 58 additions and 49 deletions

View file

@ -26,6 +26,7 @@ import hashlib
import cdist.log import cdist.log
import cdist.version import cdist.version
VERSION = cdist.version.VERSION VERSION = cdist.version.VERSION
BANNER = """ BANNER = """
@ -48,6 +49,9 @@ REMOTE_EXEC = "ssh -o User=root"
REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
MIN_SUPPORTED_PYTHON_VERSION = '3.5'
class Error(Exception): class Error(Exception):
"""Base exception class for this project""" """Base exception class for this project"""
pass pass

View file

@ -206,6 +206,13 @@ def get_parsers():
'supported. Without argument CPU count is used by default. '), 'supported. Without argument CPU count is used by default. '),
action='store', dest='jobs', action='store', dest='jobs',
const=multiprocessing.cpu_count()) const=multiprocessing.cpu_count())
parser['config_main'].add_argument(
'--log-server',
action='store_true',
help=('Start a log server for sub processes to use. '
'This is mainly useful when running cdist nested '
'from a code-local script. Log server is alwasy '
'implicitly started for \'install\' command.'))
parser['config_main'].add_argument( parser['config_main'].add_argument(
'-n', '--dry-run', '-n', '--dry-run',
help='Do not execute code.', action='store_true') help='Do not execute code.', action='store_true')
@ -250,12 +257,6 @@ def get_parsers():
'-S', '--disable-saving-output-streams', '-S', '--disable-saving-output-streams',
help='Disable saving output streams.', help='Disable saving output streams.',
action='store_false', dest='save_output_streams', default=True) action='store_false', dest='save_output_streams', default=True)
parser['config_main'].add_argument(
'--log-server',
action='store_true',
help=('Start a log server for sub processes to use.'
'This is mainly usefull when running cdist nested'
'from a code-local script.'))
# Config # Config
parser['config_args'] = argparse.ArgumentParser(add_help=False) parser['config_args'] = argparse.ArgumentParser(add_help=False)

View file

@ -29,18 +29,20 @@ import time
import itertools import itertools
import tempfile import tempfile
import multiprocessing import multiprocessing
from cdist.mputil import mp_pool_run, mp_sig_handler
import atexit import atexit
import shutil import shutil
import socket import socket
from cdist.mputil import mp_pool_run, mp_sig_handler
from cdist import core, inventory
from cdist.util.remoteutil import inspect_ssh_mux_opts
import cdist import cdist
import cdist.hostsource import cdist.hostsource
import cdist.exec.local import cdist.exec.local
import cdist.exec.remote import cdist.exec.remote
import cdist.util.ipaddr as ipaddr import cdist.util.ipaddr as ipaddr
import cdist.configuration import cdist.configuration
from cdist import core, inventory
from cdist.util.remoteutil import inspect_ssh_mux_opts
def graph_check_cycle(graph): def graph_check_cycle(graph):
@ -195,7 +197,6 @@ class Config(object):
@classmethod @classmethod
def commandline(cls, args): def commandline(cls, args):
"""Configure remote system""" """Configure remote system"""
if (args.parallel and args.parallel != 1) or args.jobs: if (args.parallel and args.parallel != 1) or args.jobs:
if args.timestamp: if args.timestamp:
cdist.log.setupTimestampingParallelLogging() cdist.log.setupTimestampingParallelLogging()
@ -382,20 +383,16 @@ class Config(object):
If operating in parallel then return tuple (host, True|False, ) If operating in parallel then return tuple (host, True|False, )
so that main process knows for which host function was successful. so that main process knows for which host function was successful.
""" """
log = logging.getLogger(host) log = logging.getLogger(host)
if args.log_server:
# Start a log server so that nested `cdist config` runs have a place
# to send their logs to.
log_server_socket_dir = tempfile.mkdtemp()
log_server_socket = os.path.join(log_server_socket_dir, 'log-server')
cls._register_path_for_removal(log_server_socket_dir)
log.debug('Starting logging server on: %s', log_server_socket)
os.environ['__cdist_log_server_socket_to_export'] = log_server_socket
cdist.log.setupLogServer(log_server_socket)
try: try:
if args.log_server:
# Start a log server so that nested `cdist config` runs
# have a place to send their logs to.
log_server_socket_dir = tempfile.mkdtemp()
cls._register_path_for_removal(log_server_socket_dir)
cdist.log.setupLogServer(log_server_socket_dir, log)
remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
args) args)
log.debug("remote_exec for host \"{}\": {}".format( log.debug("remote_exec for host \"{}\": {}".format(

View file

@ -116,9 +116,9 @@ class Code(object):
if dry_run: if dry_run:
self.env['__cdist_dry_run'] = '1' self.env['__cdist_dry_run'] = '1'
if '__cdist_log_server_socket_to_export' in os.environ: if '__cdist_log_server_socket_export' in os.environ:
self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export'] self.env['__cdist_log_server_socket'] = os.environ[
'__cdist_log_server_socket_export']
def _run_gencode(self, cdist_object, which): def _run_gencode(self, cdist_object, which):
cdist_type = cdist_object.cdist_type cdist_type = cdist_object.cdist_type

View file

@ -20,12 +20,9 @@
# #
# #
import os
import logging
import tempfile
import cdist.config import cdist.config
import cdist.core import cdist.core
import cdist.log
class Install(cdist.config.Config): class Install(cdist.config.Config):
@ -36,8 +33,9 @@ class Install(cdist.config.Config):
# Always start log server during cdist install so that nested # Always start log server during cdist install so that nested
# `cdist config` runs have a place to send their logs to. # `cdist config` runs have a place to send their logs to.
args.log_server = True args.log_server = True
super().onehost(host, host_tags, host_base_path, host_dir_name, args, super().onehost(host, host_tags, host_base_path, host_dir_name, args,
parallel, configuration, remove_remote_files_dirs=False) parallel, configuration, remove_remote_files_dirs)
def object_list(self): def object_list(self):
"""Short name for object list retrieval. """Short name for object list retrieval.

View file

@ -21,17 +21,16 @@
# #
# #
import asyncio
import contextlib
import datetime import datetime
import logging import logging
import logging.handlers import logging.handlers
import sys
import os import os
import asyncio
import contextlib
import pickle import pickle
import struct import struct
import sys
import threading import threading
import time
# Define additional cdist logging levels. # Define additional cdist logging levels.
@ -100,7 +99,8 @@ class DefaultLog(logging.Logger):
if '__cdist_log_server_socket' in os.environ: if '__cdist_log_server_socket' in os.environ:
log_server_socket = os.environ['__cdist_log_server_socket'] log_server_socket = os.environ['__cdist_log_server_socket']
socket_handler = logging.handlers.SocketHandler(log_server_socket, None) socket_handler = logging.handlers.SocketHandler(log_server_socket,
None)
self.addHandler(socket_handler) self.addHandler(socket_handler)
else: else:
formatter = CdistFormatter(self.FORMAT) formatter = CdistFormatter(self.FORMAT)
@ -184,13 +184,17 @@ async def handle_log_client(reader, writer):
def run_log_server(server_address): def run_log_server(server_address):
# Get a new loop inside the current thread to run the log server. # Get a new loop inside the current thread to run the log server.
loop = asyncio.new_event_loop() loop = asyncio.new_event_loop()
loop.create_task(asyncio.start_unix_server(handle_log_client, server_address)) loop.create_task(asyncio.start_unix_server(handle_log_client,
server_address))
loop.run_forever() loop.run_forever()
def setupLogServer(log_server_socket): def setupLogServer(socket_dir, log=logging.getLogger(__name__)):
"""Run a asyncio based unix socket log server in a background thread. """Run a asyncio based unix socket log server in a background thread.
""" """
log_server_socket = os.path.join(socket_dir, 'log-server')
log.debug('Starting logging server on: %s', log_server_socket)
os.environ['__cdist_log_server_socket_export'] = log_server_socket
with contextlib.suppress(FileNotFoundError): with contextlib.suppress(FileNotFoundError):
os.remove(log_server_socket) os.remove(log_server_socket)
t = threading.Thread(target=run_log_server, args=(log_server_socket,)) t = threading.Thread(target=run_log_server, args=(log_server_socket,))

View file

@ -17,20 +17,20 @@ SYNOPSIS
cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
[-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server]
[-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
[-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A]
[-p [HOST_MAX]] [-s] [-t] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[host [host ...]] [host [host ...]]
cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
[-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server]
[-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
[-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A]
[-p [HOST_MAX]] [-s] [-t] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[host [host ...]] [host [host ...]]
cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ...
@ -202,6 +202,12 @@ Install command is currently in beta.
are supported. Without argument CPU count is used by are supported. Without argument CPU count is used by
default. default.
**--log-server**
Start a log server for sub processes to use. This is
mainly useful when running cdist nested from a code-
local script. Log server is always implicitly started
for 'install' command.
**-n, --dry-run** **-n, --dry-run**
Do not execute code. Do not execute code.

View file

@ -60,10 +60,9 @@ def commandline():
if __name__ == "__main__": if __name__ == "__main__":
cdistpythonversion = '3.5' if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
if sys.version < cdistpythonversion:
print('Python >= {} is required on the source host.'.format( print('Python >= {} is required on the source host.'.format(
cdistpythonversion), file=sys.stderr) cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
sys.exit(1) sys.exit(1)
exit_code = 0 exit_code = 0