From 151aa80ebd1a787ef3fce151948a634ffa94745d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 11 Jun 2020 09:22:06 +0200 Subject: [PATCH] Regain Python 3.2 support --- cdist/config.py | 3 ++- cdist/core/cdist_object.py | 5 +++++ cdist/core/cdist_type.py | 5 +++++ cdist/core/manifest.py | 6 ++++++ cdist/emulator.py | 5 +++++ cdist/exec/local.py | 25 +++++++++++++++++++------ cdist/exec/remote.py | 15 +++++++++------ cdist/exec/util.py | 26 +++++++++++++++++++++----- cdist/flock.py | 5 +++++ cdist/hostsource.py | 9 ++++++--- cdist/preos.py | 9 ++++++--- 11 files changed, 89 insertions(+), 24 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index b2d72f05..c0bdabd7 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -129,7 +129,8 @@ class Config(object): @staticmethod def hosts(source): try: - yield from cdist.hostsource.HostSource(source)() + for x in cdist.hostsource.HostSource(source)(): + yield x except (IOError, OSError, UnicodeError) as e: raise cdist.Error( "Error reading hosts from \'{}\': {}".format( diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 114a47e0..79e478c2 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -28,6 +28,11 @@ import cdist.core from cdist.util import fsproperty +# FileNotFoundError is added in 3.3. +if not hasattr(__builtins__, 'FileNotFoundError'): + FileNotFoundError = (OSError, IOError, ) + + class IllegalObjectIdError(cdist.Error): def __init__(self, object_id, message=None): self.object_id = object_id diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 4500f50d..a34d611e 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -26,6 +26,11 @@ import cdist.core import logging +# FileNotFoundError is added in 3.3. +if not hasattr(__builtins__, 'FileNotFoundError'): + FileNotFoundError = (OSError, IOError, ) + + class InvalidTypeError(cdist.Error): def __init__(self, name, type_path, type_absolute_path): self.name = name diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 8b833ff2..e608c32d 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -26,6 +26,12 @@ import os import cdist from . import util + +# FileNotFoundError is added in 3.3. +if not hasattr(__builtins__, 'FileNotFoundError'): + FileNotFoundError = (OSError, IOError, ) + + ''' common: runs only locally, does not need remote diff --git a/cdist/emulator.py b/cdist/emulator.py index 4eaf2c93..175eb450 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -32,6 +32,11 @@ from cdist import flock from cdist.core.manifest import Manifest +# FileNotFoundError is added in 3.3. +if not hasattr(__builtins__, 'FileNotFoundError'): + FileNotFoundError = (OSError, IOError, ) + + class MissingRequiredEnvironmentVariableError(cdist.Error): def __init__(self, name): self.name = name diff --git a/cdist/exec/local.py b/cdist/exec/local.py index ad6c6e36..6d37aee4 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -191,9 +191,14 @@ class Local(object): close_stdout = False close_stderr = False + stderr_special_devnull = False + stdout_special_devnull = False if quiet: - stderr = subprocess.DEVNULL - stdout = subprocess.DEVNULL + stderr, close_stderr = util._get_devnull() + stderr_special_devnull = not close_stderr + + stdout, close_stdout = util._get_devnull() + stdout_special_devnull = not close_stdout elif do_save_output: if not return_output and stdout is None: stdout = util.get_std_fd(self.stdout_base_path, 'local') @@ -228,8 +233,10 @@ class Local(object): output = None if do_save_output: - util.log_std_fd(self.log, command, stderr, 'Local stderr') - util.log_std_fd(self.log, command, stdout, 'Local stdout') + if not stderr_special_devnull: + util.log_std_fd(self.log, command, stderr, 'Local stderr') + if not stdout_special_devnull: + util.log_std_fd(self.log, command, stdout, 'Local stdout') return output except (OSError, subprocess.CalledProcessError) as error: raise cdist.Error(" ".join(command) + ": " + str(error.args[1])) @@ -237,9 +244,15 @@ class Local(object): if message_prefix: message.merge_messages() if close_stdout: - stdout.close() + if isinstance(stdout, int): + os.close(stdout) + else: + stdout.close() if close_stderr: - stderr.close() + if isinstance(stderr, int): + os.close(stderr) + else: + stderr.close() def run_script(self, script, env=None, return_output=False, message_prefix=None, stdout=None, stderr=None): diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index f72bf3bf..a63bd886 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -24,12 +24,10 @@ import os import glob import subprocess import logging -import multiprocessing import cdist import cdist.exec.util as util import cdist.util.ipaddr as ipaddr -from cdist.mputil import mp_pool_run def _wrap_addr(addr): @@ -298,10 +296,11 @@ class Remote(object): os_environ['__target_fqdn'] = self.target_host[2] self.log.trace("Remote run: %s", command) + special_devnull = False try: if self.quiet_mode: - stderr = subprocess.DEVNULL - close_stderr = False + stderr, close_stderr = util._get_devnull() + special_devnull = not close_stderr if return_output: output = subprocess.check_output(command, env=os_environ, stderr=stderr).decode() @@ -311,7 +310,8 @@ class Remote(object): output = None if self.save_output_streams: - util.log_std_fd(self.log, command, stderr, 'Remote stderr') + if not special_devnull: + util.log_std_fd(self.log, command, stderr, 'Remote stderr') util.log_std_fd(self.log, command, stdout, 'Remote stdout') return output @@ -323,4 +323,7 @@ class Remote(object): if close_stdout: stdout.close() if close_stderr: - stderr.close() + if isinstance(stderr, int): + os.close(stderr) + else: + stderr.close() diff --git a/cdist/exec/util.py b/cdist/exec/util.py index 90a26ad3..ae754610 100644 --- a/cdist/exec/util.py +++ b/cdist/exec/util.py @@ -172,11 +172,19 @@ def get_std_fd(base_path, name): return stdfd -def log_std_fd(log, command, stdfd, prefix): - if stdfd is not None and stdfd != subprocess.DEVNULL: - stdfd.seek(0, 0) - log.trace("Command: {}; {}: {}".format( - command, prefix, stdfd.read().decode())) +# subprocess.DEVNULL is added in 3.3. +if hasattr(subprocess, 'DEVNULL'): + def log_std_fd(log, command, stdfd, prefix): + if stdfd is not None and stdfd != subprocess.DEVNULL: + stdfd.seek(0, 0) + log.trace("Command: {}; {}: {}".format(command, prefix, + stdfd.read().decode())) +else: + def log_std_fd(log, command, stdfd, prefix): + if stdfd is not None and not isinstance(stdfd, int): + stdfd.seek(0, 0) + log.trace("Command: {}; {}: {}".format(command, prefix, + stdfd.read().decode())) def dist_conf_dir(): @@ -210,3 +218,11 @@ def resolve_conf_dirs_from_config_and_args(args): cfg = cdist.configuration.Configuration(args) configuration = cfg.get_config(section='GLOBAL') return resolve_conf_dirs(configuration, args.conf_dir) + + +# subprocess.DEVNULL is added in 3.3. +def _get_devnull(): + if hasattr(subprocess, 'DEVNULL'): + return (subprocess.DEVNULL, False) + else: + return (os.open(os.devnull, os.O_RDWR), True) diff --git a/cdist/flock.py b/cdist/flock.py index d8bac916..5a443648 100644 --- a/cdist/flock.py +++ b/cdist/flock.py @@ -24,6 +24,11 @@ import logging import os +# FileNotFoundError is added in 3.3. +if not hasattr(__builtins__, 'FileNotFoundError'): + FileNotFoundError = (OSError, IOError, ) + + log = logging.getLogger('cdist-flock') diff --git a/cdist/hostsource.py b/cdist/hostsource.py index a7b8f0b4..975326a4 100644 --- a/cdist/hostsource.py +++ b/cdist/hostsource.py @@ -68,9 +68,12 @@ class HostSource(object): return if isinstance(self.source, str): - yield from self._hosts_from_file() + for x in self._hosts_from_file(): + yield x else: - yield from self._hosts_from_sequence() + for x in self._hosts_from_sequence(): + yield x def __call__(self): - yield from self.hosts() + for x in self.hosts(): + yield x diff --git a/cdist/preos.py b/cdist/preos.py index bf2a8e60..4969b195 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -43,18 +43,21 @@ def scan_preos_dir_plugins(dir): module_name = fname try: module = __import__(module_name) - yield from preos_plugin(module) + for x in preos_plugin(module): + yield x clsmembers = inspect.getmembers(module, inspect.isclass) for cm in clsmembers: c = cm[1] - yield from preos_plugin(c) + for x in preos_plugin(c): + yield x except ImportError as e: log.warning("Cannot import '{}': {}".format(module_name, e)) def find_preos_plugins(): for dir in _PLUGINS_PATH: - yield from scan_preos_dir_plugins(dir) + for x in scan_preos_dir_plugins(dir): + yield x def find_preoses():