From 75fe3272b306235c02d7a19f9b5ba70fbb7838e0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 11 Aug 2017 15:04:26 +0200 Subject: [PATCH] Add file locking for -j parallel execution. --- cdist/config.py | 4 +-- cdist/core/manifest.py | 4 +-- cdist/emulator.py | 34 ++++++++++++------- cdist/flock.py | 58 +++++++++++++++++++++++++++++++++ cdist/test/manifest/__init__.py | 2 +- 5 files changed, 85 insertions(+), 17 deletions(-) create mode 100644 cdist/flock.py diff --git a/cdist/config.py b/cdist/config.py index b7ca1f84..cdff47eb 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -39,11 +39,9 @@ import cdist.hostsource import cdist.exec.local import cdist.exec.remote -from cdist import inventory - import cdist.util.ipaddr as ipaddr -from cdist import core +from cdist import core, inventory from cdist.util.remoteutil import inspect_ssh_mux_opts diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 6f941550..0a26601a 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -113,8 +113,8 @@ class Manifest(object): '__target_host_tags': self.local.target_host_tags, } - if self.log.getEffectiveLevel() == logging.DEBUG: - self.env.update({'__cdist_debug': "yes"}) + self.env.update( + {'__cdist_loglevel': str(self.log.getEffectiveLevel())}) def _open_logger(self): self.log = logging.getLogger(self.target_host[0]) diff --git a/cdist/emulator.py b/cdist/emulator.py index 7c9dfcca..abc37282 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -28,6 +28,7 @@ import sys import cdist from cdist import core +from cdist import flock class MissingRequiredEnvironmentVariableError(cdist.Error): @@ -94,20 +95,25 @@ class Emulator(object): """Emulate type commands (i.e. __file and co)""" self.commandline() - self.setup_object() - self.save_stdin() - self.record_requirements() - self.record_auto_requirements() - self.log.trace("Finished %s %s" % ( - self.cdist_object.path, self.parameters)) + self.init_object() + + # locking for parallel execution + with flock.Flock(self.flock_path) as lock: + self.setup_object() + self.save_stdin() + self.record_requirements() + self.record_auto_requirements() + self.log.trace("Finished %s %s" % ( + self.cdist_object.path, self.parameters)) def __init_log(self): """Setup logging facility""" - if '__cdist_debug' in self.env: - logging.root.setLevel(logging.DEBUG) + if '__cdist_loglevel' in self.env: + level = int(self.env['__cdist_loglevel']) else: - logging.root.setLevel(logging.INFO) + level = logging.OFF + logging.root.setLevel(level) self.log = logging.getLogger(self.target_host[0]) @@ -150,8 +156,8 @@ class Emulator(object): self.args = parser.parse_args(self.argv[1:]) self.log.trace('Args: %s' % self.args) - def setup_object(self): - # Setup object - and ensure it is not in args + def init_object(self): + # Initialize object - and ensure it is not in args if self.cdist_type.is_singleton: self.object_id = '' else: @@ -162,7 +168,13 @@ class Emulator(object): self.cdist_object = core.CdistObject( self.cdist_type, self.object_base_path, self.object_marker, self.object_id) + lockfname = ('.' + self.cdist_type.name + + self.object_id + '_' + + self.object_marker + '.lock') + lockfname = lockfname.replace(os.sep, '_') + self.flock_path = os.path.join(self.object_base_path, lockfname) + def setup_object(self): # Create object with given parameters self.parameters = {} for key, value in vars(self.args).items(): diff --git a/cdist/flock.py b/cdist/flock.py new file mode 100644 index 00000000..d8bac916 --- /dev/null +++ b/cdist/flock.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import fcntl +import logging +import os + + +log = logging.getLogger('cdist-flock') + + +class Flock(): + def __init__(self, path): + self.path = path + self.lockfd = None + + def flock(self): + log.debug('Acquiring lock on %s', self.path) + self.lockfd = open(self.path, 'w+') + fcntl.flock(self.lockfd, fcntl.LOCK_EX) + log.debug('Acquired lock on %s', self.path) + + def funlock(self): + log.debug('Releasing lock on %s', self.path) + fcntl.flock(self.lockfd, fcntl.LOCK_UN) + self.lockfd.close() + self.lockfd = None + try: + os.remove(self.path) + except FileNotFoundError: + pass + log.debug('Released lock on %s', self.path) + + def __enter__(self): + self.flock() + return self + + def __exit__(self, *args): + self.funlock() + return False diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py index e0da2d9f..95bf2768 100644 --- a/cdist/test/manifest/__init__.py +++ b/cdist/test/manifest/__init__.py @@ -136,7 +136,7 @@ class ManifestTestCase(test.CdistTestCase): current_level = self.log.getEffectiveLevel() self.log.setLevel(logging.DEBUG) manifest = cdist.core.manifest.Manifest(self.target_host, self.local) - self.assertTrue("__cdist_debug" in manifest.env) + self.assertTrue("__cdist_loglevel" in manifest.env) self.log.setLevel(current_level)