Merge pull request #553 from darko-poljak/jobs-locking

Add file locking for -j parallel execution
This commit is contained in:
Darko Poljak 2017-08-13 12:28:22 +02:00 committed by GitHub
commit f96829f657
3 changed files with 78 additions and 11 deletions

View file

@ -39,11 +39,9 @@ import cdist.hostsource
import cdist.exec.local
import cdist.exec.remote
from cdist import inventory
import cdist.util.ipaddr as ipaddr
from cdist import core
from cdist import core, inventory
from cdist.util.remoteutil import inspect_ssh_mux_opts

View file

@ -28,6 +28,7 @@ import sys
import cdist
from cdist import core
from cdist import flock
class MissingRequiredEnvironmentVariableError(cdist.Error):
@ -94,12 +95,16 @@ class Emulator(object):
"""Emulate type commands (i.e. __file and co)"""
self.commandline()
self.setup_object()
self.save_stdin()
self.record_requirements()
self.record_auto_requirements()
self.log.trace("Finished %s %s" % (
self.cdist_object.path, self.parameters))
self.init_object()
# locking for parallel execution
with flock.Flock(self.flock_path) as lock:
self.setup_object()
self.save_stdin()
self.record_requirements()
self.record_auto_requirements()
self.log.trace("Finished %s %s" % (
self.cdist_object.path, self.parameters))
def __init_log(self):
"""Setup logging facility"""
@ -154,8 +159,8 @@ class Emulator(object):
self.args = parser.parse_args(self.argv[1:])
self.log.trace('Args: %s' % self.args)
def setup_object(self):
# Setup object - and ensure it is not in args
def init_object(self):
# Initialize object - and ensure it is not in args
if self.cdist_type.is_singleton:
self.object_id = ''
else:
@ -166,7 +171,13 @@ class Emulator(object):
self.cdist_object = core.CdistObject(
self.cdist_type, self.object_base_path, self.object_marker,
self.object_id)
lockfname = ('.' + self.cdist_type.name +
self.object_id + '_' +
self.object_marker + '.lock')
lockfname = lockfname.replace(os.sep, '_')
self.flock_path = os.path.join(self.object_base_path, lockfname)
def setup_object(self):
# Create object with given parameters
self.parameters = {}
for key, value in vars(self.args).items():

58
cdist/flock.py Normal file
View file

@ -0,0 +1,58 @@
# -*- coding: utf-8 -*-
#
# 2017 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import fcntl
import logging
import os
log = logging.getLogger('cdist-flock')
class Flock():
def __init__(self, path):
self.path = path
self.lockfd = None
def flock(self):
log.debug('Acquiring lock on %s', self.path)
self.lockfd = open(self.path, 'w+')
fcntl.flock(self.lockfd, fcntl.LOCK_EX)
log.debug('Acquired lock on %s', self.path)
def funlock(self):
log.debug('Releasing lock on %s', self.path)
fcntl.flock(self.lockfd, fcntl.LOCK_UN)
self.lockfd.close()
self.lockfd = None
try:
os.remove(self.path)
except FileNotFoundError:
pass
log.debug('Released lock on %s', self.path)
def __enter__(self):
self.flock()
return self
def __exit__(self, *args):
self.funlock()
return False