fancsali/cdist-backup
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

merge context back into config (it is in fact the same idea)

Browse source 
Signed-off-by: Nico Schottelius <nico@kr.ethz.ch>
This commit is contained in:
Nico Schottelius 2011-10-10 11:31:37 +02:00
commit 7823d0c997
2 changed files with 139 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

192
lib/cdist/config_install.py
View file

@ -24,10 +24,11 @@ import logging
import os import os
import stat import stat
import sys import sys
import tempfile
import time import time
import cdist.context
import cdist.core import cdist.core
import cdist.exec
import cdist.emulator import cdist.emulator
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -37,44 +38,115 @@ CODE_HEADER = "#!/bin/sh -e\n"
class ConfigInstall: class ConfigInstall:
"""Cdist main class to hold arbitrary data""" """Cdist main class to hold arbitrary data"""
def __init__(self, target_host, initial_manifest=False, def __init__(self, target_host,
base_path=False, initial_manifest=False, base_path=False, debug=False):
exec_path=sys.argv[0],
debug=False):
self.target_host = target_host self.target_host = target_host
os.environ['target_host'] = target_host
self.debug = debug self.debug = debug
self.exec_path = exec_path self.exec_path = sys.argv[0]
self.context = cdist.context.Context(self.target_host, # Base and Temp Base
initial_manifest=initial_manifest, if base_path:
base_path=base_path, self.base_path = base_path
debug=debug) else:
self.base_path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
# Local input
self.cache_path = os.path.join(self.base_path, "cache",
self.target_host)
self.conf_path = os.path.join(self.base_path, "conf")
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.manifest_path = os.path.join(self.conf_path, "manifest")
self.type_base_path = os.path.join(self.conf_path, "type")
self.lib_path = os.path.join(self.base_path, "lib")
if initial_manifest:
self.initial_manifest = initial_manifest
else:
self.initial_manifest = os.path.join(self.manifest_path, "init")
# Local output
if '__cdist_out_dir' in os.environ:
self.out_path = os.environ['__cdist_out_dir']
else:
self.out_path = os.path.join(tempfile.mkdtemp(), "out")
self.bin_path = os.path.join(self.out_path, "bin")
self.global_explorer_out_path = os.path.join(self.out_path, "explorer")
self.object_base_path = os.path.join(self.out_path, "object")
# Remote directory base
if '__cdist_remote_out_dir' in os.environ:
self.remote_base_path = os.environ['__cdist_remote_out_dir']
else:
self.remote_base_path = "/var/lib/cdist"
self.remote_conf_path = os.path.join(self.remote_base_path, "conf")
self.remote_object_path = os.path.join(self.remote_base_path, "object")
self.remote_type_path = os.path.join(self.remote_conf_path, "type")
self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer")
# Create directories
self.__init_local_paths()
self.__init_remote_paths()
def __init_remote_paths(self):
"""Initialise remote directory structure"""
self.remove_remote_path(self.remote_base_path)
self.remote_mkdir(self.remote_base_path)
self.remote_mkdir(self.remote_conf_path)
def __init_local_paths(self):
"""Initialise local directory structure"""
# Create base dir, if user supplied and not existing
if not os.path.isdir(self.base_path):
os.mkdir(self.base_path)
os.mkdir(self.out_path)
os.mkdir(self.global_explorer_out_path)
os.mkdir(self.bin_path)
def cleanup(self): def cleanup(self):
self.context.cleanup() # Do not use in __del__:
# http://docs.python.org/reference/datamodel.html#customization
# "other globals referenced by the __del__() method may already have been deleted
# or in the process of being torn down (e.g. the import machinery shutting down)"
#
log.debug("Saving " + self.out_path + " to " + self.cache_path)
# Remove previous cache
if os.path.exists(self.cache_path):
shutil.rmtree(self.cache_path)
shutil.move(self.out_path, self.cache_path)
def logfilter(self):
"""Add hostname to logs"""
def run_initial_manifest(self): def run_initial_manifest(self):
"""Run the initial manifest""" """Run the initial manifest"""
log.info("Running initial manifest %s", self.context.initial_manifest) log.info("Running initial manifest %s", self.initial_manifest)
env = { "__manifest" : self.context.manifest_path } env = { "__manifest" : self.manifest_path }
self.run_manifest(self.context.initial_manifest, extra_env=env) self.run_manifest(self.initial_manifest, extra_env=env)
def run_type_manifest(self, cdist_object): def run_type_manifest(self, cdist_object):
"""Run manifest for a specific object""" """Run manifest for a specific object"""
cdist_type = cdist_object.type cdist_type = cdist_object.type
manifest_path = os.path.join(self.context.type_base_path, manifest_path = os.path.join(self.type_base_path,
cdist_type.manifest_path) cdist_type.manifest_path)
log.debug("%s: Running %s", cdist_object.name, manifest_path) log.debug("%s: Running %s", cdist_object.name, manifest_path)
if os.path.exists(manifest_path): if os.path.exists(manifest_path):
env = { "__object" : os.path.join(self.context.object_base_path, env = { "__object" : os.path.join(self.object_base_path,
cdist_object.path), cdist_object.path),
"__object_id": cdist_object.object_id, "__object_id": cdist_object.object_id,
"__object_fq": cdist_object.name, "__object_fq": cdist_object.name,
"__type": os.path.join(self.context.type_base_path, "__type": os.path.join(self.type_base_path,
cdist_type.path) cdist_type.path)
} }
self.run_manifest(manifest_path, extra_env=env) self.run_manifest(manifest_path, extra_env=env)
@ -83,11 +155,11 @@ class ConfigInstall:
"""Run a manifest""" """Run a manifest"""
log.debug("Running manifest %s, env=%s", manifest_path, extra_env) log.debug("Running manifest %s, env=%s", manifest_path, extra_env)
env = os.environ.copy() env = os.environ.copy()
env['PATH'] = self.context.bin_path + ":" + env['PATH'] env['PATH'] = self.bin_path + ":" + env['PATH']
# Information required in every manifest # Information required in every manifest
env['__target_host'] = self.target_host env['__target_host'] = self.target_host
env['__global'] = self.context.out_path env['__global'] = self.out_path
# Submit debug flag to manifest, can be used by emulator and types # Submit debug flag to manifest, can be used by emulator and types
if self.debug: if self.debug:
@ -97,7 +169,7 @@ class ConfigInstall:
env['__cdist_manifest'] = manifest_path env['__cdist_manifest'] = manifest_path
# Required to find types in emulator # Required to find types in emulator
env['__cdist_type_base_path'] = self.context.type_base_path env['__cdist_type_base_path'] = self.type_base_path
# Other environment stuff # Other environment stuff
if extra_env: if extra_env:
@ -124,19 +196,19 @@ class ConfigInstall:
# #
env = os.environ.copy() env = os.environ.copy()
env['__target_host'] = self.target_host env['__target_host'] = self.target_host
env['__global'] = self.context.out_path env['__global'] = self.out_path
env["__object"] = os.path.join(self.context.object_base_path, cdist_object.path) env["__object"] = os.path.join(self.object_base_path, cdist_object.path)
env["__object_id"] = cdist_object.object_id env["__object_id"] = cdist_object.object_id
env["__object_fq"] = cdist_object.name env["__object_fq"] = cdist_object.name
env["__type"] = cdist_type.name env["__type"] = cdist_type.name
# gencode # gencode
for cmd in ["local", "remote"]: for cmd in ["local", "remote"]:
bin = os.path.join(self.context.type_base_path, bin = os.path.join(self.type_base_path,
getattr(cdist_type, "gencode_" + cmd + "_path")) getattr(cdist_type, "gencode_" + cmd + "_path"))
if os.path.isfile(bin): if os.path.isfile(bin):
outfile = os.path.join(self.context.object_base_path, outfile = os.path.join(self.object_base_path,
getattr(cdist_object, "code_" + cmd + "_path")) getattr(cdist_object, "code_" + cmd + "_path"))
outfile_fd = open(outfile, "w") outfile_fd = open(outfile, "w")
@ -164,12 +236,12 @@ class ConfigInstall:
cdist.exec.run_or_fail([code_local]) cdist.exec.run_or_fail([code_local])
# code remote # code remote
local_remote_code = os.path.join(self.context.object_base_path, local_remote_code = os.path.join(self.object_base_path,
cdist_object.code_remote_path) cdist_object.code_remote_path)
remote_remote_code = os.path.join(self.context.remote_object_path, remote_remote_code = os.path.join(self.remote_object_path,
cdist_object.code_remote_path) cdist_object.code_remote_path)
if os.path.isfile(local_remote_code): if os.path.isfile(local_remote_code):
self.context.transfer_path(local_remote_code, remote_remote_code) self.transfer_path(local_remote_code, remote_remote_code)
cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True)
cdist_object.ran = True cdist_object.ran = True
@ -181,12 +253,12 @@ class ConfigInstall:
self.transfer_type_explorers(cdist_type) self.transfer_type_explorers(cdist_type)
cmd = [] cmd = []
cmd.append("__explorer=" + self.context.remote_global_explorer_path) cmd.append("__explorer=" + self.remote_global_explorer_path)
cmd.append("__type_explorer=" + os.path.join( cmd.append("__type_explorer=" + os.path.join(
self.context.remote_type_path, self.remote_type_path,
cdist_type.explorer_path)) cdist_type.explorer_path))
cmd.append("__object=" + os.path.join( cmd.append("__object=" + os.path.join(
self.context.remote_object_path, self.remote_object_path,
cdist_object.path)) cdist_object.path))
cmd.append("__object_id=" + cdist_object.object_id) cmd.append("__object_id=" + cdist_object.object_id)
cmd.append("__object_fq=" + cdist_object.name) cmd.append("__object_fq=" + cdist_object.name)
@ -195,9 +267,9 @@ class ConfigInstall:
self.transfer_object_parameter(cdist_object) self.transfer_object_parameter(cdist_object)
for explorer in cdist_type.explorers: for explorer in cdist_type.explorers:
remote_cmd = cmd + [os.path.join(self.context.remote_base_path, remote_cmd = cmd + [os.path.join(self.remote_base_path,
cdist_type.explorer_path, explorer)] cdist_type.explorer_path, explorer)]
output = os.path.join(self.context.object_base_path, output = os.path.join(self.object_base_path,
cdist_object.explorer_path, explorer) cdist_object.explorer_path, explorer)
output_fd = open(output, mode='w') output_fd = open(output, mode='w')
log.debug("%s exploring %s using %s storing to %s", log.debug("%s exploring %s using %s storing to %s",
@ -210,8 +282,8 @@ class ConfigInstall:
def link_emulator(self): def link_emulator(self):
"""Link emulator to types""" """Link emulator to types"""
src = os.path.abspath(self.exec_path) src = os.path.abspath(self.exec_path)
for cdist_type in cdist.core.Type.list_types(self.context.type_base_path): for cdist_type in cdist.core.Type.list_types(self.type_base_path):
dst = os.path.join(self.context.bin_path, cdist_type.name) dst = os.path.join(self.bin_path, cdist_type.name)
log.debug("Linking emulator: %s to %s", src, dst) log.debug("Linking emulator: %s to %s", src, dst)
# FIXME: handle exception / make it more beautiful # FIXME: handle exception / make it more beautiful
@ -221,11 +293,11 @@ class ConfigInstall:
"""Run global explorers""" """Run global explorers"""
log.info("Running global explorers") log.info("Running global explorers")
src_path = self.context.global_explorer_path src_path = self.global_explorer_path
dst_path = self.context.global_explorer_out_path dst_path = self.global_explorer_out_path
remote_dst_path = self.context.remote_global_explorer_path remote_dst_path = self.remote_global_explorer_path
self.context.transfer_path(src_path, remote_dst_path) self.transfer_path(src_path, remote_dst_path)
for explorer in os.listdir(src_path): for explorer in os.listdir(src_path):
output_fd = open(os.path.join(dst_path, explorer), mode='w') output_fd = open(os.path.join(dst_path, explorer), mode='w')
@ -240,8 +312,8 @@ class ConfigInstall:
def stage_run(self): def stage_run(self):
"""The final (and real) step of deployment""" """The final (and real) step of deployment"""
log.info("Generating and executing code") log.info("Generating and executing code")
for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path, for cdist_object in cdist.core.Object.list_objects(self.object_base_path,
self.context.type_base_path): self.type_base_path):
log.debug("Run object: %s", cdist_object) log.debug("Run object: %s", cdist_object)
self.object_run(cdist_object) self.object_run(cdist_object)
@ -271,8 +343,8 @@ class ConfigInstall:
new_objects_created = True new_objects_created = True
while new_objects_created: while new_objects_created:
new_objects_created = False new_objects_created = False
for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path, for cdist_object in cdist.core.Object.list_objects(self.object_base_path,
self.context.type_base_path): self.type_base_path):
if cdist_object.prepared: if cdist_object.prepared:
log.debug("Skipping rerun of object %s", cdist_object) log.debug("Skipping rerun of object %s", cdist_object)
continue continue
@ -285,19 +357,19 @@ class ConfigInstall:
def transfer_object_parameter(self, cdist_object): def transfer_object_parameter(self, cdist_object):
"""Transfer the object parameter to the remote destination""" """Transfer the object parameter to the remote destination"""
src = os.path.join(self.context.object_base_path, src = os.path.join(self.object_base_path,
cdist_object.parameter_path) cdist_object.parameter_path)
dst = os.path.join(self.context.remote_object_path, dst = os.path.join(self.remote_object_path,
cdist_object.parameter_path) cdist_object.parameter_path)
# Synchronise parameter dir afterwards # Synchronise parameter dir afterwards
self.context.remote_mkdir(dst) self.remote_mkdir(dst)
self.context.transfer_path(src, dst) self.transfer_path(src, dst)
def transfer_global_explorers(self): def transfer_global_explorers(self):
"""Transfer the global explorers""" """Transfer the global explorers"""
self.remote_mkdir(self.context.remote_global_explorer_path) self.remote_mkdir(self.remote_global_explorer_path)
self.transfer_path(self.context.global_explorer_path, self.transfer_path(self.global_explorer_path,
self.remote_global_explorer_path) self.remote_global_explorer_path)
def transfer_type_explorers(self, cdist_type): def transfer_type_explorers(self, cdist_type):
@ -313,13 +385,27 @@ class ConfigInstall:
if len(explorers) > 0: if len(explorers) > 0:
rel_path = cdist_type.explorer_path rel_path = cdist_type.explorer_path
src = os.path.join(self.context.type_base_path, rel_path) src = os.path.join(self.type_base_path, rel_path)
dst = os.path.join(self.context.remote_type_path, rel_path) dst = os.path.join(self.remote_type_path, rel_path)
# Ensure full path until type exists: # Ensure full path until type exists:
# /var/lib/cdist/conf/type/__directory/explorer # /var/lib/cdist/conf/type/__directory/explorer
# /var/lib/cdist/conf/type/__directory may not exist, # /var/lib/cdist/conf/type/__directory may not exist,
# but remote_mkdir uses -p to fix this # but remote_mkdir uses -p to fix this
self.context.remote_mkdir(dst) self.remote_mkdir(dst)
self.context.transfer_path(src, dst) self.transfer_path(src, dst)
def remote_mkdir(self, directory):
"""Create directory on remote side"""
cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True)
def remove_remote_path(self, destination):
"""Ensure path on remote side vanished"""
cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True)
def transfer_path(self, source, destination):
"""Transfer directory and previously delete the remote destination"""
self.remove_remote_path(destination)
cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
["-r", source, self.target_host + ":" + destination])

137
lib/cdist/context.py
View file

@ -1,137 +0,0 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import os
import shutil
import sys
import tempfile
log = logging.getLogger(__name__)
import cdist.exec
class Context:
"""Storing context information"""
def __init__(self,
target_host,
initial_manifest=False,
base_path=False,
out_path=False,
remote_base_path=False,
debug=False):
self.target_host = target_host
# Base and Temp Base
if base_path:
self.base_path = base_path
else:
self.base_path = os.path.abspath(
os.path.join(os.path.dirname(__file__),
os.pardir,
os.pardir))
# Local input directories
self.cache_path = os.path.join(self.base_path, "cache", target_host)
self.conf_path = os.path.join(self.base_path, "conf")
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.manifest_path = os.path.join(self.conf_path, "manifest")
self.type_base_path = os.path.join(self.conf_path, "type")
self.lib_path = os.path.join(self.base_path, "lib")
if initial_manifest:
self.initial_manifest = initial_manifest
else:
self.initial_manifest = os.path.join(self.manifest_path, "init")
# Local output directories
if out_path:
self.out_path = out_path
else:
self.out_path = os.path.join(tempfile.mkdtemp(), "out")
self.bin_path = os.path.join(self.out_path, "bin")
self.global_explorer_out_path = os.path.join(self.out_path, "explorer")
self.object_base_path = os.path.join(self.out_path, "object")
# Remote directories
if remote_base_path:
self.remote_base_path = remote_base_path
else:
self.remote_base_path = "/var/lib/cdist"
self.remote_conf_path = os.path.join(self.remote_base_path, "conf")
self.remote_object_path = os.path.join(self.remote_base_path, "object")
self.remote_type_path = os.path.join(self.remote_conf_path, "type")
self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer")
# Create directories
self.__init_out_paths()
self.__init_remote_paths()
def cleanup(self):
# Do not use in __del__:
# http://docs.python.org/reference/datamodel.html#customization
# "other globals referenced by the __del__() method may already have been deleted
# or in the process of being torn down (e.g. the import machinery shutting down)"
#
log.debug("Saving " + self.out_path + " to " + self.cache_path)
# Remove previous cache
if os.path.exists(self.cache_path):
shutil.rmtree(self.cache_path)
shutil.move(self.out_path, self.cache_path)
def __init_out_paths(self):
"""Initialise output directory structure"""
# Create base dir, if user supplied and not existing
if not os.path.isdir(self.base_path):
os.mkdir(self.base_path)
os.mkdir(self.out_path)
os.mkdir(self.global_explorer_out_path)
os.mkdir(self.bin_path)
def __init_remote_paths(self):
"""Initialise remote directory structure"""
self.remove_remote_path(self.remote_base_path)
self.remote_mkdir(self.remote_base_path)
self.remote_mkdir(self.remote_conf_path)
def remote_mkdir(self, directory):
"""Create directory on remote side"""
cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True)
def remove_remote_path(self, destination):
cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True)
def transfer_path(self, source, destination):
"""Transfer directory and previously delete the remote destination"""
self.remove_remote_path(destination)
cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
["-r", source, self.target_host + ":" + destination])