Merge remote-tracking branch 'steven/master'

This commit is contained in:
Nico Schottelius 2011-10-12 22:44:16 +02:00
commit 7b5864dfce
20 changed files with 708 additions and 3 deletions

View file

@ -0,0 +1,94 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import logging
import os
import cdist
log = logging.getLogger(__name__)
'''
common:
runs only locally, does not need remote
env:
PATH: prepend directory with type emulator symlinks == local.bin_path
__target_host: the target host we are working on
__global: full qualified path to the global output dir == local.out_path
__cdist_manifest: full qualified path of the manifest == script
__cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator
== local.type_base_path
initial manifest is:
script: full qualified path to the initial manifest
env:
__manifest: path to .../conf/manifest/ == local.manifest_path
creates: new objects through type emulator
type manifeste is:
script: full qualified path to the type manifest
env:
__object: full qualified path to the object's dir
__object_id: the objects id
__object_fq: full qualified object id, iow: $type.name + / + object_id
__type: full qualified path to the type's dir
creates: new objects through type emulator
'''
class Manifest(object):
"""Represents a cdist manifest.
"""
def __init__(self, target_host, local):
self.target_host = target_host
self.local = local
self.env = {
'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
'__target_host': self.target_host,
'__global': self.local.out_path,
'__cdist_type_base_path': self.local.type_base_path, # for use in type emulator
}
def run_initial_manifest(self, script):
env = os.environ.copy()
env.update(self.env)
env['__manifest'] = self.local.manifest_path
return self.local.run_script(script, env=env)
def run_type_manifest(self, cdist_object):
env = os.environ.copy()
env.update(self.env)
env.update({
'__object': cdist_object.absolute_path,
'__object_id': cdist_object.object_id,
'__object_fq': cdist_object.path,
'__type': cdist_object.type.absolute_path,
})
script = os.path.join(self.local.type_base_path, cdist_object.type.manifest_path)
return self.local.run_script(script, env=env)

View file

@ -20,7 +20,6 @@
#
import logging
import os
import subprocess
import cdist
@ -44,7 +43,7 @@ class Wrapper(object):
def transfer_path(self, source, destination):
"""Transfer directory and previously delete the remote destination"""
self.remove_remote_path(destination)
self.run_or_fail(os.environ['__remote_copy'].split() +
self.run_or_fail(self.remote_copy.split() +
["-r", source, self.target_host + ":" + destination])
def shell_run_or_debug_fail(self, script, *args, remote=False, **kargs):

View file

131
lib/cdist/exec/local.py Normal file
View file

@ -0,0 +1,131 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# FIXME: common base class with Remote?
import io
import os
import sys
import subprocess
import shutil
import logging
import cdist
from cdist import core
class LocalScriptError(cdist.Error):
def __init__(self, script, command, script_content):
self.script = script
self.command = command
self.script_content = script_content
def __str__(self):
return "Local script execution failed: %s %s" % (self.script, self.command)
class Local(object):
"""Execute commands locally.
All interaction with the local side should be done through this class.
Directly accessing the local side from python code is a bug.
"""
def __init__(self, target_host, local_base_path, out_path):
self.target_host = target_host
self.base_path = local_base_path
# Local input
self.cache_path = os.path.join(self.base_path, "cache")
self.conf_path = os.path.join(self.base_path, "conf")
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.manifest_path = os.path.join(self.conf_path, "manifest")
self.type_base_path = os.path.join(self.conf_path, "type")
# FIXME: should not be needed anywhere
self.lib_path = os.path.join(self.base_path, "lib")
# Local output
self.out_path = out_path
self.bin_path = os.path.join(self.out_path, "bin")
self.global_explorer_out_path = os.path.join(self.out_path, "explorer")
self.object_base_path = os.path.join(self.out_path, "object")
self.log = logging.getLogger(self.target_host)
def create_directories(self):
self.mkdir(self.out_path)
self.mkdir(self.bin_path)
def rmdir(self, path):
"""Remove directory on the local side."""
self.log.debug("Local rmdir: %s", path)
shutil.rmtree(path)
def mkdir(self, path):
"""Create directory on the local side."""
self.log.debug("Local mkdir: %s", path)
os.makedirs(path, mode=0o700, exist_ok=True)
def run(self, command, env=None):
"""Run the given command with the given environment.
Return the output as a string.
"""
assert isinstance(command, (list, tuple)), "list or tuple argument expected, got: %s" % command
self.log.debug("Local run: %s", command)
try:
return subprocess.check_output(command, env=env).decode()
except subprocess.CalledProcessError:
raise cdist.Error("Command failed: " + " ".join(command))
except OSError as error:
raise cdist.Error(" ".join(*args) + ": " + error.args[1])
def run_script(self, script, env=None):
"""Run the given script with the given environment.
Return the output as a string.
"""
command = ["/bin/sh", "-e"]
command.append(script)
self.log.debug("Local run script: %s", command)
if env:
self.log.debug("Local run script env: %s", env)
try:
return subprocess.check_output(command, env=env).decode()
except subprocess.CalledProcessError as error:
script_content = self.run(["cat", script])
self.log.error("Code that raised the error:\n%s", script_content)
raise LocalScriptError(script, command, script_content)
except EnvironmentError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1])
def link_emulator(self, exec_path):
"""Link emulator to types"""
src = os.path.abspath(exec_path)
for cdist_type in core.Type.list_types(self.type_base_path):
dst = os.path.join(self.bin_path, cdist_type.name)
self.log.debug("Linking emulator: %s to %s", src, dst)
# FIXME: handle exceptions
os.symlink(src, dst)

134
lib/cdist/exec/remote.py Normal file
View file

@ -0,0 +1,134 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# FIXME: common base class with Local?
import io
import os
import sys
import subprocess
import logging
import cdist
class RemoteScriptError(cdist.Error):
def __init__(self, script, command, script_content):
self.script = script
self.command = command
self.script_content = script_content
def __str__(self):
return "Remote script execution failed: %s %s" % (self.script, self.command)
class Remote(object):
"""Execute commands remotely.
All interaction with the remote side should be done through this class.
Directly accessing the remote side from python code is a bug.
"""
def __init__(self, target_host, remote_base_path, remote_exec, remote_copy):
self.target_host = target_host
self.base_path = remote_base_path
self._exec = remote_exec
self._copy = remote_copy
self.conf_path = os.path.join(self.base_path, "conf")
self.object_path = os.path.join(self.base_path, "object")
self.type_path = os.path.join(self.conf_path, "type")
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.log = logging.getLogger(self.target_host)
def create_directories(self):
self.rmdir(self.base_path)
self.mkdir(self.base_path)
self.mkdir(self.conf_path)
def rmdir(self, path):
"""Remove directory on the remote side."""
self.log.debug("Remote rmdir: %s", path)
self.run(["rm", "-rf", path])
def mkdir(self, path):
"""Create directory on the remote side."""
self.log.debug("Remote mkdir: %s", path)
self.run(["mkdir", "-p", path])
def transfer(self, source, destination):
"""Transfer a file or directory to the remote side."""
self.log.debug("Remote transfer: %s -> %s", source, destination)
self.rmdir(destination)
command = self._copy.split()
command.extend(["-r", source, self.target_host + ":" + destination])
self.run_command(command)
def run(self, command, env=None):
"""Run the given command with the given environment on the remote side.
Return the output as a string.
"""
# prefix given command with remote_exec
cmd = self._exec.split()
cmd.append(self.target_host)
cmd.extend(command)
return self.run_command(cmd, env=None)
def run_command(self, command, env=None):
"""Run the given command with the given environment.
Return the output as a string.
"""
assert isinstance(command, (list, tuple)), "list or tuple argument expected, got: %s" % command
self.log.debug("Remote run: %s", command)
try:
return subprocess.check_output(command, env=env).decode()
except subprocess.CalledProcessError:
raise cdist.Error("Command failed: " + " ".join(command))
except OSError as error:
raise cdist.Error(" ".join(*args) + ": " + error.args[1])
def run_script(self, script, env=None):
"""Run the given script with the given environment on the remote side.
Return the output as a string.
"""
command = self._exec.split()
command.append(self.target_host)
command.extend(["/bin/sh", "-e"])
command.append(script)
self.log.debug("Remote run script: %s", command)
if env:
self.log.debug("Remote run script env: %s", env)
try:
return subprocess.check_output(command, env=env).decode()
except subprocess.CalledProcessError as error:
script_content = self.run(["cat", script])
self.log.error("Code that raised the error:\n%s", script_content)
raise RemoteScriptError(script, command, script_content)
except EnvironmentError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1])

View file

@ -28,7 +28,7 @@ import unittest
cdist_commands=["banner", "config", "install"]
cdist_base_path = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../"))
cdist_exec_path = os.path.join(cdist_base_path, "bin/cdist")

View file

View file

@ -0,0 +1,101 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import unittest
import os
import tempfile
import getpass
import shutil
import string
import random
#import logging
#logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s')
import cdist
from cdist.exec import local
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
local_base_path = fixtures
class LocalTestCase(unittest.TestCase):
def mkdtemp(self, **kwargs):
return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs)
def mkstemp(self, **kwargs):
return tempfile.mkstemp(prefix='tmp.cdist.test.', **kwargs)
def setUp(self):
self.temp_dir = self.mkdtemp()
target_host = 'localhost'
out_path = self.temp_dir
self.local = local.Local(target_host, local_base_path, out_path)
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_run_success(self):
self.local.run(['/bin/true'])
def test_run_fail(self):
self.assertRaises(cdist.Error, self.local.run, ['/bin/false'])
def test_run_script_success(self):
handle, script = self.mkstemp(dir=self.temp_dir)
fd = open(script, "w")
fd.writelines(["#!/bin/sh\n", "/bin/true"])
fd.close()
self.local.run_script(script)
def test_run_script_fail(self):
handle, script = self.mkstemp(dir=self.temp_dir)
fd = open(script, "w")
fd.writelines(["#!/bin/sh\n", "/bin/false"])
fd.close()
self.assertRaises(local.LocalScriptError, self.local.run_script, script)
def test_run_script_get_output(self):
handle, script = self.mkstemp(dir=self.temp_dir)
fd = open(script, "w")
fd.writelines(["#!/bin/sh\n", "echo foobar"])
fd.close()
self.assertEqual(self.local.run_script(script), "foobar\n")
def test_mkdir(self):
temp_dir = self.mkdtemp(dir=self.temp_dir)
os.rmdir(temp_dir)
self.local.mkdir(temp_dir)
self.assertTrue(os.path.isdir(temp_dir))
def test_rmdir(self):
temp_dir = self.mkdtemp(dir=self.temp_dir)
self.local.rmdir(temp_dir)
self.assertFalse(os.path.isdir(temp_dir))
def test_create_directories(self):
self.local.create_directories()
print('self.local.bin_path: %s' % self.local.bin_path)
self.assertTrue(os.path.isdir(self.local.out_path))
self.assertTrue(os.path.isdir(self.local.bin_path))

View file

@ -0,0 +1,111 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import unittest
import os
import tempfile
import getpass
import shutil
import string
import random
import cdist
from cdist.exec import remote
class RemoteTestCase(unittest.TestCase):
def mkdtemp(self, **kwargs):
return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs)
def mkstemp(self, **kwargs):
return tempfile.mkstemp(prefix='tmp.cdist.test.', **kwargs)
def setUp(self):
self.temp_dir = self.mkdtemp()
target_host = 'localhost'
remote_base_path = self.temp_dir
user = getpass.getuser()
remote_exec = "ssh -o User=%s -q" % user
remote_copy = "scp -o User=%s -q" % user
self.remote = remote.Remote(target_host, remote_base_path, remote_exec, remote_copy)
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_run_success(self):
self.remote.run(['/bin/true'])
def test_run_fail(self):
self.assertRaises(cdist.Error, self.remote.run, ['/bin/false'])
def test_run_script_success(self):
handle, script = self.mkstemp(dir=self.temp_dir)
fd = open(script, "w")
fd.writelines(["#!/bin/sh\n", "/bin/true"])
fd.close()
self.remote.run_script(script)
def test_run_script_fail(self):
handle, script = self.mkstemp(dir=self.temp_dir)
fd = open(script, "w")
fd.writelines(["#!/bin/sh\n", "/bin/false"])
fd.close()
self.assertRaises(remote.RemoteScriptError, self.remote.run_script, script)
def test_run_script_get_output(self):
handle, script = self.mkstemp(dir=self.temp_dir)
fd = open(script, "w")
fd.writelines(["#!/bin/sh\n", "echo foobar"])
fd.close()
self.assertEqual(self.remote.run_script(script), "foobar\n")
def test_mkdir(self):
temp_dir = self.mkdtemp(dir=self.temp_dir)
os.rmdir(temp_dir)
self.remote.mkdir(temp_dir)
self.assertTrue(os.path.isdir(temp_dir))
def test_rmdir(self):
temp_dir = self.mkdtemp(dir=self.temp_dir)
self.remote.rmdir(temp_dir)
self.assertFalse(os.path.isdir(temp_dir))
def test_transfer_file(self):
handle, source = self.mkstemp(dir=self.temp_dir)
target = self.mkdtemp(dir=self.temp_dir)
self.remote.transfer(source, target)
self.assertTrue(os.path.isfile(target))
def test_transfer_dir(self):
source = self.mkdtemp(dir=self.temp_dir)
# put a file in the directory as payload
handle, source_file = self.mkstemp(dir=source)
source_file_name = os.path.split(source_file)[-1]
target = self.mkdtemp(dir=self.temp_dir)
self.remote.transfer(source, target)
# test if the payload file is in the target directory
self.assertTrue(os.path.isfile(os.path.join(target, source_file_name)))
def test_create_directories(self):
self.remote.create_directories()
self.assertTrue(os.path.isdir(self.remote.base_path))
self.assertTrue(os.path.isdir(self.remote.conf_path))

View file

@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import unittest
import os
import tempfile
import getpass
import shutil
import string
import random
#import logging
#logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s')
import cdist
from cdist.exec import local
from cdist import core
from cdist.core import manifest
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
local_base_path = fixtures
class ManifestTestCase(unittest.TestCase):
def mkdtemp(self, **kwargs):
return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs)
def mkstemp(self, **kwargs):
return tempfile.mkstemp(prefix='tmp.cdist.test.', **kwargs)
def setUp(self):
self.temp_dir = self.mkdtemp()
target_host = 'localhost'
out_path = self.temp_dir
self.local = local.Local(target_host, local_base_path, out_path)
self.local.create_directories()
self.local.link_emulator(cdist.test.cdist_exec_path)
self.manifest = manifest.Manifest(target_host, self.local)
def tearDown(self):
shutil.rmtree(self.temp_dir)
def test_initial_manifest_environment(self):
initial_manifest = os.path.join(self.local.manifest_path, "dump_environment")
output_string = self.manifest.run_initial_manifest(initial_manifest)
output_dict = {}
for line in output_string.split('\n'):
if line:
key,value = line.split(': ')
output_dict[key] = value
self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path))
self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path)
self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_base_path)
self.assertEqual(output_dict['__manifest'], self.local.manifest_path)
def test_type_manifest_environment(self):
cdist_type = core.Type(self.local.type_base_path, '__dump_environment')
cdist_object = core.Object(cdist_type, self.local.object_base_path, 'whatever')
output_string = self.manifest.run_type_manifest(cdist_object)
output_dict = {}
for line in output_string.split('\n'):
if line:
key,value = line.split(': ')
output_dict[key] = value
self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path))
self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path)
self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_base_path)
self.assertEqual(output_dict['__type'], cdist_type.absolute_path)
self.assertEqual(output_dict['__object'], cdist_object.absolute_path)
self.assertEqual(output_dict['__object_id'], cdist_object.object_id)
self.assertEqual(output_dict['__object_fq'], cdist_object.path)

View file

@ -0,0 +1,7 @@
#!/bin/sh
echo "PATH: $PATH"
echo "__target_host: $__target_host"
echo "__global: $__global"
echo "__cdist_type_base_path: $__cdist_type_base_path"
echo "__manifest: $__manifest"

View file

@ -0,0 +1,4 @@
#!/bin/sh
__planet Saturn
__moon Prometheus --planet Saturn

View file

@ -0,0 +1,10 @@
#!/bin/sh
echo "PATH: $PATH"
echo "__target_host: $__target_host"
echo "__global: $__global"
echo "__cdist_type_base_path: $__cdist_type_base_path"
echo "__type: $__type"
echo "__object: $__object"
echo "__object_id: $__object_id"
echo "__object_fq: $__object_fq"

View file

@ -0,0 +1,8 @@
#!/bin/sh
if [ -f "$__object/parameter/name" ]; then
name="(cat "$__object/parameter/name")"
else
name="$__object_id"
echo "$name" > "$__object/parameter/name"
fi

View file

@ -0,0 +1 @@
planet

View file

@ -0,0 +1,8 @@
#!/bin/sh
if [ -f "$__object/parameter/name" ]; then
name="(cat "$__object/parameter/name")"
else
name="$__object_id"
echo "$name" > "$__object/parameter/name"
fi