cdist-backup/bin/cdist

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#

import argparse
import logging
import os
import subprocess
import shutil
import sys
import tempfile

BANNER = """
             ..          .       .x+=:.        s
           dF           @88>    z`    ^%      :8
          '88bu.        %8P        .   <k    .88
      .   '*88888bu      .       .@8Ned8"   :888ooo
 .udR88N    ^"*8888N   .@88u   .@^%8888"  -*8888888
<888'888k  beWE "888L ''888E` x88:  `)8b.   8888
9888 'Y"   888E  888E   888E  8888N=*8888   8888
9888       888E  888E   888E   %8"    R88   8888
9888       888E  888F   888E    @8Wou 9%   .8888Lu=
?8888u../ .888N..888    888&  .888888P`    ^%888*
 "8888P'   `"888*""     R888" `   ^"F        'Y"
   "P'        ""         ""
"""

# Given paths from installation
BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
CONF_DIR                   = os.path.join(BASE_DIR, "conf")
GLOBAL_EXPLORER_DIR        = os.path.join(CONF_DIR, "explorer")
LIB_DIR                    = os.path.join(BASE_DIR, "lib")
MANIFEST_DIR               = os.path.join(CONF_DIR, "manifest")
TYPE_DIR                   = os.path.join(CONF_DIR, "type")

REMOTE_BASE_DIR            = "/var/lib/cdist"
REMOTE_CONF_DIR            = os.path.join(REMOTE_BASE_DIR, "conf")
REMOTE_OBJECT_DIR          = os.path.join(REMOTE_BASE_DIR, "object")
REMOTE_TYPE_DIR            = os.path.join(REMOTE_CONF_DIR, "type")
REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer")

DOT_CDIST                  = ".cdist"
VERSION                    = "2.0.0"


#class Context(object):
#
#   def __init__(self, target_host):
#      self.target_host = target_host
#
#   # class variable
#   user_selber_shuld_wenn_aendert = 'bla'
#
#   # read only, aber statisch
#   @property
#   def remote_base_directory(self):
#      return "/var/lib/cdist"
#   @property.setter
#
#   @property
#   def special_foo(self):
#      return 'foo/{0}'.format(self.target_host)
#


logging.basicConfig(level=logging.DEBUG, format='%(levelname)s: %(message)s')
log = logging.getLogger()

# List types
def list_types():
   return os.listdir(TYPE_DIR)

def banner():
   """Guess what :-)"""
   print(BANNER)



class Cdist:
   """Cdist main class to hold arbitrary data"""

   def __init__(self, target_host, initial_manifest=False):
      self.target_host = target_host
      self.remote_prefix = ["ssh", "root@" + self.target_host]

      # Setup directory paths
      self.temp_dir = tempfile.mkdtemp()

      self.out_dir = os.path.join(self.temp_dir, "out")
      os.mkdir(self.out_dir)

      self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer")
      os.mkdir(self.global_explorer_out_dir)

      self.object_base_dir = os.path.join(self.out_dir, "object")

      # Setup binary directory + contents
      self.bin_dir = os.path.join(self.out_dir, "bin")
      os.mkdir(self.bin_dir)
      self.link_type_to_emulator()

      # List of type explorers transferred
      self.type_explorers_transferred = {}

      # Mostly static, but can be overwritten on user demand
      if initial_manifest:
         self.initial_manifest = initial_manifest
      else:
         self.initial_manifest = os.path.join(MANIFEST_DIR, "init")

   def cleanup(self):
      # Do not use in __del__:
      # http://docs.python.org/reference/datamodel.html#customization
      # "other globals referenced by the __del__() method may already have been deleted 
      # or in the process of being torn down (e.g. the import machinery shutting down)"
      #
      print("I should cleanup " + self.temp_dir)
      # shutil.rmtree(self.temp_dir)

   def exit_error(self, *args):
      log.error(*args)
      sys.exit(1)

   def remote_cat(filename):
      cmd = self.remote_prefix
      cmd.append("cat")
      cmd.append(filename)
      try:
         subprocess.call(cmd)
      except subprocess.CalledProcessError:
         log.error("Remote cat failed")

   def shell_run_or_debug_fail(self, script, *args, **kargs):
      # Manually execute /bin/sh, because sh -e does what we want
      # and sh -c -e does not exit if /bin/false called
      args[0][:0] = [ "/bin/sh", "-e" ]

      if "remote" in kargs:
         if kargs["remote"]:
            args[0][:0] = self.remote_prefix
            remote = true

         del kargs["remote"]

      log.debug("Shell exec: " + " ".join(*args))
      try:
         subprocess.check_call(*args, **kargs)
      except subprocess.CalledProcessError:
         log.error("Code that raised the error:\n")
         if remote:
            remote_cat(script)
         else:
            script_fd = open(script)
            print(script_fd.read())
            script_fd.close()

         self.exit_error("Non-Zero exit code exit of " + " ".join(*args))

   def run_or_fail(self, *args, **kargs):
      if "remote" in kargs:
         if kargs["remote"]:
            args[0][:0] = self.remote_prefix

         del kargs["remote"]

      log.debug("Exec: " + " ".join(*args))
      try:
         subprocess.check_call(*args, **kargs)
      except subprocess.CalledProcessError:
         self.exit_error("Command failed: " + " ".join(*args))

   def remove_remote_dir(self, destination):
      self.run_or_fail(["rm", "-rf",  destination], remote=True)

   def transfer_dir(self, source, destination):
      """Transfer directory and previously delete the remote destination"""
      self.remove_remote_dir(destination)
      self.run_or_fail(["scp", "-qr", source, "root@" + self.target_host + ":" + destination])

   def global_explorer_output_path(self, explorer):
      """Returns path of the output for a global explorer"""
      return os.path.join(self.global_explorer_out_dir, explorer)

   def type_explorer_output_dir(self, cdist_object):
      """Returns and creates dir of the output for a type explorer"""
      dir = os.path.join(self.object_dir(cdist_object), "explorer")
      if not os.path.isdir(dir):
         os.mkdir(dir)

      return dir

   def remote_global_explorer_path(self, explorer):
      """Returns path to the remote explorer"""
      return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer)

   def list_global_explorers(self):
      """Return list of available explorers"""
      return os.listdir(GLOBAL_EXPLORER_DIR)

   def list_type_explorers(self, type):
      """Return list of available explorers for a specific type"""
      dir = self.type_explorer_dir(type)
      if os.path.isdir(dir):
         list = os.listdir(dir)
      else:
         list = []

      return list

   def list_object_paths(self, starting_point = False):
      """Return list of paths of existing objects"""
      object_paths = []

      if not starting_point:
         starting_point = self.object_base_dir

      for content in os.listdir(starting_point):
         full_path = os.path.join(starting_point, content)
         print(full_path)
         if os.path.isdir(full_path):
            log.debug("Recursing for %s", full_path)
            object_paths.extend(self.list_object_paths(starting_point = full_path))

         # Directory contains .cdist -> is an object
         if content == DOT_CDIST:
            log.debug("Adding Object Path %s", starting_point)
            object_paths.append(starting_point)

      return object_paths

   def get_type_from_object(self, cdist_object):
      """Returns the first part (i.e. type) of an object"""
      return cdist_object.split(os.sep)[0]

   def object_dir(self, cdist_object):
      """Returns the full path to the object (including .cdist)"""
      return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST)

   def remote_object_dir(self, cdist_object):
      """Returns the remote full path to the object (including .cdist)"""
      return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST)

   def object_parameter_dir(self, cdist_object):
      """Returns the dir to the object parameter"""
      return os.path.join(self.object_dir(cdist_object), "parameter")

   def remote_object_parameter_dir(self, cdist_object):
      """Returns the remote dir to the object parameter"""
      return os.path.join(self.remote_object_dir(cdist_object), "parameter")

   def list_objects(self, starting_point = False):
      """Return list of existing objects"""

      if not starting_point:
         starting_point = self.object_base_dir

      object_paths = self.list_object_paths(starting_point)
      objects = []
      log.debug("Paths recieved: %s", object_paths)

      for path in object_paths:
         objects.append(os.path.relpath(path, starting_point))

      return objects

   def type_explorer_dir(self, type):
      """Return directory that holds the explorers of a type"""
      return os.path.join(TYPE_DIR, type, "explorer")

   def remote_type_explorer_dir(self, type):
      """Return remote directory that holds the explorers of a type"""
      return os.path.join(REMOTE_TYPE_DIR, type, "explorer")

   def transfer_object_parameter(self, cdist_object):
      """Transfer the object parameter to the remote destination"""
      self.transfer_dir(self.object_parameter_dir(cdist_object), 
                        self.remote_object_parameter_dir(cdist_object))

   def transfer_global_explorers(self):
      """Transfer the global explorers"""
      self.transfer_dir(GLOBAL_EXPLORER_DIR, REMOTE_GLOBAL_EXPLORER_DIR)

   def transfer_type_explorers(self, type):
      """Transfer explorers of a type, but only once"""
      if type in self.type_explorers_transferred:
         log.debug("Skipping retransfer for %s", type)
         return
      else:
         # Do not retransfer
         self.type_explorers_transferred[type] = 1

      src = self.type_explorer_dir(type)
      remote_base = os.path.join(REMOTE_TYPE_DIR, type)
      dst = self.remote_type_explorer_dir(type)

      # Only continue, if there is at least the directory
      if os.path.isdir(src):
         # Ensure that the path exists
         self.run_or_fail(["mkdir", "-p", remote_base], remote=True)
         self.transfer_dir(src, dst)


   def link_type_to_emulator(self):
      """Link type names to cdist-type-emulator"""
      for type in list_types():
         source = os.path.join(LIB_DIR, "cdist-type-emulator")
         destination = os.path.join(self.bin_dir, type)
         log.debug("Linking %s to %s", source, destination)
         os.symlink(source, destination)

   def run_global_explores(self):
      """Run global explorers"""
      explorers = self.list_global_explorers()
      if(len(explorers) == 0):
         self.exit_error("No explorers found in", GLOBAL_EXPLORER_DIR)

      self.transfer_global_explorers()
      for explorer in explorers:
         output = self.global_explorer_output_path(explorer)
         output_fd = open(output, mode='w')
         cmd = []
         cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR)
         cmd.append(self.remote_global_explorer_path(explorer))

         self.run_or_fail(cmd, stdout=output_fd, remote=True)
         output_fd.close()

   def run_type_explorer(self, cdist_object):
      """Run type specific explorers for objects"""
      # Based on bin/cdist-object-explorer-run

      # Transfering explorers for this type
      type = self.get_type_from_object(cdist_object)
      self.transfer_type_explorers(type)

      cmd = []
      cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR)
      cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type))

      # Need to transfer at least the parameters for objects to be useful
      self.transfer_object_parameter(cdist_object)
      cmd.append("__object=" + self.remote_object_dir(cdist_object))

      explorers = self.list_type_explorers(type)
      for explorer in explorers:
         remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)]
         output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer)
         output_fd = open(output, mode='w')
         self.run_or_fail(remote_cmd, stdout=output_fd, remote=True)
         output_fd.close()

   def init_deploy(self):
      log.info("Creating clean directory structure")

      # Ensure there is no old stuff, neither local nor remote
   #   remote_run_or_fail(hostname, ["rm -rf", "${__cdist_remote_base_dir}"])
   #
   #   # Create base directories
   #   remote_run_or_fail(hostname,["mkdir -p", "${__cdist_remote_base_dir}"])
   #
   #   # Link configuraion source directory - consistent with remote
   #   run_or_fail(["ln -sf", "$__cdist_conf_dir", "$__cdist_local_base_dir/$__cdist_name_conf_dir"])

   def run_initial_manifest(self):
      """Run the initial manifest"""
      log.info("Running the initial manifest")
      env = os.environ.copy()
      env['PATH'] = self.bin_dir + ":" + env['PATH']

      env['__target_host']    = self.target_host
      env['__global']         = self.out_dir
      
      # Legacy stuff to make cdist-type-emulator work
      env['__cdist_conf_dir']       = CONF_DIR
      env['__cdist_core_dir']       = os.path.join(BASE_DIR, "core")
      env['__cdist_local_base_dir'] = self.temp_dir
      env['__cdist_manifest']       = self.initial_manifest

      self.shell_run_or_debug_fail(self.initial_manifest,
                                  [self.initial_manifest],
                                  env=env)


   def deploy_to(self):
      """Mimic the old deploy to: Deploy to one host"""
      log.info("Deploying to host " + self.target_host)
      self.init_deploy()
      self.run_global_explores()
      self.run_initial_manifest()
      
      objects = self.list_objects()

      for cdist_object in objects:
         self.run_type_explorer(cdist_object)


if __name__ == "__main__":
   parser = argparse.ArgumentParser(description='cdist ' + VERSION)
   parser.add_argument('host', nargs='*', help='one or more hosts to operate on')
   parser.add_argument('-b', '--banner',
       help='Show cdist banner',
       action='store_true', dest='banner')
   parser.add_argument('-d', '--debug', help='Set log level to debug',
       action='store_true')
   parser.add_argument('-i', '--initial-manifest', 
       help='Path to a cdist manifest or - to read from stdin',
       dest='manifest', required=False)
   parser.add_argument('-p', '--parallel',
       help='Operate on multiple hosts in parallel',
       action='store_true', dest='parallel')
   parser.add_argument('-s', '--sequential',
       help='Operate on multiple hosts sequentially',
       action='store_false', dest='parallel')
   
   args = parser.parse_args(sys.argv[1:])
   if args.debug:
       logging.root.setLevel(logging.DEBUG)

   if args.banner:
      banner()
      sys.exit(0)
   
   try:
      log.debug(args)

      for host in args.host:
         c = Cdist(host, initial_manifest=args.manifest)
         c.deploy_to()
         c.cleanup()
   except KeyboardInterrupt:
       sys.exit(0)