Configuration/Setting module added

conf/ucloud.conf

@@ -1,19 +1,3 @@
-[otp]
-server = https://otp.ungleich.ch/ungleichotp/
-verify_endpoint = verify/
-auth_name = replace_me
-auth_realm = replace_me
-auth_seed = replace_me
-
-[network]
-prefix_length = 64
-prefix = 2001:db8::/48
-vxlan_phy_dev = eno1
-
-[netbox]
-url   = https://replace-me.example.com
-token = replace_me
-
 [etcd]
 url = localhost
 port = 2379
@@ -21,33 +5,3 @@ port = 2379
 ca_cert
 cert_cert
 cert_key
-
-file_prefix        = /files/
-host_prefix        = /hosts/
-image_prefix       = /images/
-image_store_prefix = /imagestore/
-network_prefix     = /networks/
-request_prefix     = /requests/
-user_prefix        = /users/
-vm_prefix          = /vms/
-
-[storage]
-
-#values = filesystem, ceph
-backend = filesystem
-
-# if STORAGE_BACKEND = filesystem
-vm_dir  = /var/lib/ucloud/vms
-image_dir = /var/lib/ucloud/images
-
-# if STORAGE_BACKEND = ceph
-ceph_vm_pool  = ssd
-ceph_image_pool = ssd
-
-# Importing uploaded files
-file_dir = /var/lib/ucloud/files
-
-# For Migrating VMs over ssh/tcp
-[ssh]
-username
-private_key_path

scripts/ucloud

@@ -3,37 +3,60 @@
 import argparse
 import logging
 import importlib
-import sys
 import os
 import multiprocessing as mp
+import sys
+
+from ucloud.configure.main import update_config, configure_parser
 
 
-COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata']
+def exception_hook(exc_type, exc_value, exc_traceback):
+        logger.error(
+            "Uncaught exception",
+            exc_info=(exc_type, exc_value, exc_traceback)
+        )
+        print(exc_type, exc_value)
 
-if __name__ == "__main__":
+
+if __name__ == '__main__':
     logging.basicConfig(level=logging.DEBUG,
                     format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s',
                     filename='/var/log/ucloud.log', filemode='a')
-
     logger = logging.getLogger("ucloud")
 
-    arg_parser = argparse.ArgumentParser(prog='ucloud',
+    sys.excepthook = exception_hook
-                                         description='Open Source Cloud Management Software')
+    mp.set_start_method('spawn')
-    arg_parser.add_argument('-c', '--conf-dir', help="Configuration directory")
+
-    arg_parser.add_argument('component', choices=COMMANDS)
+    arg_parser = argparse.ArgumentParser()
-    arg_parser.add_argument('component_args', nargs='*')
+    subparsers = arg_parser.add_subparsers(dest="command")
+    
+    api_parser = subparsers.add_parser("api")
+    
+    host_parser = subparsers.add_parser("host")
+    host_parser.add_argument("--hostname", required=True)
+
+    scheduler_parser = subparsers.add_parser("scheduler")
+
+    filescanner_parser = subparsers.add_parser("filescanner")
+
+    imagescanner_parser = subparsers.add_parser("imagescanner")
+
+    metadata_parser = subparsers.add_parser("metadata")
+
+    config_parser = subparsers.add_parser("configure")
+    configure_parser(config_parser)
     args = arg_parser.parse_args()
 
-    if args.conf_dir:
+    if not args.command:
-        os.environ['UCLOUD_CONF_DIR'] = args.conf_dir
+        arg_parser.print_help()
+    else:
+        arguments = vars(args)
+        try:
+            name = arguments.pop('command')
+            mod = importlib.import_module("ucloud.{}.main".format(name))
+            main = getattr(mod, "main")
+            main(**arguments)
 
-    try:
+        except Exception as e:
-        mp.set_start_method('spawn')
+            logger.exception(e)
-        name = args.component
+            print(e)
-        mod = importlib.import_module("ucloud.{}.main".format(name))
-        main = getattr(mod, "main")
-        main(*args.component_args)
-
-    except Exception as e:
-        logger.exception(e)
-        print(e)

ucloud/api/helper.py

@@ -29,11 +29,7 @@ def check_otp(name, realm, token):
         return 400
 
     response = requests.post(
-        "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format(
+        config['otp']['verification_controller_url'], json=data
-            OTP_SERVER=config['otp']['server'],
-            OTP_VERIFY_ENDPOINT=config['otp']['verify_endpoint']
-        ),
-        json=data,
     )
     return response.status_code
 

ucloud/config.py

@@ -5,32 +5,18 @@ import logging
 from ucloud.common.host import HostPool
 from ucloud.common.request import RequestPool
 from ucloud.common.vm import VmPool
-from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler
+from ucloud.common.storage_handlers import (FileSystemBasedImageStorageHandler,
+                                            CEPHBasedImageStorageHandler)
 from ucloud.common.etcd_wrapper import Etcd3Wrapper
+from ucloud.settings import Settings
+from os.path import join as join_path
 
-log = logging.getLogger('ucloud.config')
+logger = logging.getLogger('ucloud.config')
 
-conf_name = 'ucloud.conf'
-conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud')
-config_file = os.path.join(conf_dir, conf_name)
 
-config = configparser.ConfigParser(allow_no_value=True)
 
-if os.access(config_file, os.R_OK):
+config = Settings()
-    config.read(config_file)
+etcd_client = config.get_etcd_client()
-else:
-    log.warning('Configuration file not found - using defaults')
-
-etcd_wrapper_args = ()
-etcd_wrapper_kwargs = {
-    'host':      config['etcd']['url'],
-    'port':      config['etcd']['port'],
-    'ca_cert':   config['etcd']['ca_cert'],
-    'cert_cert': config['etcd']['cert_cert'],
-    'cert_key':  config['etcd']['cert_key']
-}
-
-etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
 
 host_pool = HostPool(etcd_client, config['etcd']['host_prefix'])
 vm_pool = VmPool(etcd_client, config['etcd']['vm_prefix'])
@@ -38,7 +24,7 @@ request_pool = RequestPool(etcd_client, config['etcd']['request_prefix'])
 
 running_vms = []
 
-__storage_backend = config['storage']['backend']
+__storage_backend = config['storage']['storage_backend']
 if __storage_backend == 'filesystem':
     image_storage_handler = FileSystemBasedImageStorageHandler(
         vm_base=config['storage']['vm_dir'],

ucloud/configure/main.py

@@ -0,0 +1,67 @@
+import argparse
+import sys
+import os
+
+from ucloud.settings import Settings
+
+config = Settings()
+etcd_client = config.get_etcd_client()
+
+def update_config(section, kwargs):
+    uncloud_config = etcd_client.get(config.config_key,
+                                     value_in_json=True)
+    if not uncloud_config:
+        uncloud_config = {}
+    else:
+        uncloud_config = uncloud_config.value
+    
+    uncloud_config[section] = kwargs
+    etcd_client.put(config.config_key, uncloud_config, value_in_json=True)
+
+
+def configure_parser(parser):
+    configure_subparsers = parser.add_subparsers(dest="subcommand")
+    
+    otp_parser = configure_subparsers.add_parser("otp")
+    otp_parser.add_argument("--verification-controller-url",
+                            required=True, metavar="URL")
+    otp_parser.add_argument("--auth-name", required=True,
+                            metavar="OTP-NAME")
+    otp_parser.add_argument("--auth-realm", required=True,
+                            metavar="OTP-REALM")
+    otp_parser.add_argument("--auth-seed", required=True,
+                            metavar="OTP-SEED")
+
+    network_parser = configure_subparsers.add_parser("network")
+    network_parser.add_argument("--prefix-length", required=True, type=int)
+    network_parser.add_argument("--prefix", required=True)
+    network_parser.add_argument("--vxlan-phy-dev", required=True)
+
+    netbox_parser = configure_subparsers.add_parser("netbox")
+    netbox_parser.add_argument("--url", required=True)
+    netbox_parser.add_argument("--token", required=True)
+
+    ssh_parser = configure_subparsers.add_parser("ssh")
+    ssh_parser.add_argument('--username', default="root")
+    ssh_parser.add_argument('--private-key-path',
+                            default=os.path.expanduser("~/.ssh/id_rsa"))
+
+    storage_parser = configure_subparsers.add_parser("storage")
+    storage_parser.add_argument('--file-dir', required=True)
+    storage_parser_subparsers = storage_parser.add_subparsers(dest="storage_backend")
+    
+    filesystem_storage_parser = storage_parser_subparsers.add_parser("filesystem")
+    filesystem_storage_parser.add_argument('--vm-dir', required=True)
+    filesystem_storage_parser.add_argument('--image-dir', required=True)
+
+    ceph_storage_parser = storage_parser_subparsers.add_parser("ceph")
+    ceph_storage_parser.add_argument('--ceph-vm-pool', required=True)
+    ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('subcommand')
+    if not subcommand:
+        pass
+    else:
+        update_config(subcommand, kwargs)

ucloud/filescanner/main.py

@@ -4,6 +4,7 @@ import pathlib
 import subprocess as sp
 import time
 import sys
+
 from uuid import uuid4
 
 from . import logger

ucloud/host/main.py

@@ -8,17 +8,16 @@ from ucloud.common.etcd_wrapper import Etcd3Wrapper
 from ucloud.common.request import RequestEntry, RequestType
 from ucloud.config import (vm_pool, request_pool,
                     etcd_client, running_vms,
-                    etcd_wrapper_args, etcd_wrapper_kwargs,
                     HostPool, config)
 
 from .helper import find_free_port
-from . import virtualmachine
+from . import virtualmachine, logger
-from ucloud.host import logger
 
 
 def update_heartbeat(hostname):
     """Update Last HeartBeat Time for :param hostname: in etcd"""
-    client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
+    
+    client = config.get_etcd_client()
     host_pool = HostPool(client, config['etcd']['host_prefix'])
     this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
 
@@ -73,7 +72,7 @@ def maintenance(host):
                 running_vms.remove(_vm)
 
 def check():
-    if config['storage']['backend'] == 'filesystem' and \
+    if config['storage']['storage_backend'] == 'filesystem' and \
         not isdir(config['storage']['vm_dir']):
         
         print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned"
@@ -81,7 +80,6 @@ def check():
         sys.exit(1)
 
 
-
 def main(hostname):
     check()
 

ucloud/imagescanner/main.py

@@ -1,7 +1,9 @@
 import json
 import os
 import subprocess
+import sys
 
+from os.path import isdir
 from os.path import join as join_path
 from ucloud.config import etcd_client, config, image_storage_handler
 from ucloud.imagescanner import logger
@@ -20,10 +22,10 @@ def qemu_img_type(path):
 
 def check():
     """ check whether settings are sane, refuse to start if they aren't """
-    if config['storage']['backend'] == 'filesystem' and not isdir(config['storage']['image_dir']):
+    if config['storage']['storage_backend'] == 'filesystem' and not isdir(config['storage']['image_dir']):
-        print("You have set STORAGE_BACKEND to filesystem, but "
+        sys.exit("You have set STORAGE_BACKEND to filesystem, but "
-              "{} does not exist. Refusing to start".format(config['storage']['image_dir']))
+                 "{} does not exist. Refusing to start".format(config['storage']['image_dir'])
-        sys.exit(1)
+        )
 
     try:
         subprocess.check_output(['which', 'qemu-img'])

ucloud/scheduler/main.py

@@ -13,8 +13,6 @@ from . import logger
 
 
 def main():
-    logger.info("%s SESSION STARTED %s", '*' * 5, '*' * 5)
-
     pending_vms = []
 
     for request_iterator in [

ucloud/settings/__init__.py

@@ -0,0 +1,86 @@
+import configparser
+import logging
+import sys
+import os
+
+from ucloud.common.etcd_wrapper import Etcd3Wrapper
+
+
+logger = logging.getLogger(__name__)
+
+
+class CustomConfigParser(configparser.RawConfigParser):
+    def __getitem__(self, key):
+        try:
+            result = super().__getitem__(key)
+        except KeyError as err:
+            raise KeyError("Key '{}' not found in config file"\
+                           .format(key)) from err
+        else:
+            return result
+
+
+class Settings(object):
+    def __init__(self, config_key='/uncloud/config/'):
+        conf_name = 'ucloud.conf'
+        conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud')
+        config_file = os.path.join(conf_dir, conf_name)
+        
+        self.config_parser = CustomConfigParser(allow_no_value=True)
+        self.config_key = config_key
+
+        self.read_internal_values()
+        self.read_config_file_values(config_file)
+
+        self.etcd_wrapper_args = tuple()
+        self.etcd_wrapper_kwargs = {
+            'host':      self.config_parser['etcd']['url'],
+            'port':      self.config_parser['etcd']['port'],
+            'ca_cert':   self.config_parser['etcd']['ca_cert'],
+            'cert_cert': self.config_parser['etcd']['cert_cert'],
+            'cert_key':  self.config_parser['etcd']['cert_key']
+        }
+
+    
+    def get_etcd_client(self):
+        args = self.etcd_wrapper_args
+        kwargs = self.etcd_wrapper_kwargs
+        return Etcd3Wrapper(*args, **kwargs)
+        
+    def read_internal_values(self):
+        self.config_parser.read_dict({
+            'etcd': {
+                'file_prefix': '/files/',
+                'host_prefix': '/hosts/',
+                'image_prefix': '/images/',
+                'image_store_prefix': '/imagestore/',
+                'network_prefix': '/networks/',
+                'request_prefix': '/requests/',
+                'user_prefix': '/users/',
+                'vm_prefix': '/vms/',
+            }
+        })
+
+    def read_config_file_values(self, config_file):
+        try:
+            # Trying to read configuration file
+            with open(config_file, "r") as config_file_handle:
+                self.config_parser.read_file(config_file_handle)
+        except FileNotFoundError:
+            sys.exit('Configuration file {} not found!'.format(config_file))
+        except Exception as err:
+            logger.exception(err)
+            sys.exit("Error occurred while reading configuration file")
+
+    def read_values_from_etcd(self):
+        etcd_client = self.get_etcd_client()
+        config_from_etcd = etcd_client.get(self.config_key, value_in_json=True)
+        if config_from_etcd:
+            self.config_parser.read_dict(config_from_etcd.value)
+        else:
+            return
+            sys.exit("No settings found in etcd at key {}".format(self.config_key))
+    
+    def __getitem__(self, key):
+        self.read_values_from_etcd()
+        return self.config_parser[key]