Added --conf-dir, --etcd-{host,port,ca_cert,cert_cert,cert_key} parameters to...

Added --conf-dir, --etcd-{host,port,ca_cert,cert_cert,cert_key} parameters to cli and settings is now accessbile through uncloud.shared.shared.settings

scripts/uncloud

@@ -3,14 +3,13 @@ import logging
 import sys
 import importlib
 import argparse
+import os
 
-from uncloud import UncloudException
-
-# the components that use etcd
-ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
+from etcd3.exceptions import ConnectionFailedError
 
-ALL_COMPONENTS = ETCD_COMPONENTS.copy()
-ALL_COMPONENTS.append('cli')
+from uncloud.common import settings
+from uncloud import UncloudException
+from uncloud.common.cli import resolve_otp_credentials
 
 
 def exception_hook(exc_type, exc_value, exc_traceback):
@@ -22,6 +21,13 @@ def exception_hook(exc_type, exc_value, exc_traceback):
 
 sys.excepthook = exception_hook
 
+# the components that use etcd
+ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
+
+ALL_COMPONENTS = ETCD_COMPONENTS.copy()
+ALL_COMPONENTS.append('cli')
+
+
 if __name__ == '__main__':
     # Setting up root logger
     logger = logging.getLogger()
@@ -31,15 +37,13 @@ if __name__ == '__main__':
     subparsers = arg_parser.add_subparsers(dest='command')
 
     parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument('--debug', '-d',
-                               action='store_true',
-                               default=False,
+    parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
                                help='More verbose logging')
-    parent_parser.add_argument('--conf-dir', '-c',
-                               help='Configuration directory')
+    parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
+                               default=os.path.expanduser('~/uncloud'))
 
     etcd_parser = argparse.ArgumentParser(add_help=False)
-    etcd_parser.add_argument('--etcd-host')
+    etcd_parser.add_argument('--etcd-host', dest='etcd_url')
     etcd_parser.add_argument('--etcd-port')
     etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
     etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
@@ -54,25 +58,36 @@ if __name__ == '__main__':
         else:
             subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
 
-    args = arg_parser.parse_args()
-    if not args.command:
+    arguments = vars(arg_parser.parse_args())
+    etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
+    etcd_arguments = {
+        'etcd': {
+            key.replace('etcd_', ''): arguments[key]
+            for key in etcd_arguments
+        }
+    }
+    if not arguments['command']:
         arg_parser.print_help()
     else:
-        arguments = vars(args)
+        # Initializing Settings and resolving otp_credentials
+        # It is neccessary to resolve_otp_credentials after argument parsing is done because
+        # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
+        # providing the default values for --name, --realm and --seed arguments from the values
+        # we read from file. But, now we are asking user about where the config file lives. So,
+        # to providing default value is not possible before parsing arguments. So, we are doing
+        # it after..
+        settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
+        resolve_otp_credentials(arguments)
+
         name = arguments.pop('command')
         mod = importlib.import_module('uncloud.{}.main'.format(name))
         main = getattr(mod, 'main')
 
-        # If the component requires etcd3, we import it and catch the
-        # etcd3.exceptions.ConnectionFailedError
-        if name in ETCD_COMPONENTS:
-            import etcd3
-
         try:
             main(arguments)
         except UncloudException as err:
             logger.error(err)
-            sys.exit(1)
+        except ConnectionFailedError:
+            logger.error('Cannot connect to etcd')
         except Exception as err:
             logger.exception(err)
-            sys.exit(1)

uncloud/api/common_fields.py

 import os
 
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 
 
 class Optional:
@@ -54,9 +53,7 @@ class VmUUIDField(Field):
 
     def vm_uuid_validation(self):
         r = shared.etcd_client.get(
-            os.path.join(settings["etcd"]["vm_prefix"], self.uuid)
+            os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
         )
         if not r:
-            self.add_error(
-                "VM with uuid {} does not exists".format(self.uuid)
-            )
+            self.add_error("VM with uuid {} does not exists".format(self.uuid))

uncloud/api/create_image_store.py

@@ -4,7 +4,6 @@ import os
 from uuid import uuid4
 
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 
 data = {
     'is_public': True,
@@ -15,6 +14,6 @@ data = {
 }
 
 shared.etcd_client.put(
-    os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex),
+    os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
     json.dumps(data),
 )

uncloud/api/helper.py

@@ -7,7 +7,6 @@ import requests
 from pyotp import TOTP
 
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 
 logger = logging.getLogger(__name__)
 
@@ -15,9 +14,9 @@ logger = logging.getLogger(__name__)
 def check_otp(name, realm, token):
     try:
         data = {
-            "auth_name": settings["otp"]["auth_name"],
-            "auth_token": TOTP(settings["otp"]["auth_seed"]).now(),
-            "auth_realm": settings["otp"]["auth_realm"],
+            "auth_name": shared.settings["otp"]["auth_name"],
+            "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
+            "auth_realm": shared.settings["otp"]["auth_realm"],
             "name": name,
             "realm": realm,
             "token": token,
@@ -25,13 +24,13 @@ def check_otp(name, realm, token):
     except binascii.Error as err:
         logger.error(
             "Cannot compute OTP for seed: {}".format(
-                settings["otp"]["auth_seed"]
+                shared.settings["otp"]["auth_seed"]
             )
         )
         return 400
 
     response = requests.post(
-        settings["otp"]["verification_controller_url"], json=data
+        shared.settings["otp"]["verification_controller_url"], json=data
     )
     return response.status_code
 
@@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client):
         )
 
     images = etcd_client.get_prefix(
-        settings["etcd"]["image_prefix"], value_in_json=True
+        shared.settings["etcd"]["image_prefix"], value_in_json=True
     )
 
     # Try to find image with name == image_name and store_name == store_name
@@ -111,9 +110,7 @@ def random_bytes(num=6):
     return [random.randrange(256) for _ in range(num)]
 
 
-def generate_mac(
-    uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
-):
+def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
     mac = random_bytes()
     if oui:
         if type(oui) == str:
@@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix):
     lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
     prefix = ipaddress.IPv6Address(prefix)
     return str(prefix + int(lower_part))
+

uncloud/api/schemas.py

@@ -22,7 +22,6 @@ import bitmath
 from uncloud.common.host import HostStatus
 from uncloud.common.vm import VMStatus
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 from . import helper, logger
 from .common_fields import Field, VmUUIDField
 from .helper import check_otp, resolve_vm_name
@@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema):
     def file_uuid_validation(self):
         file_entry = shared.etcd_client.get(
             os.path.join(
-                settings["etcd"]["file_prefix"], self.uuid.value
+                shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
             )
         )
         if file_entry is None:
@@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema):
     def image_store_name_validation(self):
         image_stores = list(
             shared.etcd_client.get_prefix(
-                settings["etcd"]["image_store_prefix"]
+                shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
             )
         )
 
@@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema):
             for net in _network:
                 network = shared.etcd_client.get(
                     os.path.join(
-                        settings["etcd"]["network_prefix"],
+                        shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
                         self.name.value,
                         net,
                     ),
@@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema):
             self.add_error("Can't migrate non-running VM")
 
         if vm.hostname == os.path.join(
-            settings["etcd"]["host_prefix"], self.destination.value
+            shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
         ):
             self.add_error(
                 "Destination host couldn't be same as Source Host"
@@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema):
         super().__init__(data, fields=fields)
 
     def network_name_validation(self):
-        print(self.name.value, self.network_name.value)
-        key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
-        print(key)
+        key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
         network = shared.etcd_client.get(key, value_in_json=True)
         if network:
             self.add_error(

uncloud/cli/helper.py

@@ -5,23 +5,14 @@ import binascii
 
 from pyotp import TOTP
 from os.path import join as join_path
-from uncloud.common.settings import settings
+from uncloud.common.shared import shared
 
 
 def get_otp_parser():
     otp_parser = argparse.ArgumentParser('otp')
-    try:
-        name = settings['client']['name']
-        realm = settings['client']['realm']
-        seed = settings['client']['seed']
-    except Exception:
-        otp_parser.add_argument('--name', required=True)
-        otp_parser.add_argument('--realm', required=True)
-        otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED')
-    else:
-        otp_parser.add_argument('--name', default=name)
-        otp_parser.add_argument('--realm', default=realm)
-        otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED')
+    otp_parser.add_argument('--name')
+    otp_parser.add_argument('--realm')
+    otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
 
     return otp_parser
 
@@ -34,11 +25,15 @@ def load_dump_pretty(content):
 
 
 def make_request(*args, data=None, request_method=requests.post):
-    r = request_method(join_path(settings['client']['api_server'], *args), json=data)
     try:
-        print(load_dump_pretty(r.content))
-    except Exception:
-        print('Error occurred while getting output from api server.')
+        r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
+    except requests.exceptions.RequestException:
+        print('Error occurred while connecting to API server.')
+    else:
+        try:
+            print(load_dump_pretty(r.content))
+        except Exception:
+            print('Error occurred while getting output from api server.')
 
 
 def get_token(seed):

uncloud/common/cli.py

+from uncloud.common.shared import shared
+from pyotp import TOTP
+
+
+def get_token(seed):
+    if seed is not None:
+        try:
+            token = TOTP(seed).now()
+        except Exception:
+            raise Exception('Invalid seed')
+        else:
+            return token
+
+
+def resolve_otp_credentials(kwargs):
+    d = {
+        'name': shared.settings['client']['name'],
+        'realm': shared.settings['client']['realm'],
+        'token': get_token(shared.settings['client']['seed'])
+    }
+
+    for k, v in d.items():
+        if k in kwargs and kwargs[k] is None:
+            kwargs.update({k: v})
+
+    return d

uncloud/common/settings.py

@@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper
 from os.path import join as join_path
 
 logger = logging.getLogger(__name__)
+settings = None
 
 
 class CustomConfigParser(configparser.RawConfigParser):
@@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser):
 
 
 class Settings(object):
-    def __init__(self):
+    def __init__(self, conf_dir, seed_value=None):
         conf_name = 'uncloud.conf'
-        conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
         self.config_file = join_path(conf_dir, conf_name)
 
         # this is used to cache config from etcd for 1 minutes. Without this we
@@ -38,15 +38,19 @@ class Settings(object):
         self.config_parser.add_section('etcd')
         self.config_parser.set('etcd', 'base_prefix', '/')
 
-        try:
+        if os.access(self.config_file, os.R_OK):
             self.config_parser.read(self.config_file)
-        except Exception as err:
-            logger.error('%s', err)
-
+        else:
+            raise FileNotFoundError('Config file %s not found!', self.config_file)
         self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
 
         self.read_internal_values()
 
+        if seed_value is None:
+            seed_value = dict()
+
+        self.config_parser.read_dict(seed_value)
+
     def get_etcd_client(self):
         args = tuple()
         try:
@@ -128,4 +132,5 @@ class Settings(object):
         return self.config_parser[key]
 
 
-settings = Settings()
+def get_settings():
+    return settings

uncloud/common/shared.py

-from uncloud.common.settings import settings
+from uncloud.common.settings import get_settings
 from uncloud.common.vm import VmPool
 from uncloud.common.host import HostPool
 from uncloud.common.request import RequestPool
-from uncloud.common.storage_handlers import get_storage_handler
+import uncloud.common.storage_handlers as storage_handlers
 
 
 class Shared:
+    @property
+    def settings(self):
+        return get_settings()
+
     @property
     def etcd_client(self):
-        return settings.get_etcd_client()
+        return self.settings.get_etcd_client()
 
     @property
     def host_pool(self):
-        return HostPool(
-            self.etcd_client, settings["etcd"]["host_prefix"]
-        )
+        return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
 
     @property
     def vm_pool(self):
-        return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"])
+        return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
 
     @property
     def request_pool(self):
-        return RequestPool(
-            self.etcd_client, settings["etcd"]["request_prefix"]
-        )
+        return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
 
     @property
     def storage_handler(self):
-        return get_storage_handler()
+        return storage_handlers.get_storage_handler()
 
 
 shared = Shared()

uncloud/common/storage_handlers.py

@@ -6,8 +6,7 @@ import stat
 from abc import ABC
 from . import logger
 from os.path import join as join_path
-
-from uncloud.common.settings import settings as config
+import uncloud.common.shared as shared
 
 
 class ImageStorageHandler(ABC):
@@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
 
 
 def get_storage_handler():
-    __storage_backend = config["storage"]["storage_backend"]
+    __storage_backend = shared.shared.settings["storage"]["storage_backend"]
     if __storage_backend == "filesystem":
         return FileSystemBasedImageStorageHandler(
-            vm_base=config["storage"]["vm_dir"],
-            image_base=config["storage"]["image_dir"],
+            vm_base=shared.shared.settings["storage"]["vm_dir"],
+            image_base=shared.shared.settings["storage"]["image_dir"],
         )
     elif __storage_backend == "ceph":
         return CEPHBasedImageStorageHandler(
-            vm_base=config["storage"]["ceph_vm_pool"],
-            image_base=config["storage"]["ceph_image_pool"],
+            vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
+            image_base=shared.shared.settings["storage"]["ceph_image_pool"],
         )
     else:
-        raise Exception("Unknown Image Storage Handler")
+        raise Exception("Unknown Image Storage Handler")
\ No newline at end of file

uncloud/configure/main.py

 import os
 import argparse
 
-from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 
 arg_parser = argparse.ArgumentParser('configure', add_help=False)
@@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
 
 
 def update_config(section, kwargs):
-    uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
+    uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
     if not uncloud_config:
         uncloud_config = {}
     else:
         uncloud_config = uncloud_config.value
 
     uncloud_config[section] = kwargs
-    shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
+    shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
 
 
-def main(**kwargs):
-    subcommand = kwargs.pop('subcommand')
+def main(arguments):
+    subcommand = arguments['subcommand']
     if not subcommand:
         arg_parser.print_help()
     else:
-        update_config(subcommand, kwargs)
+        update_config(subcommand, arguments)

uncloud/filescanner/main.py

@@ -9,7 +9,6 @@ import bitmath
 from uuid import uuid4
 
 from . import logger
-from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 
 arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
@@ -53,7 +52,7 @@ def track_file(file, base_dir, host):
         file_path = file_path.relative_to(owner)
         creation_date = time.ctime(os.stat(file_str).st_ctime)
 
-        entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
+        entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
         entry_value = {
             'filename': str(file_path),
             'owner': owner,
@@ -70,7 +69,7 @@ def track_file(file, base_dir, host):
 
 def main(arguments):
     hostname = arguments['hostname']
-    base_dir = settings['storage']['file_dir']
+    base_dir = shared.settings['storage']['file_dir']
     # Recursively Get All Files and Folder below BASE_DIR
     files = glob.glob('{}/**'.format(base_dir), recursive=True)
     files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
@@ -78,7 +77,7 @@ def main(arguments):
     # Files that are already tracked
     tracked_files = [
         pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
-        for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
+        for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
         if f.value['host'] == hostname
     ]
     untracked_files = set(files) - set(tracked_files)

uncloud/host/main.py

@@ -6,7 +6,6 @@ from uuid import uuid4
 
 from uncloud.common.request import RequestEntry, RequestType
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 from uncloud.common.vm import VMStatus
 from uncloud.vmm import VMM
 from os.path import join as join_path
@@ -36,7 +35,7 @@ def maintenance(host):
         if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
             logger.debug('VM {} is running on {}'.format(vm_uuid, host))
             vm = shared.vm_pool.get(
-                join_path(settings['etcd']['vm_prefix'], vm_uuid)
+                join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
             )
             vm.status = VMStatus.running
             vm.vnc_socket = vmm.get_vnc(vm_uuid)
@@ -52,7 +51,7 @@ def main(arguments):
     # Does not yet exist, create it
     if not host:
         host_key = join_path(
-            settings['etcd']['host_prefix'], uuid4().hex
+            shared.settings['etcd']['host_prefix'], uuid4().hex
         )
         host_entry = {
             'specs': '',
@@ -80,9 +79,9 @@ def main(arguments):
     # get prefix until either success or deamon death comes.
     while True:
         for events_iterator in [
-            shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
+            shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
                                           raise_exception=False),
-            shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
+            shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
                                             raise_exception=False)
         ]:
             for request_event in events_iterator:
@@ -95,7 +94,7 @@ def main(arguments):
 
                     shared.request_pool.client.client.delete(request_event.key)
                     vm_entry = shared.etcd_client.get(
-                        join_path(settings['etcd']['vm_prefix'], request_event.uuid)
+                        join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid)
                     )
 
                     logger.debug('VM hostname: {}'.format(vm_entry.value))

uncloud/host/virtualmachine.py

@@ -17,7 +17,6 @@ from uncloud.common.network import create_dev, delete_network_interface
 from uncloud.common.schemas import VMSchema, NetworkSchema
 from uncloud.host import logger
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 from uncloud.vmm import VMM
 
 from marshmallow import ValidationError
@@ -91,7 +90,7 @@ class VM:
                         self.vmm.socket_dir, self.uuid
                     ),
                     destination_host_key=destination_host_key,  # Where source host transfer VM
-                    request_prefix=settings["etcd"]["request_prefix"],
+                    request_prefix=shared.settings["etcd"]["request_prefix"],
                 )
                 shared.request_pool.put(r)
             else:
@@ -119,7 +118,7 @@ class VM:
             network_name, mac, tap = network_mac_and_tap
 
             _key = os.path.join(
-                settings["etcd"]["network_prefix"],
+                shared.settings["etcd"]["network_prefix"],
                 self.vm["owner"],
                 network_name,
             )
@@ -133,13 +132,13 @@ class VM:
             if network["type"] == "vxlan":
                 tap = create_vxlan_br_tap(
                     _id=network["id"],
-                    _dev=settings["network"]["vxlan_phy_dev"],
+                    _dev=shared.settings["network"]["vxlan_phy_dev"],
                     tap_id=tap,
                     ip=network["ipv6"],
                 )
 
                 all_networks = shared.etcd_client.get_prefix(
-                    settings["etcd"]["network_prefix"],
+                    shared.settings["etcd"]["network_prefix"],
                     value_in_json=True,
                 )
 
@@ -229,7 +228,7 @@ class VM:
 def resolve_network(network_name, network_owner):
     network = shared.etcd_client.get(
         join_path(
-            settings["etcd"]["network_prefix"],
+            shared.settings["etcd"]["network_prefix"],
             network_owner,
             network_name,
         ),

uncloud/imagescanner/main.py

@@ -4,7 +4,6 @@ import argparse
 import subprocess as sp
 
 from os.path import join as join_path
-from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 from uncloud.imagescanner import logger
 
@@ -33,7 +32,7 @@ def qemu_img_type(path):
 def main(arguments):
     # We want to get images entries that requests images to be created
     images = shared.etcd_client.get_prefix(
-        settings["etcd"]["image_prefix"], value_in_json=True
+        shared.settings["etcd"]["image_prefix"], value_in_json=True
     )
     images_to_be_created = list(
         filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
@@ -46,13 +45,13 @@ def main(arguments):
             image_filename = image.value["filename"]
             image_store_name = image.value["store_name"]
             image_full_path = join_path(