From c3b42aabc626f4cca1617dfcba9b52fbf587502a Mon Sep 17 00:00:00 2001 From: Ahmed Bilal <49-ahmedbilal@users.noreply.code.ungleich.ch> Date: Mon, 13 Jan 2020 05:57:41 +0100 Subject: [PATCH] Added --conf-dir, --etcd-{host,port,ca_cert,cert_cert,cert_key} parameters to cli and settings is now accessbile through uncloud.shared.shared.settings --- scripts/uncloud | 59 ++++++++++++++++++----------- uncloud/api/common_fields.py | 7 +--- uncloud/api/create_image_store.py | 3 +- uncloud/api/helper.py | 18 ++++----- uncloud/api/main.py | 61 +++++++++++++++--------------- uncloud/api/schemas.py | 13 +++---- uncloud/cli/helper.py | 29 ++++++-------- uncloud/common/cli.py | 26 +++++++++++++ uncloud/common/settings.py | 19 ++++++---- uncloud/common/shared.py | 22 +++++------ uncloud/common/storage_handlers.py | 15 ++++---- uncloud/configure/main.py | 11 +++--- uncloud/filescanner/main.py | 7 ++-- uncloud/host/main.py | 11 +++--- uncloud/host/virtualmachine.py | 11 +++--- uncloud/imagescanner/main.py | 7 ++-- uncloud/metadata/main.py | 3 +- uncloud/scheduler/helper.py | 3 +- uncloud/scheduler/main.py | 5 +-- 19 files changed, 176 insertions(+), 154 deletions(-) create mode 100644 uncloud/common/cli.py diff --git a/scripts/uncloud b/scripts/uncloud index a6e61aa..1a6483b 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -3,14 +3,13 @@ import logging import sys import importlib import argparse +import os +from etcd3.exceptions import ConnectionFailedError + +from uncloud.common import settings from uncloud import UncloudException - -# the components that use etcd -ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] - -ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('cli') +from uncloud.common.cli import resolve_otp_credentials def exception_hook(exc_type, exc_value, exc_traceback): @@ -22,6 +21,13 @@ def exception_hook(exc_type, exc_value, exc_traceback): sys.excepthook = exception_hook +# the components that use etcd +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('cli') + + if __name__ == '__main__': # Setting up root logger logger = logging.getLogger() @@ -31,15 +37,13 @@ if __name__ == '__main__': subparsers = arg_parser.add_subparsers(dest='command') parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument('--debug', '-d', - action='store_true', - default=False, + parent_parser.add_argument('--debug', '-d', action='store_true', default=False, help='More verbose logging') - parent_parser.add_argument('--conf-dir', '-c', - help='Configuration directory') + parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory', + default=os.path.expanduser('~/uncloud')) etcd_parser = argparse.ArgumentParser(add_help=False) - etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-host', dest='etcd_url') etcd_parser.add_argument('--etcd-port') etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') @@ -54,25 +58,36 @@ if __name__ == '__main__': else: subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - args = arg_parser.parse_args() - if not args.command: + arguments = vars(arg_parser.parse_args()) + etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value] + etcd_arguments = { + 'etcd': { + key.replace('etcd_', ''): arguments[key] + for key in etcd_arguments + } + } + if not arguments['command']: arg_parser.print_help() else: - arguments = vars(args) + # Initializing Settings and resolving otp_credentials + # It is neccessary to resolve_otp_credentials after argument parsing is done because + # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and + # providing the default values for --name, --realm and --seed arguments from the values + # we read from file. But, now we are asking user about where the config file lives. So, + # to providing default value is not possible before parsing arguments. So, we are doing + # it after.. + settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) + resolve_otp_credentials(arguments) + name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') - # If the component requires etcd3, we import it and catch the - # etcd3.exceptions.ConnectionFailedError - if name in ETCD_COMPONENTS: - import etcd3 - try: main(arguments) except UncloudException as err: logger.error(err) - sys.exit(1) + except ConnectionFailedError: + logger.error('Cannot connect to etcd') except Exception as err: logger.exception(err) - sys.exit(1) diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py index d1fcb64..ba9fb37 100755 --- a/uncloud/api/common_fields.py +++ b/uncloud/api/common_fields.py @@ -1,7 +1,6 @@ import os from uncloud.common.shared import shared -from uncloud.common.settings import settings class Optional: @@ -54,9 +53,7 @@ class VmUUIDField(Field): def vm_uuid_validation(self): r = shared.etcd_client.get( - os.path.join(settings["etcd"]["vm_prefix"], self.uuid) + os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid) ) if not r: - self.add_error( - "VM with uuid {} does not exists".format(self.uuid) - ) + self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 1040e97..90e0f92 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -4,7 +4,6 @@ import os from uuid import uuid4 from uncloud.common.shared import shared -from uncloud.common.settings import settings data = { 'is_public': True, @@ -15,6 +14,6 @@ data = { } shared.etcd_client.put( - os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), + os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data), ) diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index 0805280..8ceb3a6 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -7,7 +7,6 @@ import requests from pyotp import TOTP from uncloud.common.shared import shared -from uncloud.common.settings import settings logger = logging.getLogger(__name__) @@ -15,9 +14,9 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": settings["otp"]["auth_name"], - "auth_token": TOTP(settings["otp"]["auth_seed"]).now(), - "auth_realm": settings["otp"]["auth_realm"], + "auth_name": shared.settings["otp"]["auth_name"], + "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), + "auth_realm": shared.settings["otp"]["auth_realm"], "name": name, "realm": realm, "token": token, @@ -25,13 +24,13 @@ def check_otp(name, realm, token): except binascii.Error as err: logger.error( "Cannot compute OTP for seed: {}".format( - settings["otp"]["auth_seed"] + shared.settings["otp"]["auth_seed"] ) ) return 400 response = requests.post( - settings["otp"]["verification_controller_url"], json=data + shared.settings["otp"]["verification_controller_url"], json=data ) return response.status_code @@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client): ) images = etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) # Try to find image with name == image_name and store_name == store_name @@ -111,9 +110,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix): lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) prefix = ipaddress.IPv6Address(prefix) return str(prefix + int(lower_part)) + diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 34e1dd1..73e8e21 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -15,9 +15,8 @@ from uncloud.common.shared import shared from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType -from uncloud.common.settings import settings -from . import schemas -from .helper import generate_mac, mac2ipv6 +from uncloud.api import schemas +from uncloud.api.helper import generate_mac, mac2ipv6 from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -50,7 +49,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) + vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) specs = { 'cpu': validator.specs['cpu'], 'ram': validator.specs['ram'], @@ -60,7 +59,7 @@ class CreateVM(Resource): macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['tap_counter'] + shared.etcd_client, shared.settings['etcd']['tap_counter'] ) for _ in range(len(data['network'])) ] @@ -84,7 +83,7 @@ class CreateVM(Resource): r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) @@ -99,7 +98,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) vm_value = vm.value.copy() vm_value['ip'] = [] @@ -107,7 +106,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( join_path( - settings['etcd']['network_prefix'], + shared.settings['etcd']['network_prefix'], data['name'], network_name, ), @@ -130,7 +129,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = shared.etcd_client.get( - join_path(settings['etcd']['file_prefix'], data['uuid']) + join_path(shared.settings['etcd']['file_prefix'], data['uuid']) ) file_entry_value = json.loads(file_entry.value) @@ -144,7 +143,7 @@ class CreateImage(Resource): } shared.etcd_client.put( join_path( - settings['etcd']['image_prefix'], data['uuid'] + shared.settings['etcd']['image_prefix'], data['uuid'] ), json.dumps(image_entry_json), ) @@ -157,7 +156,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = shared.etcd_client.get_prefix( - settings['etcd']['image_prefix'], value_in_json=True + shared.settings['etcd']['image_prefix'], value_in_json=True ) r = {'images': []} for image in images: @@ -178,7 +177,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) action = data['action'] @@ -208,7 +207,7 @@ class VMAction(Resource): type='{}VM'.format(action.title()), uuid=data['uuid'], hostname=vm_entry.hostname, - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( @@ -231,10 +230,10 @@ class VMMigration(Resource): type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( - settings['etcd']['host_prefix'], + shared.settings['etcd']['host_prefix'], validator.destination.value, ), - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) @@ -254,7 +253,7 @@ class ListUserVM(Resource): if validator.is_valid(): vms = shared.etcd_client.get_prefix( - settings['etcd']['vm_prefix'], value_in_json=True + shared.settings['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter( @@ -287,7 +286,7 @@ class ListUserFiles(Resource): if validator.is_valid(): files = shared.etcd_client.get_prefix( - settings['etcd']['file_prefix'], value_in_json=True + shared.settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] user_files = [f for f in files if f.value['owner'] == data['name']] @@ -312,7 +311,7 @@ class CreateHost(Resource): validator = schemas.CreateHostSchema(data) if validator.is_valid(): host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': data['specs'], @@ -354,7 +353,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -372,7 +371,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -405,7 +404,7 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -439,7 +438,7 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -471,23 +470,23 @@ class CreateNetwork(Resource): network_entry = { 'id': counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['vxlan_counter'] + shared.etcd_client, shared.settings['etcd']['vxlan_counter'] ), 'type': data['type'], } if validator.user.value: try: nb = pynetbox.api( - url=settings['netbox']['url'], - token=settings['netbox']['token'], + url=shared.settings['netbox']['url'], + token=shared.settings['netbox']['token'], ) nb_prefix = nb.ipam.prefixes.get( - prefix=settings['network']['prefix'] + prefix=shared.settings['network']['prefix'] ) prefix = nb_prefix.available_prefixes.create( data={ 'prefix_length': int( - settings['network']['prefix_length'] + shared.settings['network']['prefix_length'] ), 'description': '{}\'s network "{}"'.format( data['name'], data['network_name'] @@ -506,7 +505,7 @@ class CreateNetwork(Resource): network_entry['ipv6'] = 'fd00::/64' network_key = join_path( - settings['etcd']['network_prefix'], + shared.settings['etcd']['network_prefix'], data['name'], data['network_name'], ) @@ -526,7 +525,7 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - settings['etcd']['network_prefix'], data['name'] + shared.settings['etcd']['network_prefix'], data['name'] ) networks = shared.etcd_client.get_prefix( prefix, value_in_json=True @@ -570,7 +569,7 @@ def main(arguments): try: image_stores = list( shared.etcd_client.get_prefix( - settings['etcd']['image_store_prefix'], value_in_json=True + shared.settings['etcd']['image_store_prefix'], value_in_json=True ) ) except KeyError: @@ -590,7 +589,7 @@ def main(arguments): # shared.etcd_client.put( # join_path( - # settings['etcd']['image_store_prefix'], uuid4().hex + # shared.settings['etcd']['image_store_prefix'], uuid4().hex # ), # json.dumps(data), # ) diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index e4de9a8..87f20c9 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -22,7 +22,6 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from uncloud.common.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema): def file_uuid_validation(self): file_entry = shared.etcd_client.get( os.path.join( - settings["etcd"]["file_prefix"], self.uuid.value + shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value ) ) if file_entry is None: @@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema): def image_store_name_validation(self): image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"] + shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] ) ) @@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema): for net in _network: network = shared.etcd_client.get( os.path.join( - settings["etcd"]["network_prefix"], + shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, net, ), @@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema): self.add_error("Can't migrate non-running VM") if vm.hostname == os.path.join( - settings["etcd"]["host_prefix"], self.destination.value + shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value ): self.add_error( "Destination host couldn't be same as Source Host" @@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - print(self.name.value, self.network_name.value) - key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) - print(key) + key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) network = shared.etcd_client.get(key, value_in_json=True) if network: self.add_error( diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py index 3c63073..51a4355 100644 --- a/uncloud/cli/helper.py +++ b/uncloud/cli/helper.py @@ -5,23 +5,14 @@ import binascii from pyotp import TOTP from os.path import join as join_path -from uncloud.common.settings import settings +from uncloud.common.shared import shared def get_otp_parser(): otp_parser = argparse.ArgumentParser('otp') - try: - name = settings['client']['name'] - realm = settings['client']['realm'] - seed = settings['client']['seed'] - except Exception: - otp_parser.add_argument('--name', required=True) - otp_parser.add_argument('--realm', required=True) - otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED') - else: - otp_parser.add_argument('--name', default=name) - otp_parser.add_argument('--realm', default=realm) - otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED') + otp_parser.add_argument('--name') + otp_parser.add_argument('--realm') + otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') return otp_parser @@ -34,11 +25,15 @@ def load_dump_pretty(content): def make_request(*args, data=None, request_method=requests.post): - r = request_method(join_path(settings['client']['api_server'], *args), json=data) try: - print(load_dump_pretty(r.content)) - except Exception: - print('Error occurred while getting output from api server.') + r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) + except requests.exceptions.RequestException: + print('Error occurred while connecting to API server.') + else: + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') def get_token(seed): diff --git a/uncloud/common/cli.py b/uncloud/common/cli.py new file mode 100644 index 0000000..3d3c248 --- /dev/null +++ b/uncloud/common/cli.py @@ -0,0 +1,26 @@ +from uncloud.common.shared import shared +from pyotp import TOTP + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except Exception: + raise Exception('Invalid seed') + else: + return token + + +def resolve_otp_credentials(kwargs): + d = { + 'name': shared.settings['client']['name'], + 'realm': shared.settings['client']['realm'], + 'token': get_token(shared.settings['client']['seed']) + } + + for k, v in d.items(): + if k in kwargs and kwargs[k] is None: + kwargs.update({k: v}) + + return d diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 0d524a7..8503f42 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper from os.path import join as join_path logger = logging.getLogger(__name__) +settings = None class CustomConfigParser(configparser.RawConfigParser): @@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): - def __init__(self): + def __init__(self, conf_dir, seed_value=None): conf_name = 'uncloud.conf' - conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/')) self.config_file = join_path(conf_dir, conf_name) # this is used to cache config from etcd for 1 minutes. Without this we @@ -38,15 +38,19 @@ class Settings(object): self.config_parser.add_section('etcd') self.config_parser.set('etcd', 'base_prefix', '/') - try: + if os.access(self.config_file, os.R_OK): self.config_parser.read(self.config_file) - except Exception as err: - logger.error('%s', err) - + else: + raise FileNotFoundError('Config file %s not found!', self.config_file) self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') self.read_internal_values() + if seed_value is None: + seed_value = dict() + + self.config_parser.read_dict(seed_value) + def get_etcd_client(self): args = tuple() try: @@ -128,4 +132,5 @@ class Settings(object): return self.config_parser[key] -settings = Settings() +def get_settings(): + return settings diff --git a/uncloud/common/shared.py b/uncloud/common/shared.py index 918dd0c..aea7cbc 100644 --- a/uncloud/common/shared.py +++ b/uncloud/common/shared.py @@ -1,34 +1,34 @@ -from uncloud.common.settings import settings +from uncloud.common.settings import get_settings from uncloud.common.vm import VmPool from uncloud.common.host import HostPool from uncloud.common.request import RequestPool -from uncloud.common.storage_handlers import get_storage_handler +import uncloud.common.storage_handlers as storage_handlers class Shared: + @property + def settings(self): + return get_settings() + @property def etcd_client(self): - return settings.get_etcd_client() + return self.settings.get_etcd_client() @property def host_pool(self): - return HostPool( - self.etcd_client, settings["etcd"]["host_prefix"] - ) + return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) @property def vm_pool(self): - return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) + return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) @property def request_pool(self): - return RequestPool( - self.etcd_client, settings["etcd"]["request_prefix"] - ) + return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) @property def storage_handler(self): - return get_storage_handler() + return storage_handlers.get_storage_handler() shared = Shared() diff --git a/uncloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py index 6f9b29e..58c2dc2 100644 --- a/uncloud/common/storage_handlers.py +++ b/uncloud/common/storage_handlers.py @@ -6,8 +6,7 @@ import stat from abc import ABC from . import logger from os.path import join as join_path - -from uncloud.common.settings import settings as config +import uncloud.common.shared as shared class ImageStorageHandler(ABC): @@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): def get_storage_handler(): - __storage_backend = config["storage"]["storage_backend"] + __storage_backend = shared.shared.settings["storage"]["storage_backend"] if __storage_backend == "filesystem": return FileSystemBasedImageStorageHandler( - vm_base=config["storage"]["vm_dir"], - image_base=config["storage"]["image_dir"], + vm_base=shared.shared.settings["storage"]["vm_dir"], + image_base=shared.shared.settings["storage"]["image_dir"], ) elif __storage_backend == "ceph": return CEPHBasedImageStorageHandler( - vm_base=config["storage"]["ceph_vm_pool"], - image_base=config["storage"]["ceph_image_pool"], + vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], + image_base=shared.shared.settings["storage"]["ceph_image_pool"], ) else: - raise Exception("Unknown Image Storage Handler") + raise Exception("Unknown Image Storage Handler") \ No newline at end of file diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index e190460..87f5752 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,7 +1,6 @@ import os import argparse -from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('configure', add_help=False) @@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True) def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) + uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) + shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) -def main(**kwargs): - subcommand = kwargs.pop('subcommand') +def main(arguments): + subcommand = arguments['subcommand'] if not subcommand: arg_parser.print_help() else: - update_config(subcommand, kwargs) + update_config(subcommand, arguments) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index c5660dd..046f915 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -9,7 +9,6 @@ import bitmath from uuid import uuid4 from . import logger -from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('filescanner', add_help=False) @@ -53,7 +52,7 @@ def track_file(file, base_dir, host): file_path = file_path.relative_to(owner) creation_date = time.ctime(os.stat(file_str).st_ctime) - entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) + entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) entry_value = { 'filename': str(file_path), 'owner': owner, @@ -70,7 +69,7 @@ def track_file(file, base_dir, host): def main(arguments): hostname = arguments['hostname'] - base_dir = settings['storage']['file_dir'] + base_dir = shared.settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] @@ -78,7 +77,7 @@ def main(arguments): # Files that are already tracked tracked_files = [ pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) - for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) + for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) if f.value['host'] == hostname ] untracked_files = set(files) - set(tracked_files) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index ccffd77..f680991 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -6,7 +6,6 @@ from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared -from uncloud.common.settings import settings from uncloud.common.vm import VMStatus from uncloud.vmm import VMM from os.path import join as join_path @@ -36,7 +35,7 @@ def maintenance(host): if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], vm_uuid) + join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) ) vm.status = VMStatus.running vm.vnc_socket = vmm.get_vnc(vm_uuid) @@ -52,7 +51,7 @@ def main(arguments): # Does not yet exist, create it if not host: host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': '', @@ -80,9 +79,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for events_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False) ]: for request_event in events_iterator: @@ -95,7 +94,7 @@ def main(arguments): shared.request_pool.client.client.delete(request_event.key) vm_entry = shared.etcd_client.get( - join_path(settings['etcd']['vm_prefix'], request_event.uuid) + join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) ) logger.debug('VM hostname: {}'.format(vm_entry.value)) diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index 2f6a5e3..a592efc 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -17,7 +17,6 @@ from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger from uncloud.common.shared import shared -from uncloud.common.settings import settings from uncloud.vmm import VMM from marshmallow import ValidationError @@ -91,7 +90,7 @@ class VM: self.vmm.socket_dir, self.uuid ), destination_host_key=destination_host_key, # Where source host transfer VM - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) else: @@ -119,7 +118,7 @@ class VM: network_name, mac, tap = network_mac_and_tap _key = os.path.join( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], self.vm["owner"], network_name, ) @@ -133,13 +132,13 @@ class VM: if network["type"] == "vxlan": tap = create_vxlan_br_tap( _id=network["id"], - _dev=settings["network"]["vxlan_phy_dev"], + _dev=shared.settings["network"]["vxlan_phy_dev"], tap_id=tap, ip=network["ipv6"], ) all_networks = shared.etcd_client.get_prefix( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], value_in_json=True, ) @@ -229,7 +228,7 @@ class VM: def resolve_network(network_name, network_owner): network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], network_owner, network_name, ), diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index 1803213..ee9da2e 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -4,7 +4,6 @@ import argparse import subprocess as sp from os.path import join as join_path -from uncloud.common.settings import settings from uncloud.common.shared import shared from uncloud.imagescanner import logger @@ -33,7 +32,7 @@ def qemu_img_type(path): def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) images_to_be_created = list( filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) @@ -46,13 +45,13 @@ def main(arguments): image_filename = image.value["filename"] image_store_name = image.value["store_name"] image_full_path = join_path( - settings["storage"]["file_dir"], + shared.settings["storage"]["file_dir"], image_owner, image_filename, ) image_stores = shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], + shared.settings["etcd"]["image_store_prefix"], value_in_json=True, ) user_image_store = next( diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index c47364e..374260e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -5,7 +5,6 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from uncloud.common.settings import settings from uncloud.common.shared import shared app = Flask(__name__) @@ -74,7 +73,7 @@ class Root(Resource): ) else: etcd_key = os.path.join( - settings["etcd"]["user_prefix"], + shared.settings["etcd"]["user_prefix"], data.value["owner_realm"], data.value["owner"], "key", diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index 108d126..79db322 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -7,7 +7,6 @@ from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from uncloud.common.settings import settings def accumulated_specs(vms_specs): @@ -130,7 +129,7 @@ def assign_host(vm): type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index c25700b..38c07bf 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -6,7 +6,6 @@ import argparse -from uncloud.common.settings import settings from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared from uncloud.scheduler import logger @@ -24,9 +23,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for request_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), ]: for request_event in request_iterator: