From 71279a968f649f203f6b5577f495594176ca818d Mon Sep 17 00:00:00 2001 From: meow Date: Sat, 14 Dec 2019 20:23:31 +0500 Subject: [PATCH] Fix issues in naming and few other things --- conf/ucloud.conf | 75 ++++++++-------- scripts/ucloud | 14 ++- setup.py | 9 +- ucloud/api/common_fields.py | 2 +- ucloud/api/create_image_store.py | 2 +- ucloud/api/helper.py | 21 +++-- ucloud/api/main.py | 93 +++++++++++--------- ucloud/api/schemas.py | 13 +-- ucloud/common/classes.py | 2 +- ucloud/common/counters.py | 2 +- ucloud/common/etcd_wrapper.py | 74 ++++++++++++++++ ucloud/common/helpers.py | 20 +---- ucloud/common/request.py | 3 +- ucloud/config.py | 144 +++++++------------------------ ucloud/filescanner/main.py | 24 +++--- ucloud/host/main.py | 19 ++-- ucloud/host/virtualmachine.py | 14 +-- ucloud/imagescanner/main.py | 11 +-- ucloud/metadata/main.py | 3 +- ucloud/scheduler/helper.py | 2 +- ucloud/scheduler/main.py | 8 +- 21 files changed, 274 insertions(+), 281 deletions(-) create mode 100644 ucloud/common/etcd_wrapper.py diff --git a/conf/ucloud.conf b/conf/ucloud.conf index a98250a..222cc4f 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -1,60 +1,53 @@ -# This section contains default values for all other sections -# -[DEFAULT] - -AUTH_NAME = "replace me" -AUTH_SEED = "replace me" -AUTH_REALM = "replace me" - -NETWORK_PREFIX = moo - -OTP_VERIFY_ENDPOINT = verify/ - -[api] -NETWORK_PREFIX = foo +[otp] +server = https://otp.ungleich.ch/ungleichotp/ +verify_endpoint = verify/ +auth_name = replace_me +auth_realm = replace_me +auth_seed = replace_me [network] -PREFIX_LENGTH = 64 -PREFIX = 2001:db8::/48 +prefix_length = 64 +prefix = 2001:db8::/48 +vxlan_phy_dev = eno1 [netbox] -NETBOX_URL = https://replace-me.example.com -NETBOX_TOKEN = replace me +url = https://replace-me.example.com +token = replace_me [etcd] -ETCD_URL = localhost -ETCD_PORT = 2379 +url = localhost +port = 2379 -CA_CERT -CERT_CERT -CERT_KEY +ca_cert +cert_cert +cert_key - -FILE_PREFIX = files -HOST_PREFIx = hosts -IMAGE_PREFIX = images -IMAGE_STORE_PREFIX = imagestore - -NETWORK_PREFIX = networks -REQUEST_PREFIX = requests -USER_PREFIX = users -VM_PREFIX = vms +file_prefix = /files/ +host_prefix = /hosts/ +image_prefix = /images/ +image_store_prefix = /imagestore/ +network_prefix = /networks/ +request_prefix = /requests/ +user_prefix = /users/ +vm_prefix = /vms/ [storage] + #values = filesystem, ceph -STORAGE_BACKEND = +backend = filesystem # if STORAGE_BACKEND = filesystem -VM_DIR = -IMG_DIR = +vm_dir = /var/lib/ucloud/vms +image_dir = /var/lib/ucloud/images # if STORAGE_BACKEND = ceph -CEPH_VM_POOL = -CEPH_IMG_POOL = +ceph_vm_pool = ssd +ceph_image_pool = ssd # Importing uploaded files -FILE_DIR = /var/lib/ucloud/files +file_dir = /var/lib/ucloud/files +# For Migrating VMs over ssh/tcp [ssh] -SSH_USERNAME = -SSH_PRIVATEKEY = \ No newline at end of file +username +private_key_path \ No newline at end of file diff --git a/scripts/ucloud b/scripts/ucloud index 0780fb4..2da1094 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -5,11 +5,17 @@ import logging import importlib import sys import os +import multiprocessing as mp + COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata'] if __name__ == "__main__": - log = logging.getLogger("ucloud") + logging.basicConfig(level=logging.DEBUG, + format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s', + filename='/var/log/ucloud.log', filemode='a') + + logger = logging.getLogger("ucloud") arg_parser = argparse.ArgumentParser(prog='ucloud', description='Open Source Cloud Management Software') @@ -22,12 +28,12 @@ if __name__ == "__main__": os.environ['UCLOUD_CONF_DIR'] = args.conf_dir try: + mp.set_start_method('spawn') name = args.component mod = importlib.import_module("ucloud.{}.main".format(name)) main = getattr(mod, "main") - - main() + main(*args.component_args) except Exception as e: - logging.exception(e) + logger.exception(e) print(e) diff --git a/setup.py b/setup.py index 14dffb7..a7ddbe6 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ try: version = ucloud.version.VERSION except: import subprocess - c = subprocess.run(["git", "describe"], capture_output=True) - version = c.stdout.decode("utf-8") + c = subprocess.check_output(['git', 'describe']) + version = c.decode("utf-8").strip() setup(name='ucloud', @@ -28,8 +28,7 @@ setup(name='ucloud', packages=find_packages(), install_requires=[ 'requests', - 'python-decouple', - 'flask', + 'Flask', 'flask-restful', 'bitmath', 'pyotp', @@ -37,8 +36,8 @@ setup(name='ucloud', 'sphinx', 'pynetbox', 'sphinx-rtd-theme', - 'etcd3_wrapper @ https://code.ungleich.ch/ungleich-public/etcd3_wrapper/repository/master/archive.tar.gz#egg=etcd3_wrapper', 'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3', ], scripts=['scripts/ucloud'], + data_files=[('/etc/ucloud/', ['conf/ucloud.conf'])], zip_safe=False) diff --git a/ucloud/api/common_fields.py b/ucloud/api/common_fields.py index 1ceb1b0..cf86283 100755 --- a/ucloud/api/common_fields.py +++ b/ucloud/api/common_fields.py @@ -47,6 +47,6 @@ class VmUUIDField(Field): self.validation = self.vm_uuid_validation def vm_uuid_validation(self): - r = etcd_client.get(os.path.join(config['api']['VM_PREFIX'], self.uuid)) + r = etcd_client.get(os.path.join(config['etcd']['vm_prefix'], self.uuid)) if not r: self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/ucloud/api/create_image_store.py b/ucloud/api/create_image_store.py index 259b9c8..9023bd6 100755 --- a/ucloud/api/create_image_store.py +++ b/ucloud/api/create_image_store.py @@ -13,4 +13,4 @@ data = { "attributes": {"list": [], "key": [], "pool": "images"}, } -etcd_client.put(os.path.join(config['api']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data)) +etcd_client.put(os.path.join(config['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index 6735f05..2dfb7de 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -2,7 +2,7 @@ import binascii import ipaddress import random import subprocess as sp - +import logging import requests from pyotp import TOTP @@ -10,23 +10,28 @@ from pyotp import TOTP from ucloud.config import vm_pool, config +logger = logging.getLogger("ucloud.api.helper") + def check_otp(name, realm, token): try: data = { - "auth_name": config['api']["AUTH_NAME"], - "auth_token": TOTP(config['api']["AUTH_SEED"]).now(), - "auth_realm": config['api']["AUTH_REALM"], + "auth_name": config['otp']['auth_name'], + "auth_token": TOTP(config['otp']['auth_seed']).now(), + "auth_realm": config['otp']['auth_realm'], "name": name, "realm": realm, "token": token, } - except binascii.Error: + except binascii.Error as err: + logger.error( + "Cannot compute OTP for seed: {}".format(config['otp']['auth_seed']) + ) return 400 response = requests.post( "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( - OTP_SERVER=config['api']["OTP_SERVER"], - OTP_VERIFY_ENDPOINT=config['api']["OTP_VERIFY_ENDPOINT"] + OTP_SERVER=config['otp']['server'], + OTP_VERIFY_ENDPOINT=config['otp']['verify_endpoint'] ), json=data, ) @@ -80,7 +85,7 @@ def resolve_image_name(name, etcd_client): except Exception: raise ValueError("Image name not in correct format i.e {store_name}:{image_name}") - images = etcd_client.get_prefix(config['api']['IMAGE_PREFIX'], value_in_json=True) + images = etcd_client.get_prefix(config['etcd']['image_prefix'], value_in_json=True) # Try to find image with name == image_name and store_name == store_name try: diff --git a/ucloud/api/main.py b/ucloud/api/main.py index bbda7e9..0e202d8 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -10,7 +10,10 @@ from flask_restful import Resource, Api from ucloud.common import counters from ucloud.common.vm import VMStatus from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, config, image_storage_handler) +from ucloud.config import ( + etcd_client, request_pool, vm_pool, + host_pool, config, image_storage_handler +) from . import schemas from .helper import generate_mac, mac2ipv6 from . import logger @@ -28,7 +31,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(config['etcd']["VM_PREFIX"], vm_uuid) + vm_key = join_path(config['etcd']['vm_prefix'], vm_uuid) specs = { "cpu": validator.specs["cpu"], "ram": validator.specs["ram"], @@ -56,7 +59,7 @@ class CreateVM(Resource): # Create ScheduleVM Request r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=config['etcd']["REQUEST_PREFIX"] + request_prefix=config['etcd']['request_prefix'] ) request_pool.put(r) @@ -71,7 +74,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = vm_pool.get( - join_path(config['etcd']["VM_PREFIX"], data["uuid"]) + join_path(config['etcd']['vm_prefix'], data["uuid"]) ) vm_value = vm.value.copy() vm_value["ip"] = [] @@ -79,7 +82,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = etcd_client.get( join_path( - config['etcd']["NETWORK_PREFIX"], + config['etcd']['network_prefix'], data["name"], network_name, ), @@ -100,7 +103,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = etcd_client.get( - join_path(config['etcd']["FILE_PREFIX"], data["uuid"]) + join_path(config['etcd']['file_prefix'], data["uuid"]) ) file_entry_value = json.loads(file_entry.value) @@ -113,7 +116,7 @@ class CreateImage(Resource): "visibility": "public", } etcd_client.put( - join_path(config['etcd']["IMAGE_PREFIX"], data["uuid"]), + join_path(config['etcd']['image_prefix'], data["uuid"]), json.dumps(image_entry_json), ) @@ -125,7 +128,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = etcd_client.get_prefix( - config['etcd']["IMAGE_PREFIX"], value_in_json=True + config['etcd']['image_prefix'], value_in_json=True ) r = { "images": [] @@ -148,7 +151,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = vm_pool.get( - join_path(config['etcd']["VM_PREFIX"], data["uuid"]) + join_path(config['etcd']['vm_prefix'], data["uuid"]) ) action = data["action"] @@ -172,7 +175,7 @@ class VMAction(Resource): type="{}VM".format(action.title()), uuid=data["uuid"], hostname=vm_entry.hostname, - request_prefix=config['etcd']["REQUEST_PREFIX"] + request_prefix=config['etcd']['request_prefix'] ) request_pool.put(r) return {"message": "VM {} Queued".format(action.title())}, 200 @@ -193,10 +196,10 @@ class VMMigration(Resource): type=RequestType.ScheduleVM, uuid=vm.uuid, destination=join_path( - config['etcd']["HOST_PREFIX"], validator.destination.value + config['etcd']['host_prefix'], validator.destination.value ), migration=True, - request_prefix=config['etcd']["REQUEST_PREFIX"] + request_prefix=config['etcd']['request_prefix'] ) request_pool.put(r) return {"message": "VM Migration Initialization Queued"}, 200 @@ -212,7 +215,7 @@ class ListUserVM(Resource): if validator.is_valid(): vms = etcd_client.get_prefix( - config['etcd']["VM_PREFIX"], value_in_json=True + config['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) @@ -246,7 +249,7 @@ class ListUserFiles(Resource): if validator.is_valid(): files = etcd_client.get_prefix( - config['etcd']["FILE_PREFIX"], value_in_json=True + config['etcd']['file_prefix'], value_in_json=True ) return_files = [] user_files = list( @@ -270,7 +273,7 @@ class CreateHost(Resource): data = request.json validator = schemas.CreateHostSchema(data) if validator.is_valid(): - host_key = join_path(config['etcd']["HOST_PREFIX"], uuid4().hex) + host_key = join_path(config['etcd']['host_prefix'], uuid4().hex) host_entry = { "specs": data["specs"], "hostname": data["hostname"], @@ -309,7 +312,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - config['etcd']['USER_PREFIX'], + config['etcd']['user_prefix'], data["realm"], data["name"], "key", @@ -326,7 +329,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']['USER_PREFIX'], + config['etcd']['user_prefix'], data["realm"], data["name"], "key", @@ -355,7 +358,7 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']["USER_PREFIX"], + config['etcd']['user_prefix'], data["realm"], data["name"], "key", @@ -385,7 +388,7 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']["USER_PREFIX"], + config['etcd']['user_prefix'], data["realm"], data["name"], "key", @@ -420,31 +423,35 @@ class CreateNetwork(Resource): "type": data["type"], } if validator.user.value: - nb = pynetbox.api( - url=config['netbox']["NETBOX_URL"], - token=config['netbox']["NETBOX_TOKEN"], - ) - nb_prefix = nb.ipam.prefixes.get( - prefix=config['network']["PREFIX"] - ) - - prefix = nb_prefix.available_prefixes.create( - data={ - "prefix_length": config['network']["PREFIX_LENGTH"], - "description": '{}\'s network "{}"'.format( - data["name"], data["network_name"] - ), - "is_pool": True, - } - ) - network_entry["ipv6"] = prefix["prefix"] + try: + nb = pynetbox.api( + url=config['netbox']['url'], + token=config['netbox']['token'], + ) + nb_prefix = nb.ipam.prefixes.get( + prefix=config['network']['prefix'] + ) + prefix = nb_prefix.available_prefixes.create( + data={ + "prefix_length": int(config['network']['prefix_length']), + "description": '{}\'s network "{}"'.format( + data["name"], data["network_name"] + ), + "is_pool": True, + } + ) + except Exception: + logger.exception("Exception occur while contacting netbox") + return {"message": "Error occured while creating network."} + else: + network_entry["ipv6"] = prefix["prefix"] else: network_entry["ipv6"] = "fd00::/64" network_key = join_path( - config['network']["NETWORK_PREFIX"], - data["name"], - data["network_name"], + config['etcd']['network_prefix'], + data['name'], + data['network_name'], ) etcd_client.put(network_key, network_entry, value_in_json=True) return {"message": "Network successfully added."} @@ -460,7 +467,7 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - config['network']["NETWORK_PREFIX"], data["name"] + config['etcd']['network_prefix'], data["name"] ) networks = etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] @@ -496,7 +503,7 @@ api.add_resource(CreateNetwork, "/network/create") def main(): - image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True)) + image_stores = list(etcd_client.get_prefix(config['etcd']['image_store_prefix'], value_in_json=True)) if len(image_stores) == 0: data = { "is_public": True, @@ -506,7 +513,7 @@ def main(): "attributes": {"list": [], "key": [], "pool": "images"}, } - etcd_client.put(join_path(config['etcd']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data)) + etcd_client.put(join_path(config['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) app.run(host="::", debug=True) diff --git a/ucloud/api/schemas.py b/ucloud/api/schemas.py index 23db184..a3e0aa8 100755 --- a/ucloud/api/schemas.py +++ b/ucloud/api/schemas.py @@ -22,7 +22,7 @@ import bitmath from ucloud.common.host import HostStatus from ucloud.common.vm import VMStatus from ucloud.config import etcd_client, config, vm_pool, host_pool -from . import helper +from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -102,14 +102,14 @@ class CreateImageSchema(BaseSchema): super().__init__(data, fields) def file_uuid_validation(self): - file_entry = etcd_client.get(os.path.join(config['etcd']['FILE_PREFIX'], self.uuid.value)) + file_entry = etcd_client.get(os.path.join(config['etcd']['file_prefix'], self.uuid.value)) if file_entry is None: self.add_error( "Image File with uuid '{}' Not Found".format(self.uuid.value) ) def image_store_name_validation(self): - image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'])) + image_stores = list(etcd_client.get_prefix(config['etcd']['image_store_prefix'])) image_store = next( filter( @@ -220,6 +220,7 @@ class CreateVMSchema(OTPSchema): try: image_uuid = helper.resolve_image_name(self.image.value, etcd_client) except Exception as e: + logger.exception("Cannot resolve image name = %s", self.image.value) self.add_error(str(e)) else: self.image_uuid = image_uuid @@ -235,7 +236,7 @@ class CreateVMSchema(OTPSchema): if _network: for net in _network: - network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'], + network = etcd_client.get(os.path.join(config['etcd']['network_prefix'], self.name.value, net), value_in_json=True) if not network: @@ -400,7 +401,7 @@ class VmMigrationSchema(OTPSchema): if vm.status != VMStatus.running: self.add_error("Can't migrate non-running VM") - if vm.hostname == os.path.join(config['etcd']['HOST_PREFIX'], self.destination.value): + if vm.hostname == os.path.join(config['etcd']['host_prefix'], self.destination.value): self.add_error("Destination host couldn't be same as Source Host") @@ -442,7 +443,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'], + network = etcd_client.get(os.path.join(config['etcd']['network_prefix'], self.name.value, self.network_name.value), value_in_json=True) diff --git a/ucloud/common/classes.py b/ucloud/common/classes.py index 2eae809..29dffd4 100644 --- a/ucloud/common/classes.py +++ b/ucloud/common/classes.py @@ -1,4 +1,4 @@ -from etcd3_wrapper import EtcdEntry +from .etcd_wrapper import EtcdEntry class SpecificEtcdEntryBase: diff --git a/ucloud/common/counters.py b/ucloud/common/counters.py index 066a870..2d4a8e9 100644 --- a/ucloud/common/counters.py +++ b/ucloud/common/counters.py @@ -1,4 +1,4 @@ -from etcd3_wrapper import Etcd3Wrapper +from .etcd_wrapper import Etcd3Wrapper def increment_etcd_counter(etcd_client: Etcd3Wrapper, key): diff --git a/ucloud/common/etcd_wrapper.py b/ucloud/common/etcd_wrapper.py new file mode 100644 index 0000000..a3fb83f --- /dev/null +++ b/ucloud/common/etcd_wrapper.py @@ -0,0 +1,74 @@ +import etcd3 +import json +import queue +import copy + +from collections import namedtuple + +PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) + +class EtcdEntry: + # key: str + # value: str + + def __init__(self, meta, value, value_in_json=False): + self.key = meta.key.decode("utf-8") + self.value = value.decode("utf-8") + + if value_in_json: + self.value = json.loads(self.value) + +class Etcd3Wrapper: + def __init__(self, *args, **kwargs): + self.client = etcd3.client(*args, **kwargs) + + def get(self, *args, value_in_json=False, **kwargs): + _value, _key = self.client.get(*args, **kwargs) + if _key is None or _value is None: + return None + return EtcdEntry(_key, _value, value_in_json=value_in_json) + + def put(self, *args, value_in_json=False, **kwargs): + _key, _value = args + if value_in_json: + _value = json.dumps(_value) + + if not isinstance(_key, str): + _key = _key.decode("utf-8") + + return self.client.put(_key, _value, **kwargs) + + def get_prefix(self, *args, value_in_json=False, **kwargs): + r = self.client.get_prefix(*args, **kwargs) + for entry in r: + e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) + if e.value: + yield e + + def watch_prefix(self, key, timeout=0, value_in_json=False): + timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"), + value=str.encode(json.dumps({"status": "TIMEOUT", + "type": "TIMEOUT"})), + value_in_json=value_in_json) + + event_queue = queue.Queue() + + def add_event_to_queue(event): + for e in event.events: + if e.value: + event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json)) + + self.client.add_watch_prefix_callback(key, add_event_to_queue) + + while True: + try: + while True: + v = event_queue.get(timeout=timeout) + yield v + except queue.Empty: + event_queue.put(copy.deepcopy(timeout_event)) + + +class PsuedoEtcdEntry(EtcdEntry): + def __init__(self, key, value, value_in_json=False): + super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json) diff --git a/ucloud/common/helpers.py b/ucloud/common/helpers.py index 1bdf0b4..501aa90 100644 --- a/ucloud/common/helpers.py +++ b/ucloud/common/helpers.py @@ -6,21 +6,7 @@ import json from ipaddress import ip_address from os.path import join as join_path - - -def create_package_loggers(packages, base_path, mode="a"): - loggers = {} - for pkg in packages: - logger = logging.getLogger(pkg) - logger_handler = logging.FileHandler( - join_path(base_path, "{}.txt".format(pkg)), - mode=mode - ) - logger.setLevel(logging.DEBUG) - logger_handler.setFormatter(logging.Formatter(fmt="%(asctime)s: %(levelname)s - %(message)s", - datefmt="%d-%b-%y %H:%M:%S")) - logger.addHandler(logger_handler) - loggers[pkg] = logger +from . import logger # TODO: Should be removed as soon as migration @@ -35,7 +21,7 @@ def get_ipv4_address(): except socket.timeout: address = "127.0.0.1" except Exception as e: - logging.getLogger().exception(e) + logger.exception(e) address = "127.0.0.1" else: address = s.getsockname()[0] @@ -49,6 +35,6 @@ def get_ipv6_address(): content = json.loads(r.content.decode("utf-8")) ip = ip_address(content["ip"]).exploded except Exception as e: - logging.exception(e) + logger.exception(e) else: return ip diff --git a/ucloud/common/request.py b/ucloud/common/request.py index cadac80..1e4594d 100644 --- a/ucloud/common/request.py +++ b/ucloud/common/request.py @@ -2,8 +2,7 @@ import json from os.path import join from uuid import uuid4 -from etcd3_wrapper.etcd3_wrapper import PsuedoEtcdEntry - +from .etcd_wrapper import PsuedoEtcdEntry from .classes import SpecificEtcdEntryBase diff --git a/ucloud/config.py b/ucloud/config.py index 5662e64..3eee897 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -1,133 +1,53 @@ +import configparser +import os +import logging + from ucloud.common.host import HostPool from ucloud.common.request import RequestPool from ucloud.common.vm import VmPool from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler +from ucloud.common.etcd_wrapper import Etcd3Wrapper -# Replacing decouple inline -import configparser -import os -import os.path - -import logging -log = logging.getLogger("ucloud.config") - -conf_name = "ucloud.conf" - -try: - conf_dir = os.environ["UCLOUD_CONF_DIR"] -except KeyError: - conf_dir = "/etc/ucloud" +log = logging.getLogger('ucloud.config') +conf_name = 'ucloud.conf' +conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud') config_file = os.path.join(conf_dir, conf_name) -config = configparser.ConfigParser() +config = configparser.ConfigParser(allow_no_value=True) -try: +if os.access(config_file, os.R_OK): config.read(config_file) -except FileNotFoundError: - log.warn("Configuration file not found - using defaults") - - -################################################################################ -# ETCD3 support -import etcd3 -import json -import queue -import copy -from collections import namedtuple - -PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) - -class EtcdEntry: - # key: str - # value: str - - def __init__(self, meta, value, value_in_json=False): - self.key = meta.key.decode("utf-8") - self.value = value.decode("utf-8") - - if value_in_json: - self.value = json.loads(self.value) - -class Etcd3Wrapper: - def __init__(self, *args, **kwargs): - self.client = etcd3.client(*args, **kwargs) - - def get(self, *args, value_in_json=False, **kwargs): - _value, _key = self.client.get(*args, **kwargs) - if _key is None or _value is None: - return None - return EtcdEntry(_key, _value, value_in_json=value_in_json) - - def put(self, *args, value_in_json=False, **kwargs): - _key, _value = args - if value_in_json: - _value = json.dumps(_value) - - if not isinstance(_key, str): - _key = _key.decode("utf-8") - - return self.client.put(_key, _value, **kwargs) - - def get_prefix(self, *args, value_in_json=False, **kwargs): - r = self.client.get_prefix(*args, **kwargs) - for entry in r: - e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) - if e.value: - yield e - - def watch_prefix(self, key, timeout=0, value_in_json=False): - timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"), - value=str.encode(json.dumps({"status": "TIMEOUT", - "type": "TIMEOUT"})), - value_in_json=value_in_json) - - event_queue = queue.Queue() - - def add_event_to_queue(event): - for e in event.events: - if e.value: - event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json)) - - self.client.add_watch_prefix_callback(key, add_event_to_queue) - - while True: - try: - while True: - v = event_queue.get(timeout=timeout) - yield v - except queue.Empty: - event_queue.put(copy.deepcopy(timeout_event)) - - -class PsuedoEtcdEntry(EtcdEntry): - def __init__(self, key, value, value_in_json=False): - super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json) - +else: + log.warning('Configuration file not found - using defaults') etcd_wrapper_args = () etcd_wrapper_kwargs = { - 'host': config['etcd']['ETCD_URL'], - 'port': config['etcd']['ETCD_PORT'], - 'ca_cert': config['etcd']['CA_CERT'], - 'cert_cert': config['etcd']['CERT_CERT'], - 'cert_key': config['etcd']['CERT_KEY'] + 'host': config['etcd']['url'], + 'port': config['etcd']['port'], + 'ca_cert': config['etcd']['ca_cert'], + 'cert_cert': config['etcd']['cert_cert'], + 'cert_key': config['etcd']['cert_key'] } etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) -host_pool = HostPool(etcd_client, config['etcd']['HOST_PREFIX']) -vm_pool = VmPool(etcd_client, config['etcd']['VM_PREFIX']) -request_pool = RequestPool(etcd_client, config['etcd']['REQUEST_PREFIX']) +host_pool = HostPool(etcd_client, config['etcd']['host_prefix']) +vm_pool = VmPool(etcd_client, config['etcd']['vm_prefix']) +request_pool = RequestPool(etcd_client, config['etcd']['request_prefix']) running_vms = [] -__storage_backend = config['storage']["STORAGE_BACKEND"] -if __storage_backend == "filesystem": - image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=config['storage']["VM_DIR"], - image_base=config['storage']["IMAGE_DIR"]) -elif __storage_backend == "ceph": - image_storage_handler = CEPHBasedImageStorageHandler(vm_base=config['storage']["CEPH_VM_POOL"], - image_base=config['storage']["CEPH_IMAGE_POOL"]) +__storage_backend = config['storage']['backend'] +if __storage_backend == 'filesystem': + image_storage_handler = FileSystemBasedImageStorageHandler( + vm_base=config['storage']['vm_dir'], + image_base=config['storage']['image_dir'] + ) +elif __storage_backend == 'ceph': + image_storage_handler = CEPHBasedImageStorageHandler( + vm_base=config['storage']['ceph_vm_pool'], + image_base=config['storage']['ceph_image_pool'] + ) else: - raise Exception("Unknown Image Storage Handler") + raise Exception('Unknown Image Storage Handler') diff --git a/ucloud/filescanner/main.py b/ucloud/filescanner/main.py index ef6fee6..265f9d9 100755 --- a/ucloud/filescanner/main.py +++ b/ucloud/filescanner/main.py @@ -3,6 +3,7 @@ import os import pathlib import subprocess as sp import time +import sys from uuid import uuid4 from . import logger @@ -19,7 +20,6 @@ def getxattr(file, attr): '--absolute-names'], stderr=sp.DEVNULL) value = value.decode("utf-8") except sp.CalledProcessError as e: - logger.exception(e) value = None return value @@ -63,14 +63,13 @@ try: sp.check_output(['which', 'getfattr']) sp.check_output(['which', 'setfattr']) except Exception as e: - logger.exception(e) - print('Make sure you have getfattr and setfattr available') - exit(1) + logger.error("You don't seems to have both getfattr and setfattr") + sys.exit(1) def main(): - BASE_DIR = config['storage']["FILE_DIR"] - FILE_PREFIX = config['storage']["FILE_PREFIX"] + BASE_DIR = config['storage']['file_dir'] + FILE_PREFIX = config['etcd']['file_prefix'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob("{}/**".format(BASE_DIR), recursive=True) @@ -79,7 +78,7 @@ def main(): files = list(filter(os.path.isfile, files)) untracked_files = list( - filter(lambda f: not bool(getxattr(f, "user.utracked")), files) + filter(lambda f: not bool(getxattr(f, "utracked")), files) ) tracked_files = list( @@ -89,7 +88,8 @@ def main(): file_id = uuid4() # Get Username - owner = pathlib.Path(file).parts[3] + owner = pathlib.Path(file).parts[len(pathlib.Path(BASE_DIR).parts)] + # Get Creation Date of File # Here, we are assuming that ctime is creation time # which is mostly not true. @@ -101,9 +101,7 @@ def main(): # Compute sha512 sum sha_sum = sha512sum(file) - # File Path excluding base and username - file_path = pathlib.Path(file).parts[4:] - file_path = os.path.join(*file_path) + file_path = pathlib.Path(file).parts[-1] # Create Entry entry_key = os.path.join(FILE_PREFIX, str(file_id)) @@ -115,10 +113,10 @@ def main(): "size": size } - print("Tracking {}".format(file)) + logger.info("Tracking %s", file) # Insert Entry etcd_client.put(entry_key, entry_value, value_in_json=True) - setxattr(file, "user.utracked", True) + setxattr(file, "utracked", True) if __name__ == "__main__": diff --git a/ucloud/host/main.py b/ucloud/host/main.py index 0ca345b..bd03a08 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -1,9 +1,10 @@ import argparse import multiprocessing as mp import time +import sys -from etcd3_wrapper import Etcd3Wrapper - +from os.path import isdir +from ucloud.common.etcd_wrapper import Etcd3Wrapper from ucloud.common.request import RequestEntry, RequestType from ucloud.config import (vm_pool, request_pool, etcd_client, running_vms, @@ -18,7 +19,7 @@ from ucloud.host import logger def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) - host_pool = HostPool(client, config['etcd']['HOST_PREFIX']) + host_pool = HostPool(client, config['etcd']['host_prefix']) this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) while True: @@ -72,9 +73,11 @@ def maintenance(host): running_vms.remove(_vm) def check(): - if config['etcd']['STORAGE_BACKEND'] == 'filesystem' and not isdir(config['etcd']['VM_DIR']): + if config['storage']['backend'] == 'filesystem' and \ + not isdir(config['storage']['vm_dir']): + print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" - " in .env file must exists. But, it don't.") + " in /etc/ucloud/ucloud.conf file must exists. But, it don't.") sys.exit(1) @@ -84,7 +87,7 @@ def main(hostname): heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) - host_pool = HostPool(etcd_client, config['etcd']['HOST_PREFIX']) + host_pool = HostPool(etcd_client, config['etcd']['host_prefix']) host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) assert host is not None, "No such host with name = {}".format(hostname) @@ -106,8 +109,8 @@ def main(hostname): # beat updating mechanism in separated thread for events_iterator in [ - etcd_client.get_prefix(config['etcd']['REQUEST_PREFIX'], value_in_json=True), - etcd_client.watch_prefix(config['etcd']['REQUEST_PREFIX'], timeout=10, value_in_json=True), + etcd_client.get_prefix(config['etcd']['request_prefix'], value_in_json=True), + etcd_client.watch_prefix(config['etcd']['request_prefix'], timeout=10, value_in_json=True), ]: for request_event in events_iterator: request_event = RequestEntry(request_event) diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index 33e4433..4a7584a 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -46,7 +46,7 @@ def delete_network_interface(iface): def resolve_network(network_name, network_owner): - network = etcd_client.get(join_path(config['etcd']["NETWORK_PREFIX"], + network = etcd_client.get(join_path(config['etcd']['network_prefix'], network_owner, network_name), value_in_json=True) @@ -179,7 +179,7 @@ def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, mi for network_mac_and_tap in vm_networks: network_name, mac, tap = network_mac_and_tap - _key = os.path.join(config['etcd']['NETWORK_PREFIX'], vm_entry.owner, network_name) + _key = os.path.join(config['etcd']['network_prefix'], vm_entry.owner, network_name) network = etcd_client.get(_key, value_in_json=True) network_type = network.value["type"] network_id = str(network.value["id"]) @@ -187,7 +187,7 @@ def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, mi if network_type == "vxlan": tap = create_vxlan_br_tap(_id=network_id, - _dev=config['etcd']["VXLAN_PHY_DEV"], + _dev=config['network']['vxlan_phy_dev'], tap_id=tap, ip=network_ipv6) update_radvd_conf(etcd_client) @@ -303,13 +303,13 @@ def transfer(request_event): _host, _port = request_event.parameters["host"], request_event.parameters["port"] _uuid = request_event.uuid _destination = request_event.destination_host_key - vm = get_vm(running_vms, join_path(config['etcd']['VM_PREFIX'], _uuid)) + vm = get_vm(running_vms, join_path(config['etcd']['vm_prefix'], _uuid)) if vm: tunnel = sshtunnel.SSHTunnelForwarder( _host, - ssh_username=config['ssh']["ssh_username"], - ssh_pkey=config['ssh']["SSH_PRIVATEKEY"], + ssh_username=config['ssh']['username'], + ssh_pkey=config['ssh']['private_key_path'], remote_bind_address=("127.0.0.1", _port), ssh_proxy_enabled=True, ssh_proxy=(_host, 22) @@ -373,7 +373,7 @@ def launch_vm(vm_entry, migration=False, migration_port=None, destination_host_k parameters={"host": get_ipv6_address(), "port": migration_port}, uuid=vm_entry.uuid, destination_host_key=destination_host_key, - request_prefix=config['etcd']["REQUEST_PREFIX"] + request_prefix=config['etcd']['request_prefix'] ) request_pool.put(r) else: diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index df4dfad..2658641 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -20,9 +20,9 @@ def qemu_img_type(path): def check(): """ check whether settings are sane, refuse to start if they aren't """ - if config['etcd']['STORAGE_BACKEND'] == 'filesystem' and not isdir(config['etcd']['IMAGE_DIR']): + if config['storage']['backend'] == 'filesystem' and not isdir(config['storage']['image_dir']): print("You have set STORAGE_BACKEND to filesystem, but " - "{} does not exist. Refusing to start".format(config['etcd']['IMAGE_DIR'])) + "{} does not exist. Refusing to start".format(config['storage']['image_dir'])) sys.exit(1) try: @@ -34,7 +34,7 @@ def check(): def main(): # We want to get images entries that requests images to be created - images = etcd_client.get_prefix(config['etcd']['IMAGE_PREFIX'], value_in_json=True) + images = etcd_client.get_prefix(config['etcd']['image_prefix'], value_in_json=True) images_to_be_created = list(filter(lambda im: im.value['status'] == 'TO_BE_CREATED', images)) for image in images_to_be_created: @@ -43,9 +43,10 @@ def main(): image_owner = image.value['owner'] image_filename = image.value['filename'] image_store_name = image.value['store_name'] - image_full_path = join_path(config['etcd']['BASE_DIR'], image_owner, image_filename) + image_full_path = join_path(config['storage']['file_dir'], image_owner, image_filename) - image_stores = etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True) + image_stores = etcd_client.get_prefix(config['etcd']['image_store_prefix'], + value_in_json=True) user_image_store = next(filter( lambda s, store_name=image_store_name: s.value["name"] == store_name, image_stores diff --git a/ucloud/metadata/main.py b/ucloud/metadata/main.py index 9281d7c..16b7c6d 100644 --- a/ucloud/metadata/main.py +++ b/ucloud/metadata/main.py @@ -43,7 +43,8 @@ class Root(Resource): if not data: return {'message': 'Metadata for such VM does not exists.'}, 404 else: - etcd_key = os.path.join(config['etcd']['USER_PREFIX'], data.value['owner_realm'], + etcd_key = os.path.join(config['etcd']['user_prefix'], + data.value['owner_realm'], data.value['owner'], 'key') etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True) user_personal_ssh_keys = [key.value for key in etcd_entry] diff --git a/ucloud/scheduler/helper.py b/ucloud/scheduler/helper.py index 1754045..560bdbc 100755 --- a/ucloud/scheduler/helper.py +++ b/ucloud/scheduler/helper.py @@ -106,7 +106,7 @@ def assign_host(vm): r = RequestEntry.from_scratch(type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=config['etcd']['REQUEST_PREFIX']) + request_prefix=config['etcd']['request_prefix']) request_pool.put(r) vm.log.append("VM scheduled for starting") diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index 33e94f2..91a333e 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -18,8 +18,8 @@ def main(): pending_vms = [] for request_iterator in [ - etcd_client.get_prefix(config['etcd']['REQUEST_PREFIX'], value_in_json=True), - etcd_client.watch_prefix(config['etcd']['REQUEST_PREFIX'], timeout=5, value_in_json=True), + etcd_client.get_prefix(config['etcd']['request_prefix'], value_in_json=True), + etcd_client.watch_prefix(config['etcd']['request_prefix'], timeout=5, value_in_json=True), ]: for request_event in request_iterator: request_entry = RequestEntry(request_event) @@ -46,7 +46,7 @@ def main(): r = RequestEntry.from_scratch(type="ScheduleVM", uuid=pending_vm_entry.uuid, hostname=pending_vm_entry.hostname, - request_prefix=config['etcd']['REQUEST_PREFIX']) + request_prefix=config['etcd']['request_prefix']) request_pool.put(r) elif request_entry.type == RequestType.ScheduleVM: @@ -72,7 +72,7 @@ def main(): r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, uuid=request_entry.uuid, destination=request_entry.destination, - request_prefix=config['etcd']['REQUEST_PREFIX']) + request_prefix=config['etcd']['request_prefix']) request_pool.put(r) # If the Request is about a VM that just want to get started/created