Fix issues in naming and few other things

conf/ucloud.conf

@@ -1,60 +1,53 @@
-# This section contains default values for all other sections
+[otp]
-#
+server = https://otp.ungleich.ch/ungleichotp/
-[DEFAULT]
+verify_endpoint = verify/
-
+auth_name = replace_me
-AUTH_NAME = "replace me"
+auth_realm = replace_me
-AUTH_SEED = "replace me"
+auth_seed = replace_me
-AUTH_REALM = "replace me"
-
-NETWORK_PREFIX = moo
-
-OTP_VERIFY_ENDPOINT = verify/
-
-[api]
-NETWORK_PREFIX = foo
 
 [network]
-PREFIX_LENGTH = 64
+prefix_length = 64
-PREFIX = 2001:db8::/48
+prefix = 2001:db8::/48
+vxlan_phy_dev = eno1
 
 [netbox]
-NETBOX_URL   = https://replace-me.example.com
+url   = https://replace-me.example.com
-NETBOX_TOKEN = replace me
+token = replace_me
 
 [etcd]
-ETCD_URL = localhost
+url = localhost
-ETCD_PORT = 2379
+port = 2379
 
-CA_CERT
+ca_cert
-CERT_CERT
+cert_cert
-CERT_KEY
+cert_key
 
-
+file_prefix        = /files/
-FILE_PREFIX        = files
+host_prefix        = /hosts/
-HOST_PREFIx        = hosts
+image_prefix       = /images/
-IMAGE_PREFIX       = images
+image_store_prefix = /imagestore/
-IMAGE_STORE_PREFIX = imagestore
+network_prefix     = /networks/
-
+request_prefix     = /requests/
-NETWORK_PREFIX     = networks
+user_prefix        = /users/
-REQUEST_PREFIX     = requests
+vm_prefix          = /vms/
-USER_PREFIX        = users
-VM_PREFIX          = vms
 
 [storage]
+
 #values = filesystem, ceph
-STORAGE_BACKEND =
+backend = filesystem
 
 # if STORAGE_BACKEND = filesystem
-VM_DIR =
+vm_dir  = /var/lib/ucloud/vms
-IMG_DIR =
+image_dir = /var/lib/ucloud/images
 
 # if STORAGE_BACKEND = ceph
-CEPH_VM_POOL =
+ceph_vm_pool  = ssd
-CEPH_IMG_POOL =
+ceph_image_pool = ssd
 
 # Importing uploaded files
-FILE_DIR = /var/lib/ucloud/files
+file_dir = /var/lib/ucloud/files
 
+# For Migrating VMs over ssh/tcp
 [ssh]
-SSH_USERNAME =
+username
-SSH_PRIVATEKEY =
+private_key_path

scripts/ucloud

@@ -5,11 +5,17 @@ import logging
 import importlib
 import sys
 import os
+import multiprocessing as mp
+
 
 COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata']
 
 if __name__ == "__main__":
-    log = logging.getLogger("ucloud")
+    logging.basicConfig(level=logging.DEBUG,
+                    format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s',
+                    filename='/var/log/ucloud.log', filemode='a')
+
+    logger = logging.getLogger("ucloud")
 
     arg_parser = argparse.ArgumentParser(prog='ucloud',
                                          description='Open Source Cloud Management Software')
@@ -22,12 +28,12 @@ if __name__ == "__main__":
         os.environ['UCLOUD_CONF_DIR'] = args.conf_dir
 
     try:
+        mp.set_start_method('spawn')
         name = args.component
         mod = importlib.import_module("ucloud.{}.main".format(name))
         main = getattr(mod, "main")
-
+        main(*args.component_args)
-        main()
 
     except Exception as e:
-        logging.exception(e)
+        logger.exception(e)
         print(e)

setup.py

@@ -8,8 +8,8 @@ try:
     version = ucloud.version.VERSION
 except:
     import subprocess
-    c = subprocess.run(["git", "describe"], capture_output=True)
+    c = subprocess.check_output(['git', 'describe'])
-    version = c.stdout.decode("utf-8")
+    version = c.decode("utf-8").strip()
 
 
 setup(name='ucloud',
@@ -28,8 +28,7 @@ setup(name='ucloud',
       packages=find_packages(),
       install_requires=[
           'requests',
-          'python-decouple',
+          'Flask',
-          'flask',
           'flask-restful',
           'bitmath',
           'pyotp',
@@ -37,8 +36,8 @@ setup(name='ucloud',
           'sphinx',
           'pynetbox',
           'sphinx-rtd-theme',
-          'etcd3_wrapper @ https://code.ungleich.ch/ungleich-public/etcd3_wrapper/repository/master/archive.tar.gz#egg=etcd3_wrapper',
           'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3',
       ],
       scripts=['scripts/ucloud'],
+      data_files=[('/etc/ucloud/', ['conf/ucloud.conf'])],
       zip_safe=False)

ucloud/api/common_fields.py

@@ -47,6 +47,6 @@ class VmUUIDField(Field):
         self.validation = self.vm_uuid_validation
 
     def vm_uuid_validation(self):
-        r = etcd_client.get(os.path.join(config['api']['VM_PREFIX'], self.uuid))
+        r = etcd_client.get(os.path.join(config['etcd']['vm_prefix'], self.uuid))
         if not r:
             self.add_error("VM with uuid {} does not exists".format(self.uuid))

ucloud/api/create_image_store.py

@@ -13,4 +13,4 @@ data = {
     "attributes": {"list": [], "key": [], "pool": "images"},
 }
 
-etcd_client.put(os.path.join(config['api']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data))
+etcd_client.put(os.path.join(config['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data))

ucloud/api/helper.py

@@ -2,7 +2,7 @@ import binascii
 import ipaddress
 import random
 import subprocess as sp
-
+import logging
 import requests
 
 from pyotp import TOTP
@@ -10,23 +10,28 @@ from pyotp import TOTP
 from ucloud.config import vm_pool, config
 
 
+logger = logging.getLogger("ucloud.api.helper")
+
 def check_otp(name, realm, token):
     try:
         data = {
-            "auth_name": config['api']["AUTH_NAME"],
+            "auth_name": config['otp']['auth_name'],
-            "auth_token": TOTP(config['api']["AUTH_SEED"]).now(),
+            "auth_token": TOTP(config['otp']['auth_seed']).now(),
-            "auth_realm": config['api']["AUTH_REALM"],
+            "auth_realm": config['otp']['auth_realm'],
             "name": name,
             "realm": realm,
             "token": token,
         }
-    except binascii.Error:
+    except binascii.Error as err:
+        logger.error(
+            "Cannot compute OTP for seed: {}".format(config['otp']['auth_seed'])
+        )
         return 400
 
     response = requests.post(
         "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format(
-            OTP_SERVER=config['api']["OTP_SERVER"],
+            OTP_SERVER=config['otp']['server'],
-            OTP_VERIFY_ENDPOINT=config['api']["OTP_VERIFY_ENDPOINT"]
+            OTP_VERIFY_ENDPOINT=config['otp']['verify_endpoint']
         ),
         json=data,
     )
@@ -80,7 +85,7 @@ def resolve_image_name(name, etcd_client):
     except Exception:
         raise ValueError("Image name not in correct format i.e {store_name}:{image_name}")
 
-    images = etcd_client.get_prefix(config['api']['IMAGE_PREFIX'], value_in_json=True)
+    images = etcd_client.get_prefix(config['etcd']['image_prefix'], value_in_json=True)
 
     # Try to find image with name == image_name and store_name == store_name
     try:

ucloud/api/main.py

@@ -10,7 +10,10 @@ from flask_restful import Resource, Api
 from ucloud.common import counters
 from ucloud.common.vm import VMStatus
 from ucloud.common.request import RequestEntry, RequestType
-from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, config, image_storage_handler)
+from ucloud.config import (
+    etcd_client, request_pool, vm_pool,
+    host_pool, config, image_storage_handler
+)
 from . import schemas
 from .helper import generate_mac, mac2ipv6
 from . import logger
@@ -28,7 +31,7 @@ class CreateVM(Resource):
         validator = schemas.CreateVMSchema(data)
         if validator.is_valid():
             vm_uuid = uuid4().hex
-            vm_key = join_path(config['etcd']["VM_PREFIX"], vm_uuid)
+            vm_key = join_path(config['etcd']['vm_prefix'], vm_uuid)
             specs = {
                 "cpu": validator.specs["cpu"],
                 "ram": validator.specs["ram"],
@@ -56,7 +59,7 @@ class CreateVM(Resource):
             # Create ScheduleVM Request
             r = RequestEntry.from_scratch(
                 type=RequestType.ScheduleVM, uuid=vm_uuid,
-                request_prefix=config['etcd']["REQUEST_PREFIX"]
+                request_prefix=config['etcd']['request_prefix']
             )
             request_pool.put(r)
 
@@ -71,7 +74,7 @@ class VmStatus(Resource):
         validator = schemas.VMStatusSchema(data)
         if validator.is_valid():
             vm = vm_pool.get(
-                join_path(config['etcd']["VM_PREFIX"], data["uuid"])
+                join_path(config['etcd']['vm_prefix'], data["uuid"])
             )
             vm_value = vm.value.copy()
             vm_value["ip"] = []
@@ -79,7 +82,7 @@ class VmStatus(Resource):
                 network_name, mac, tap = network_mac_and_tap
                 network = etcd_client.get(
                     join_path(
-                        config['etcd']["NETWORK_PREFIX"],
+                        config['etcd']['network_prefix'],
                         data["name"],
                         network_name,
                     ),
@@ -100,7 +103,7 @@ class CreateImage(Resource):
         validator = schemas.CreateImageSchema(data)
         if validator.is_valid():
             file_entry = etcd_client.get(
-                join_path(config['etcd']["FILE_PREFIX"], data["uuid"])
+                join_path(config['etcd']['file_prefix'], data["uuid"])
             )
             file_entry_value = json.loads(file_entry.value)
 
@@ -113,7 +116,7 @@ class CreateImage(Resource):
                 "visibility": "public",
             }
             etcd_client.put(
-                join_path(config['etcd']["IMAGE_PREFIX"], data["uuid"]),
+                join_path(config['etcd']['image_prefix'], data["uuid"]),
                 json.dumps(image_entry_json),
             )
 
@@ -125,7 +128,7 @@ class ListPublicImages(Resource):
     @staticmethod
     def get():
         images = etcd_client.get_prefix(
-            config['etcd']["IMAGE_PREFIX"], value_in_json=True
+            config['etcd']['image_prefix'], value_in_json=True
         )
         r = {
             "images": []
@@ -148,7 +151,7 @@ class VMAction(Resource):
 
         if validator.is_valid():
             vm_entry = vm_pool.get(
-                join_path(config['etcd']["VM_PREFIX"], data["uuid"])
+                join_path(config['etcd']['vm_prefix'], data["uuid"])
             )
             action = data["action"]
 
@@ -172,7 +175,7 @@ class VMAction(Resource):
                 type="{}VM".format(action.title()),
                 uuid=data["uuid"],
                 hostname=vm_entry.hostname,
-                request_prefix=config['etcd']["REQUEST_PREFIX"]
+                request_prefix=config['etcd']['request_prefix']
             )
             request_pool.put(r)
             return {"message": "VM {} Queued".format(action.title())}, 200
@@ -193,10 +196,10 @@ class VMMigration(Resource):
                 type=RequestType.ScheduleVM,
                 uuid=vm.uuid,
                 destination=join_path(
-                    config['etcd']["HOST_PREFIX"], validator.destination.value
+                    config['etcd']['host_prefix'], validator.destination.value
                 ),
                 migration=True,
-                request_prefix=config['etcd']["REQUEST_PREFIX"]
+                request_prefix=config['etcd']['request_prefix']
             )
             request_pool.put(r)
             return {"message": "VM Migration Initialization Queued"}, 200
@@ -212,7 +215,7 @@ class ListUserVM(Resource):
 
         if validator.is_valid():
             vms = etcd_client.get_prefix(
-                config['etcd']["VM_PREFIX"], value_in_json=True
+                config['etcd']['vm_prefix'], value_in_json=True
             )
             return_vms = []
             user_vms = filter(lambda v: v.value["owner"] == data["name"], vms)
@@ -246,7 +249,7 @@ class ListUserFiles(Resource):
 
         if validator.is_valid():
             files = etcd_client.get_prefix(
-                config['etcd']["FILE_PREFIX"], value_in_json=True
+                config['etcd']['file_prefix'], value_in_json=True
             )
             return_files = []
             user_files = list(
@@ -270,7 +273,7 @@ class CreateHost(Resource):
         data = request.json
         validator = schemas.CreateHostSchema(data)
         if validator.is_valid():
-            host_key = join_path(config['etcd']["HOST_PREFIX"], uuid4().hex)
+            host_key = join_path(config['etcd']['host_prefix'], uuid4().hex)
             host_entry = {
                 "specs": data["specs"],
                 "hostname": data["hostname"],
@@ -309,7 +312,7 @@ class GetSSHKeys(Resource):
 
                 # {user_prefix}/{realm}/{name}/key/
                 etcd_key = join_path(
-                    config['etcd']['USER_PREFIX'],
+                    config['etcd']['user_prefix'],
                     data["realm"],
                     data["name"],
                     "key",
@@ -326,7 +329,7 @@ class GetSSHKeys(Resource):
 
                 # {user_prefix}/{realm}/{name}/key/{key_name}
                 etcd_key = join_path(
-                    config['etcd']['USER_PREFIX'],
+                    config['etcd']['user_prefix'],
                     data["realm"],
                     data["name"],
                     "key",
@@ -355,7 +358,7 @@ class AddSSHKey(Resource):
 
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                config['etcd']["USER_PREFIX"],
+                config['etcd']['user_prefix'],
                 data["realm"],
                 data["name"],
                 "key",
@@ -385,7 +388,7 @@ class RemoveSSHKey(Resource):
 
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                config['etcd']["USER_PREFIX"],
+                config['etcd']['user_prefix'],
                 data["realm"],
                 data["name"],
                 "key",
@@ -420,31 +423,35 @@ class CreateNetwork(Resource):
                 "type": data["type"],
             }
             if validator.user.value:
-                nb = pynetbox.api(
+                try:
-                    url=config['netbox']["NETBOX_URL"],
+                    nb = pynetbox.api(
-                    token=config['netbox']["NETBOX_TOKEN"],
+                        url=config['netbox']['url'],
-                )
+                        token=config['netbox']['token'],
-                nb_prefix = nb.ipam.prefixes.get(
+                    )
-                    prefix=config['network']["PREFIX"]
+                    nb_prefix = nb.ipam.prefixes.get(
-                )
+                        prefix=config['network']['prefix']
-
+                    )
-                prefix = nb_prefix.available_prefixes.create(
+                    prefix = nb_prefix.available_prefixes.create(
-                    data={
+                        data={
-                        "prefix_length": config['network']["PREFIX_LENGTH"],
+                            "prefix_length": int(config['network']['prefix_length']),
-                        "description": '{}\'s network "{}"'.format(
+                            "description": '{}\'s network "{}"'.format(
-                            data["name"], data["network_name"]
+                                data["name"], data["network_name"]
-                        ),
+                            ),
-                        "is_pool": True,
+                            "is_pool": True,
-                    }
+                        }
-                )
+                    )
-                network_entry["ipv6"] = prefix["prefix"]
+                except Exception:
+                    logger.exception("Exception occur while contacting netbox")
+                    return {"message": "Error occured while creating network."}
+                else:
+                    network_entry["ipv6"] = prefix["prefix"]
             else:
                 network_entry["ipv6"] = "fd00::/64"
 
             network_key = join_path(
-                config['network']["NETWORK_PREFIX"],
+                config['etcd']['network_prefix'],
-                data["name"],
+                data['name'],
-                data["network_name"],
+                data['network_name'],
             )
             etcd_client.put(network_key, network_entry, value_in_json=True)
             return {"message": "Network successfully added."}
@@ -460,7 +467,7 @@ class ListUserNetwork(Resource):
 
         if validator.is_valid():
             prefix = join_path(
-                config['network']["NETWORK_PREFIX"], data["name"]
+                config['etcd']['network_prefix'], data["name"]
             )
             networks = etcd_client.get_prefix(prefix, value_in_json=True)
             user_networks = []
@@ -496,7 +503,7 @@ api.add_resource(CreateNetwork, "/network/create")
 
 
 def main():
-    image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True))
+    image_stores = list(etcd_client.get_prefix(config['etcd']['image_store_prefix'], value_in_json=True))
     if len(image_stores) == 0:
         data = {
             "is_public": True,
@@ -506,7 +513,7 @@ def main():
             "attributes": {"list": [], "key": [], "pool": "images"},
         }
 
-        etcd_client.put(join_path(config['etcd']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data))
+        etcd_client.put(join_path(config['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data))
 
     app.run(host="::", debug=True)
 

ucloud/api/schemas.py

@@ -22,7 +22,7 @@ import bitmath
 from ucloud.common.host import HostStatus
 from ucloud.common.vm import VMStatus
 from ucloud.config import etcd_client, config, vm_pool, host_pool
-from . import helper
+from . import helper, logger
 from .common_fields import Field, VmUUIDField
 from .helper import check_otp, resolve_vm_name
 
@@ -102,14 +102,14 @@ class CreateImageSchema(BaseSchema):
         super().__init__(data, fields)
 
     def file_uuid_validation(self):
-        file_entry = etcd_client.get(os.path.join(config['etcd']['FILE_PREFIX'], self.uuid.value))
+        file_entry = etcd_client.get(os.path.join(config['etcd']['file_prefix'], self.uuid.value))
         if file_entry is None:
             self.add_error(
                 "Image File with uuid '{}' Not Found".format(self.uuid.value)
             )
 
     def image_store_name_validation(self):
-        image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX']))
+        image_stores = list(etcd_client.get_prefix(config['etcd']['image_store_prefix']))
 
         image_store = next(
             filter(
@@ -220,6 +220,7 @@ class CreateVMSchema(OTPSchema):
         try:
             image_uuid = helper.resolve_image_name(self.image.value, etcd_client)
         except Exception as e:
+            logger.exception("Cannot resolve image name = %s", self.image.value)
             self.add_error(str(e))
         else:
             self.image_uuid = image_uuid
@@ -235,7 +236,7 @@ class CreateVMSchema(OTPSchema):
 
         if _network:
             for net in _network:
-                network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'],
+                network = etcd_client.get(os.path.join(config['etcd']['network_prefix'],
                                                        self.name.value,
                                                        net), value_in_json=True)
                 if not network:
@@ -400,7 +401,7 @@ class VmMigrationSchema(OTPSchema):
         if vm.status != VMStatus.running:
             self.add_error("Can't migrate non-running VM")
 
-        if vm.hostname == os.path.join(config['etcd']['HOST_PREFIX'], self.destination.value):
+        if vm.hostname == os.path.join(config['etcd']['host_prefix'], self.destination.value):
             self.add_error("Destination host couldn't be same as Source Host")
 
 
@@ -442,7 +443,7 @@ class CreateNetwork(OTPSchema):
         super().__init__(data, fields=fields)
 
     def network_name_validation(self):
-        network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'],
+        network = etcd_client.get(os.path.join(config['etcd']['network_prefix'],
                                                self.name.value,
                                                self.network_name.value),
                                   value_in_json=True)

ucloud/common/classes.py

@@ -1,4 +1,4 @@
-from etcd3_wrapper import EtcdEntry
+from .etcd_wrapper import EtcdEntry
 
 
 class SpecificEtcdEntryBase:

ucloud/common/counters.py

@@ -1,4 +1,4 @@
-from etcd3_wrapper import Etcd3Wrapper
+from .etcd_wrapper import Etcd3Wrapper
 
 
 def increment_etcd_counter(etcd_client: Etcd3Wrapper, key):

ucloud/common/etcd_wrapper.py

@@ -0,0 +1,74 @@
+import etcd3
+import json
+import queue
+import copy
+
+from collections import namedtuple
+
+PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"])
+
+class EtcdEntry:
+    # key: str
+    # value: str
+
+    def __init__(self, meta, value, value_in_json=False):
+        self.key = meta.key.decode("utf-8")
+        self.value = value.decode("utf-8")
+
+        if value_in_json:
+            self.value = json.loads(self.value)
+
+class Etcd3Wrapper:
+    def __init__(self, *args, **kwargs):
+        self.client = etcd3.client(*args, **kwargs)
+
+    def get(self, *args, value_in_json=False, **kwargs):
+        _value, _key = self.client.get(*args, **kwargs)
+        if _key is None or _value is None:
+            return None
+        return EtcdEntry(_key, _value, value_in_json=value_in_json)
+
+    def put(self, *args, value_in_json=False, **kwargs):
+        _key, _value = args
+        if value_in_json:
+            _value = json.dumps(_value)
+
+        if not isinstance(_key, str):
+            _key = _key.decode("utf-8")
+
+        return self.client.put(_key, _value, **kwargs)
+
+    def get_prefix(self, *args, value_in_json=False, **kwargs):
+        r = self.client.get_prefix(*args, **kwargs)
+        for entry in r:
+            e = EtcdEntry(*entry[::-1], value_in_json=value_in_json)
+            if e.value:
+                yield e
+
+    def watch_prefix(self, key, timeout=0, value_in_json=False):
+        timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"),
+                                  value=str.encode(json.dumps({"status": "TIMEOUT",
+                                                               "type": "TIMEOUT"})),
+                                  value_in_json=value_in_json)
+
+        event_queue = queue.Queue()
+
+        def add_event_to_queue(event):
+            for e in event.events:
+                if e.value:
+                    event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json))
+
+        self.client.add_watch_prefix_callback(key, add_event_to_queue)
+
+        while True:
+            try:
+                while True:
+                    v = event_queue.get(timeout=timeout)
+                    yield v
+            except queue.Empty:
+                event_queue.put(copy.deepcopy(timeout_event))
+
+
+class PsuedoEtcdEntry(EtcdEntry):
+    def __init__(self, key, value, value_in_json=False):
+        super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json)

ucloud/common/helpers.py

@@ -6,21 +6,7 @@ import json
 from ipaddress import ip_address
 
 from os.path import join as join_path
-
+from . import logger
-
-def create_package_loggers(packages, base_path, mode="a"):
-    loggers = {}
-    for pkg in packages:
-        logger = logging.getLogger(pkg)
-        logger_handler = logging.FileHandler(
-            join_path(base_path, "{}.txt".format(pkg)),
-            mode=mode
-        )
-        logger.setLevel(logging.DEBUG)
-        logger_handler.setFormatter(logging.Formatter(fmt="%(asctime)s: %(levelname)s - %(message)s",
-                                                      datefmt="%d-%b-%y %H:%M:%S"))
-        logger.addHandler(logger_handler)
-        loggers[pkg] = logger
 
 
 # TODO: Should be removed as soon as migration
@@ -35,7 +21,7 @@ def get_ipv4_address():
         except socket.timeout:
             address = "127.0.0.1"
         except Exception as e:
-            logging.getLogger().exception(e)
+            logger.exception(e)
             address = "127.0.0.1"
         else:
             address = s.getsockname()[0]
@@ -49,6 +35,6 @@ def get_ipv6_address():
         content = json.loads(r.content.decode("utf-8"))
         ip = ip_address(content["ip"]).exploded
     except Exception as e:
-        logging.exception(e)
+        logger.exception(e)
     else:
         return ip

ucloud/common/request.py

@@ -2,8 +2,7 @@ import json
 from os.path import join
 from uuid import uuid4
 
-from etcd3_wrapper.etcd3_wrapper import PsuedoEtcdEntry
+from .etcd_wrapper import PsuedoEtcdEntry
-
 from .classes import SpecificEtcdEntryBase
 
 

ucloud/config.py

@@ -1,133 +1,53 @@
+import configparser
+import os
+import logging
+
 from ucloud.common.host import HostPool
 from ucloud.common.request import RequestPool
 from ucloud.common.vm import VmPool
 from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler
+from ucloud.common.etcd_wrapper import Etcd3Wrapper
 
-# Replacing decouple inline
+log = logging.getLogger('ucloud.config')
-import configparser
-import os
-import os.path
-
-import logging
-log = logging.getLogger("ucloud.config")
-
-conf_name = "ucloud.conf"
-
-try:
-    conf_dir = os.environ["UCLOUD_CONF_DIR"]
-except KeyError:
-    conf_dir = "/etc/ucloud"
 
+conf_name = 'ucloud.conf'
+conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud')
 config_file = os.path.join(conf_dir, conf_name)
 
-config = configparser.ConfigParser()
+config = configparser.ConfigParser(allow_no_value=True)
 
-try:
+if os.access(config_file, os.R_OK):
     config.read(config_file)
-except FileNotFoundError:
+else:
-    log.warn("Configuration file not found - using defaults")
+    log.warning('Configuration file not found - using defaults')
-
-
-################################################################################
-# ETCD3 support
-import etcd3
-import json
-import queue
-import copy
-from collections import namedtuple
-
-PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"])
-
-class EtcdEntry:
-    # key: str
-    # value: str
-
-    def __init__(self, meta, value, value_in_json=False):
-        self.key = meta.key.decode("utf-8")
-        self.value = value.decode("utf-8")
-
-        if value_in_json:
-            self.value = json.loads(self.value)
-
-class Etcd3Wrapper:
-    def __init__(self, *args, **kwargs):
-        self.client = etcd3.client(*args, **kwargs)
-
-    def get(self, *args, value_in_json=False, **kwargs):
-        _value, _key = self.client.get(*args, **kwargs)
-        if _key is None or _value is None:
-            return None
-        return EtcdEntry(_key, _value, value_in_json=value_in_json)
-
-    def put(self, *args, value_in_json=False, **kwargs):
-        _key, _value = args
-        if value_in_json:
-            _value = json.dumps(_value)
-
-        if not isinstance(_key, str):
-            _key = _key.decode("utf-8")
-
-        return self.client.put(_key, _value, **kwargs)
-
-    def get_prefix(self, *args, value_in_json=False, **kwargs):
-        r = self.client.get_prefix(*args, **kwargs)
-        for entry in r:
-            e = EtcdEntry(*entry[::-1], value_in_json=value_in_json)
-            if e.value:
-                yield e
-
-    def watch_prefix(self, key, timeout=0, value_in_json=False):
-        timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"),
-                                  value=str.encode(json.dumps({"status": "TIMEOUT",
-                                                               "type": "TIMEOUT"})),
-                                  value_in_json=value_in_json)
-
-        event_queue = queue.Queue()
-
-        def add_event_to_queue(event):
-            for e in event.events:
-                if e.value:
-                    event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json))
-
-        self.client.add_watch_prefix_callback(key, add_event_to_queue)
-
-        while True:
-            try:
-                while True:
-                    v = event_queue.get(timeout=timeout)
-                    yield v
-            except queue.Empty:
-                event_queue.put(copy.deepcopy(timeout_event))
-
-
-class PsuedoEtcdEntry(EtcdEntry):
-    def __init__(self, key, value, value_in_json=False):
-        super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json)
-
 
 etcd_wrapper_args = ()
 etcd_wrapper_kwargs = {
-    'host':      config['etcd']['ETCD_URL'],
+    'host':      config['etcd']['url'],
-    'port':      config['etcd']['ETCD_PORT'],
+    'port':      config['etcd']['port'],
-    'ca_cert':   config['etcd']['CA_CERT'],
+    'ca_cert':   config['etcd']['ca_cert'],
-    'cert_cert': config['etcd']['CERT_CERT'],
+    'cert_cert': config['etcd']['cert_cert'],
-    'cert_key':  config['etcd']['CERT_KEY']
+    'cert_key':  config['etcd']['cert_key']
 }
 
 etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
 
-host_pool = HostPool(etcd_client, config['etcd']['HOST_PREFIX'])
+host_pool = HostPool(etcd_client, config['etcd']['host_prefix'])
-vm_pool = VmPool(etcd_client, config['etcd']['VM_PREFIX'])
+vm_pool = VmPool(etcd_client, config['etcd']['vm_prefix'])
-request_pool = RequestPool(etcd_client, config['etcd']['REQUEST_PREFIX'])
+request_pool = RequestPool(etcd_client, config['etcd']['request_prefix'])
 
 running_vms = []
 
-__storage_backend = config['storage']["STORAGE_BACKEND"]
+__storage_backend = config['storage']['backend']
-if __storage_backend == "filesystem":
+if __storage_backend == 'filesystem':
-    image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=config['storage']["VM_DIR"],
+    image_storage_handler = FileSystemBasedImageStorageHandler(
-                                                               image_base=config['storage']["IMAGE_DIR"])
+        vm_base=config['storage']['vm_dir'],
-elif __storage_backend == "ceph":
+        image_base=config['storage']['image_dir']
-    image_storage_handler = CEPHBasedImageStorageHandler(vm_base=config['storage']["CEPH_VM_POOL"],
+    )
-                                                         image_base=config['storage']["CEPH_IMAGE_POOL"])
+elif __storage_backend == 'ceph':
+    image_storage_handler = CEPHBasedImageStorageHandler(
+        vm_base=config['storage']['ceph_vm_pool'],
+        image_base=config['storage']['ceph_image_pool']
+    )
 else:
-    raise Exception("Unknown Image Storage Handler")
+    raise Exception('Unknown Image Storage Handler')

ucloud/filescanner/main.py

@@ -3,6 +3,7 @@ import os
 import pathlib
 import subprocess as sp
 import time
+import sys
 from uuid import uuid4
 
 from . import logger
@@ -19,7 +20,6 @@ def getxattr(file, attr):
                                  '--absolute-names'], stderr=sp.DEVNULL)
         value = value.decode("utf-8")
     except sp.CalledProcessError as e:
-        logger.exception(e)
         value = None
 
     return value
@@ -63,14 +63,13 @@ try:
     sp.check_output(['which', 'getfattr'])
     sp.check_output(['which', 'setfattr'])
 except Exception as e:
-    logger.exception(e)
+    logger.error("You don't seems to have both getfattr and setfattr")
-    print('Make sure you have getfattr and setfattr available')
+    sys.exit(1)
-    exit(1)
 
 
 def main():
-    BASE_DIR    = config['storage']["FILE_DIR"]
+    BASE_DIR    = config['storage']['file_dir']
-    FILE_PREFIX = config['storage']["FILE_PREFIX"]
+    FILE_PREFIX = config['etcd']['file_prefix']
 
     # Recursively Get All Files and Folder below BASE_DIR
     files = glob.glob("{}/**".format(BASE_DIR), recursive=True)
@@ -79,7 +78,7 @@ def main():
     files = list(filter(os.path.isfile, files))
 
     untracked_files = list(
-        filter(lambda f: not bool(getxattr(f, "user.utracked")), files)
+        filter(lambda f: not bool(getxattr(f, "utracked")), files)
     )
 
     tracked_files = list(
@@ -89,7 +88,8 @@ def main():
         file_id = uuid4()
 
         # Get Username
-        owner = pathlib.Path(file).parts[3]
+        owner = pathlib.Path(file).parts[len(pathlib.Path(BASE_DIR).parts)]
+        
         # Get Creation Date of File
         # Here, we are assuming that ctime is creation time
         # which is mostly not true.
@@ -101,9 +101,7 @@ def main():
         # Compute sha512 sum
         sha_sum = sha512sum(file)
 
-        # File Path excluding base and username
+        file_path = pathlib.Path(file).parts[-1]
-        file_path = pathlib.Path(file).parts[4:]
-        file_path = os.path.join(*file_path)
 
         # Create Entry
         entry_key = os.path.join(FILE_PREFIX, str(file_id))
@@ -115,10 +113,10 @@ def main():
             "size": size
         }
 
-        print("Tracking {}".format(file))
+        logger.info("Tracking %s", file)
         # Insert Entry
         etcd_client.put(entry_key, entry_value, value_in_json=True)
-        setxattr(file, "user.utracked", True)
+        setxattr(file, "utracked", True)
 
 
 if __name__ == "__main__":

ucloud/host/main.py

@@ -1,9 +1,10 @@
 import argparse
 import multiprocessing as mp
 import time
+import sys
 
-from etcd3_wrapper import Etcd3Wrapper
+from os.path import isdir
-
+from ucloud.common.etcd_wrapper import Etcd3Wrapper
 from ucloud.common.request import RequestEntry, RequestType
 from ucloud.config import (vm_pool, request_pool,
                     etcd_client, running_vms,
@@ -18,7 +19,7 @@ from ucloud.host import logger
 def update_heartbeat(hostname):
     """Update Last HeartBeat Time for :param hostname: in etcd"""
     client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
-    host_pool = HostPool(client, config['etcd']['HOST_PREFIX'])
+    host_pool = HostPool(client, config['etcd']['host_prefix'])
     this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
 
     while True:
@@ -72,9 +73,11 @@ def maintenance(host):
                 running_vms.remove(_vm)
 
 def check():
-    if config['etcd']['STORAGE_BACKEND'] == 'filesystem' and not isdir(config['etcd']['VM_DIR']):
+    if config['storage']['backend'] == 'filesystem' and \
+        not isdir(config['storage']['vm_dir']):
+        
         print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned"
-              " in .env file must exists. But, it don't.")
+              " in /etc/ucloud/ucloud.conf file must exists. But, it don't.")
         sys.exit(1)
 
 
@@ -84,7 +87,7 @@ def main(hostname):
 
     heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,))
 
-    host_pool = HostPool(etcd_client, config['etcd']['HOST_PREFIX'])
+    host_pool = HostPool(etcd_client, config['etcd']['host_prefix'])
     host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
     assert host is not None, "No such host with name = {}".format(hostname)
 
@@ -106,8 +109,8 @@ def main(hostname):
     # beat updating mechanism in separated thread
 
     for events_iterator in [
-        etcd_client.get_prefix(config['etcd']['REQUEST_PREFIX'], value_in_json=True),
+        etcd_client.get_prefix(config['etcd']['request_prefix'], value_in_json=True),
-        etcd_client.watch_prefix(config['etcd']['REQUEST_PREFIX'], timeout=10, value_in_json=True),
+        etcd_client.watch_prefix(config['etcd']['request_prefix'], timeout=10, value_in_json=True),
     ]:
         for request_event in events_iterator:
             request_event = RequestEntry(request_event)

ucloud/host/virtualmachine.py

@@ -46,7 +46,7 @@ def delete_network_interface(iface):
 
 
 def resolve_network(network_name, network_owner):
-    network = etcd_client.get(join_path(config['etcd']["NETWORK_PREFIX"],
+    network = etcd_client.get(join_path(config['etcd']['network_prefix'],
                                         network_owner,
                                         network_name),
                               value_in_json=True)
@@ -179,7 +179,7 @@ def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, mi
     for network_mac_and_tap in vm_networks:
         network_name, mac, tap = network_mac_and_tap
 
-        _key = os.path.join(config['etcd']['NETWORK_PREFIX'], vm_entry.owner, network_name)
+        _key = os.path.join(config['etcd']['network_prefix'], vm_entry.owner, network_name)
         network = etcd_client.get(_key, value_in_json=True)
         network_type = network.value["type"]
         network_id = str(network.value["id"])
@@ -187,7 +187,7 @@ def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, mi
 
         if network_type == "vxlan":
             tap = create_vxlan_br_tap(_id=network_id,
-                                      _dev=config['etcd']["VXLAN_PHY_DEV"],
+                                      _dev=config['network']['vxlan_phy_dev'],
                                       tap_id=tap,
                                       ip=network_ipv6)
             update_radvd_conf(etcd_client)
@@ -303,13 +303,13 @@ def transfer(request_event):
     _host, _port = request_event.parameters["host"], request_event.parameters["port"]
     _uuid = request_event.uuid
     _destination = request_event.destination_host_key
-    vm = get_vm(running_vms, join_path(config['etcd']['VM_PREFIX'], _uuid))
+    vm = get_vm(running_vms, join_path(config['etcd']['vm_prefix'], _uuid))
 
     if vm:
         tunnel = sshtunnel.SSHTunnelForwarder(
             _host,
-            ssh_username=config['ssh']["ssh_username"],
+            ssh_username=config['ssh']['username'],
-            ssh_pkey=config['ssh']["SSH_PRIVATEKEY"],
+            ssh_pkey=config['ssh']['private_key_path'],
             remote_bind_address=("127.0.0.1", _port),
             ssh_proxy_enabled=True,
             ssh_proxy=(_host, 22)
@@ -373,7 +373,7 @@ def launch_vm(vm_entry, migration=False, migration_port=None, destination_host_k
                 parameters={"host": get_ipv6_address(), "port": migration_port},
                 uuid=vm_entry.uuid,
                 destination_host_key=destination_host_key,
-                request_prefix=config['etcd']["REQUEST_PREFIX"]
+                request_prefix=config['etcd']['request_prefix']
             )
             request_pool.put(r)
         else:

ucloud/imagescanner/main.py

@@ -20,9 +20,9 @@ def qemu_img_type(path):
 
 def check():
     """ check whether settings are sane, refuse to start if they aren't """
-    if config['etcd']['STORAGE_BACKEND'] == 'filesystem' and not isdir(config['etcd']['IMAGE_DIR']):
+    if config['storage']['backend'] == 'filesystem' and not isdir(config['storage']['image_dir']):
         print("You have set STORAGE_BACKEND to filesystem, but "
-              "{} does not exist. Refusing to start".format(config['etcd']['IMAGE_DIR']))
+              "{} does not exist. Refusing to start".format(config['storage']['image_dir']))
         sys.exit(1)
 
     try:
@@ -34,7 +34,7 @@ def check():
 
 def main():
     # We want to get images entries that requests images to be created
-    images = etcd_client.get_prefix(config['etcd']['IMAGE_PREFIX'], value_in_json=True)
+    images = etcd_client.get_prefix(config['etcd']['image_prefix'], value_in_json=True)
     images_to_be_created = list(filter(lambda im: im.value['status'] == 'TO_BE_CREATED', images))
 
     for image in images_to_be_created:
@@ -43,9 +43,10 @@ def main():
             image_owner = image.value['owner']
             image_filename = image.value['filename']
             image_store_name = image.value['store_name']
-            image_full_path = join_path(config['etcd']['BASE_DIR'], image_owner, image_filename)
+            image_full_path = join_path(config['storage']['file_dir'], image_owner, image_filename)
 
-            image_stores = etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True)
+            image_stores = etcd_client.get_prefix(config['etcd']['image_store_prefix'],
+                                                  value_in_json=True)
             user_image_store = next(filter(
                 lambda s, store_name=image_store_name: s.value["name"] == store_name,
                 image_stores

ucloud/metadata/main.py

@@ -43,7 +43,8 @@ class Root(Resource):
         if not data:
             return {'message': 'Metadata for such VM does not exists.'}, 404
         else:
-            etcd_key = os.path.join(config['etcd']['USER_PREFIX'], data.value['owner_realm'],
+            etcd_key = os.path.join(config['etcd']['user_prefix'],
+                                    data.value['owner_realm'],
                                     data.value['owner'], 'key')
             etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True)
             user_personal_ssh_keys = [key.value for key in etcd_entry]

ucloud/scheduler/helper.py

@@ -106,7 +106,7 @@ def assign_host(vm):
     r = RequestEntry.from_scratch(type=RequestType.StartVM,
                                   uuid=vm.uuid,
                                   hostname=vm.hostname,
-                                  request_prefix=config['etcd']['REQUEST_PREFIX'])
+                                  request_prefix=config['etcd']['request_prefix'])
     request_pool.put(r)
 
     vm.log.append("VM scheduled for starting")

ucloud/scheduler/main.py

@@ -18,8 +18,8 @@ def main():
     pending_vms = []
 
     for request_iterator in [
-        etcd_client.get_prefix(config['etcd']['REQUEST_PREFIX'], value_in_json=True),
+        etcd_client.get_prefix(config['etcd']['request_prefix'], value_in_json=True),
-        etcd_client.watch_prefix(config['etcd']['REQUEST_PREFIX'], timeout=5, value_in_json=True),
+        etcd_client.watch_prefix(config['etcd']['request_prefix'], timeout=5, value_in_json=True),
     ]:
         for request_event in request_iterator:
             request_entry = RequestEntry(request_event)
@@ -46,7 +46,7 @@ def main():
                     r = RequestEntry.from_scratch(type="ScheduleVM",
                                                   uuid=pending_vm_entry.uuid,
                                                   hostname=pending_vm_entry.hostname,
-                                                  request_prefix=config['etcd']['REQUEST_PREFIX'])
+                                                  request_prefix=config['etcd']['request_prefix'])
                     request_pool.put(r)
 
             elif request_entry.type == RequestType.ScheduleVM:
@@ -72,7 +72,7 @@ def main():
                         r = RequestEntry.from_scratch(type=RequestType.InitVMMigration,
                                                       uuid=request_entry.uuid,
                                                       destination=request_entry.destination,
-                                                      request_prefix=config['etcd']['REQUEST_PREFIX'])
+                                                      request_prefix=config['etcd']['request_prefix'])
                         request_pool.put(r)
 
                 # If the Request is about a VM that just want to get started/created