Remove ucloud_common and put its files under ucloud.common subpackage.

Remove individual config.py used by every component and put them into single config.py ucloud/config.py
Use /etc/ucloud/ucloud.conf for Environment Variables
Refactoring and a lot of it
Make ucloud repo a package and different components of ucloud a subpackage for avoiding code duplication.
Improved logging.

common/classes.py

@@ -0,0 +1,48 @@
+from decouple import Config, RepositoryEnv, UndefinedValueError
+from etcd3_wrapper import EtcdEntry
+
+
+class EnvironmentVariables:
+    def __init__(self, env_file):
+        try:
+            env_config = Config(RepositoryEnv(env_file))
+        except FileNotFoundError:
+            print("{} does not exists".format(env_file))
+            exit(1)
+        else:
+            self.config = env_config
+
+    def get(self, *args, **kwargs):
+        """Return value of var from env_vars"""
+        try:
+            value = self.config.get(*args, **kwargs)
+        except UndefinedValueError as e:
+            print(e)
+            exit(1)
+        else:
+            return value
+
+
+class SpecificEtcdEntryBase:
+    def __init__(self, e: EtcdEntry):
+        self.key = e.key
+
+        for k in e.value.keys():
+            self.__setattr__(k, e.value[k])
+
+    def original_keys(self):
+        r = dict(self.__dict__)
+        if "key" in r:
+            del r["key"]
+        return r
+
+    @property
+    def value(self):
+        return self.original_keys()
+
+    @value.setter
+    def value(self, v):
+        self.__dict__ = v
+
+    def __repr__(self):
+        return str(dict(self.__dict__))

common/counters.py

@@ -0,0 +1,21 @@
+from etcd3_wrapper import Etcd3Wrapper
+
+
+def increment_etcd_counter(etcd_client: Etcd3Wrapper, key):
+    kv = etcd_client.get(key)
+
+    if kv:
+        counter = int(kv.value)
+        counter = counter + 1
+    else:
+        counter = 1
+
+    etcd_client.put(key, str(counter))
+    return counter
+
+
+def get_etcd_counter(etcd_client: Etcd3Wrapper, key):
+    kv = etcd_client.get(key)
+    if kv:
+        return int(kv.value)
+    return None

common/helpers.py

@@ -0,0 +1,39 @@
+import logging
+import socket
+
+from os.path import join as join_path
+
+
+def create_package_loggers(packages, base_path, mode="a"):
+    loggers = {}
+    for pkg in packages:
+        logger = logging.getLogger(pkg)
+        logger_handler = logging.FileHandler(
+            join_path(base_path, "{}.txt".format(pkg)),
+            mode=mode
+        )
+        logger.setLevel(logging.DEBUG)
+        logger_handler.setFormatter(logging.Formatter(fmt="%(asctime)s: %(levelname)s - %(message)s",
+                                                      datefmt="%d-%b-%y %H:%M:%S"))
+        logger.addHandler(logger_handler)
+        loggers[pkg] = logger
+
+
+# TODO: Should be removed as soon as migration
+#       mechanism is finalized inside ucloud
+def get_ipv4_address():
+    # If host is connected to internet
+    #   Return IPv4 address of machine
+    # Otherwise, return 127.0.0.1
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+        try:
+            s.connect(("8.8.8.8", 80))
+        except socket.timeout:
+            address = "127.0.0.1"
+        except Exception as e:
+            logging.getLogger().exception(e)
+            address = "127.0.0.1"
+        else:
+            address = s.getsockname()[0]
+
+    return address

common/host.py

@@ -0,0 +1,67 @@
+import time
+from datetime import datetime
+from os.path import join
+from typing import List
+
+from .classes import SpecificEtcdEntryBase
+
+
+class HostStatus:
+    """Possible Statuses of ucloud host."""
+
+    alive = "ALIVE"
+    dead = "DEAD"
+
+
+class HostEntry(SpecificEtcdEntryBase):
+    """Represents Host Entry Structure and its supporting methods."""
+
+    def __init__(self, e):
+        self.specs = None  # type: dict
+        self.hostname = None  # type: str
+        self.status = None  # type: str
+        self.last_heartbeat = None  # type: str
+
+        super().__init__(e)
+
+    def update_heartbeat(self):
+        self.status = HostStatus.alive
+        self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S")
+
+    def is_alive(self):
+        last_heartbeat = datetime.strptime(self.last_heartbeat, "%Y-%m-%d %H:%M:%S")
+        delta = datetime.now() - last_heartbeat
+        if delta.total_seconds() > 60:
+            return False
+        return True
+
+    def declare_dead(self):
+        self.status = HostStatus.dead
+        self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S")
+
+
+class HostPool:
+    def __init__(self, etcd_client, host_prefix):
+        self.client = etcd_client
+        self.prefix = host_prefix
+
+    @property
+    def hosts(self) -> List[HostEntry]:
+        _hosts = self.client.get_prefix(self.prefix, value_in_json=True)
+        return [HostEntry(host) for host in _hosts]
+
+    def get(self, key):
+        if not key.startswith(self.prefix):
+            key = join(self.prefix, key)
+        v = self.client.get(key, value_in_json=True)
+        if v:
+            return HostEntry(v)
+        return None
+
+    def put(self, obj: HostEntry):
+        self.client.put(obj.key, obj.value, value_in_json=True)
+
+    def by_status(self, status, _hosts=None):
+        if _hosts is None:
+            _hosts = self.hosts
+        return list(filter(lambda x: x.status == status, _hosts))

common/request.py

@@ -0,0 +1,46 @@
+import json
+from os.path import join
+from uuid import uuid4
+
+from etcd3_wrapper.etcd3_wrapper import PsuedoEtcdEntry
+
+from .classes import SpecificEtcdEntryBase
+
+
+class RequestType:
+    CreateVM = "CreateVM"
+    ScheduleVM = "ScheduleVM"
+    StartVM = "StartVM"
+    StopVM = "StopVM"
+    InitVMMigration = "InitVMMigration"
+    TransferVM = "TransferVM"
+    DeleteVM = "DeleteVM"
+
+
+class RequestEntry(SpecificEtcdEntryBase):
+
+    def __init__(self, e):
+        self.type = None  # type: str
+        self.migration = None  # type: bool
+        self.destination = None  # type: str
+        self.uuid = None  # type: str
+        self.hostname = None  # type: str
+        super().__init__(e)
+
+    @classmethod
+    def from_scratch(cls, request_prefix, **kwargs):
+        e = PsuedoEtcdEntry(join(request_prefix, uuid4().hex),
+                            value=json.dumps(kwargs).encode("utf-8"), value_in_json=True)
+        return cls(e)
+
+
+class RequestPool:
+    def __init__(self, etcd_client, request_prefix):
+        self.client = etcd_client
+        self.prefix = request_prefix
+
+    def put(self, obj: RequestEntry):
+        if not obj.key.startswith(self.prefix):
+            obj.key = join(self.prefix, obj.key)
+
+        self.client.put(obj.key, obj.value, value_in_json=True)

common/vm.py

@@ -0,0 +1,110 @@
+from contextlib import contextmanager
+from datetime import datetime
+from os.path import join
+
+from .classes import SpecificEtcdEntryBase
+
+
+class VMStatus:
+    # Must be only assigned to brand new VM
+    requested_new = "REQUESTED_NEW"
+
+    # Only Assigned to already created vm
+    requested_start = "REQUESTED_START"
+
+    # These all are for running vms
+    requested_shutdown = "REQUESTED_SHUTDOWN"
+    requested_migrate = "REQUESTED_MIGRATE"
+    requested_delete = "REQUESTED_DELETE"
+    # either its image is not found or user requested
+    # to delete it
+    deleted = "DELETED"
+
+    stopped = "STOPPED"  # After requested_shutdown
+    killed = "KILLED"  # either host died or vm died itself
+
+    running = "RUNNING"
+
+    error = "ERROR"  # An error occurred that cannot be resolved automatically
+
+
+class VMEntry(SpecificEtcdEntryBase):
+
+    def __init__(self, e):
+        self.owner = None  # type: str
+        self.specs = None  # type: dict
+        self.hostname = None  # type: str
+        self.status = None  # type: str
+        self.image_uuid = None  # type: str
+        self.log = None  # type: list
+        self.in_migration = None  # type: bool
+
+        super().__init__(e)
+
+    @property
+    def uuid(self):
+        return self.key.split("/")[-1]
+
+    def declare_killed(self):
+        self.hostname = ""
+        self.in_migration = False
+        if self.status == VMStatus.running:
+            self.status = VMStatus.killed
+
+    def declare_stopped(self):
+        self.hostname = ""
+        self.in_migration = False
+        self.status = VMStatus.stopped
+
+    def add_log(self, msg):
+        self.log = self.log[:5]
+        self.log.append("{} - {}".format(datetime.now().isoformat(), msg))
+
+    @property
+    def path(self):
+        return "rbd:uservms/{}".format(self.uuid)
+
+
+class VmPool:
+    def __init__(self, etcd_client, vm_prefix):
+        self.client = etcd_client
+        self.prefix = vm_prefix
+
+    @property
+    def vms(self):
+        _vms = self.client.get_prefix(self.prefix, value_in_json=True)
+        return [VMEntry(vm) for vm in _vms]
+
+    def by_host(self, host, _vms=None):
+        if _vms is None:
+            _vms = self.vms
+        return list(filter(lambda x: x.hostname == host, _vms))
+
+    def by_status(self, status, _vms=None):
+        if _vms is None:
+            _vms = self.vms
+        return list(filter(lambda x: x.status == status, _vms))
+
+    def except_status(self, status, _vms=None):
+        if _vms is None:
+            _vms = self.vms
+        return list(filter(lambda x: x.status != status, _vms))
+
+    def get(self, key):
+        if not key.startswith(self.prefix):
+            key = join(self.prefix, key)
+        v = self.client.get(key, value_in_json=True)
+        if v:
+            return VMEntry(v)
+        return None
+
+    def put(self, obj: VMEntry):
+        self.client.put(obj.key, obj.value, value_in_json=True)
+
+    @contextmanager
+    def get_put(self, key) -> VMEntry:
+        # Updates object at key on exit
+        obj = self.get(key)
+        yield obj
+        if obj:
+            self.put(obj)