Efforts to make ucloud a python package

ucloud/common/__init__.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger(__name__)

ucloud/common/classes.py

@@ -0,0 +1,26 @@
+from etcd3_wrapper import EtcdEntry
+
+
+class SpecificEtcdEntryBase:
+    def __init__(self, e: EtcdEntry):
+        self.key = e.key
+
+        for k in e.value.keys():
+            self.__setattr__(k, e.value[k])
+
+    def original_keys(self):
+        r = dict(self.__dict__)
+        if "key" in r:
+            del r["key"]
+        return r
+
+    @property
+    def value(self):
+        return self.original_keys()
+
+    @value.setter
+    def value(self, v):
+        self.__dict__ = v
+
+    def __repr__(self):
+        return str(dict(self.__dict__))

ucloud/common/counters.py

@@ -0,0 +1,21 @@
+from etcd3_wrapper import Etcd3Wrapper
+
+
+def increment_etcd_counter(etcd_client: Etcd3Wrapper, key):
+    kv = etcd_client.get(key)
+
+    if kv:
+        counter = int(kv.value)
+        counter = counter + 1
+    else:
+        counter = 1
+
+    etcd_client.put(key, str(counter))
+    return counter
+
+
+def get_etcd_counter(etcd_client: Etcd3Wrapper, key):
+    kv = etcd_client.get(key)
+    if kv:
+        return int(kv.value)
+    return None

ucloud/common/helpers.py

@@ -0,0 +1,54 @@
+import logging
+import socket
+import requests
+import json
+
+from ipaddress import ip_address
+
+from os.path import join as join_path
+
+
+def create_package_loggers(packages, base_path, mode="a"):
+    loggers = {}
+    for pkg in packages:
+        logger = logging.getLogger(pkg)
+        logger_handler = logging.FileHandler(
+            join_path(base_path, "{}.txt".format(pkg)),
+            mode=mode
+        )
+        logger.setLevel(logging.DEBUG)
+        logger_handler.setFormatter(logging.Formatter(fmt="%(asctime)s: %(levelname)s - %(message)s",
+                                                      datefmt="%d-%b-%y %H:%M:%S"))
+        logger.addHandler(logger_handler)
+        loggers[pkg] = logger
+
+
+# TODO: Should be removed as soon as migration
+#       mechanism is finalized inside ucloud
+def get_ipv4_address():
+    # If host is connected to internet
+    #   Return IPv4 address of machine
+    # Otherwise, return 127.0.0.1
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+        try:
+            s.connect(("8.8.8.8", 80))
+        except socket.timeout:
+            address = "127.0.0.1"
+        except Exception as e:
+            logging.getLogger().exception(e)
+            address = "127.0.0.1"
+        else:
+            address = s.getsockname()[0]
+
+    return address
+
+
+def get_ipv6_address():
+    try:
+        r = requests.get("https://api6.ipify.org?format=json")
+        content = json.loads(r.content.decode("utf-8"))
+        ip = ip_address(content["ip"]).exploded
+    except Exception as e:
+        logging.exception(e)
+    else:
+        return ip

ucloud/common/host.py

@@ -0,0 +1,67 @@
+import time
+from datetime import datetime
+from os.path import join
+from typing import List
+
+from .classes import SpecificEtcdEntryBase
+
+
+class HostStatus:
+    """Possible Statuses of ucloud host."""
+
+    alive = "ALIVE"
+    dead = "DEAD"
+
+
+class HostEntry(SpecificEtcdEntryBase):
+    """Represents Host Entry Structure and its supporting methods."""
+
+    def __init__(self, e):
+        self.specs = None  # type: dict
+        self.hostname = None  # type: str
+        self.status = None  # type: str
+        self.last_heartbeat = None  # type: str
+
+        super().__init__(e)
+
+    def update_heartbeat(self):
+        self.status = HostStatus.alive
+        self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S")
+
+    def is_alive(self):
+        last_heartbeat = datetime.strptime(self.last_heartbeat, "%Y-%m-%d %H:%M:%S")
+        delta = datetime.now() - last_heartbeat
+        if delta.total_seconds() > 60:
+            return False
+        return True
+
+    def declare_dead(self):
+        self.status = HostStatus.dead
+        self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S")
+
+
+class HostPool:
+    def __init__(self, etcd_client, host_prefix):
+        self.client = etcd_client
+        self.prefix = host_prefix
+
+    @property
+    def hosts(self) -> List[HostEntry]:
+        _hosts = self.client.get_prefix(self.prefix, value_in_json=True)
+        return [HostEntry(host) for host in _hosts]
+
+    def get(self, key):
+        if not key.startswith(self.prefix):
+            key = join(self.prefix, key)
+        v = self.client.get(key, value_in_json=True)
+        if v:
+            return HostEntry(v)
+        return None
+
+    def put(self, obj: HostEntry):
+        self.client.put(obj.key, obj.value, value_in_json=True)
+
+    def by_status(self, status, _hosts=None):
+        if _hosts is None:
+            _hosts = self.hosts
+        return list(filter(lambda x: x.status == status, _hosts))

ucloud/common/request.py

@@ -0,0 +1,46 @@
+import json
+from os.path import join
+from uuid import uuid4
+
+from etcd3_wrapper.etcd3_wrapper import PsuedoEtcdEntry
+
+from .classes import SpecificEtcdEntryBase
+
+
+class RequestType:
+    CreateVM = "CreateVM"
+    ScheduleVM = "ScheduleVM"
+    StartVM = "StartVM"
+    StopVM = "StopVM"
+    InitVMMigration = "InitVMMigration"
+    TransferVM = "TransferVM"
+    DeleteVM = "DeleteVM"
+
+
+class RequestEntry(SpecificEtcdEntryBase):
+
+    def __init__(self, e):
+        self.type = None  # type: str
+        self.migration = None  # type: bool
+        self.destination = None  # type: str
+        self.uuid = None  # type: str
+        self.hostname = None  # type: str
+        super().__init__(e)
+
+    @classmethod
+    def from_scratch(cls, request_prefix, **kwargs):
+        e = PsuedoEtcdEntry(join(request_prefix, uuid4().hex),
+                            value=json.dumps(kwargs).encode("utf-8"), value_in_json=True)
+        return cls(e)
+
+
+class RequestPool:
+    def __init__(self, etcd_client, request_prefix):
+        self.client = etcd_client
+        self.prefix = request_prefix
+
+    def put(self, obj: RequestEntry):
+        if not obj.key.startswith(self.prefix):
+            obj.key = join(self.prefix, obj.key)
+
+        self.client.put(obj.key, obj.value, value_in_json=True)

ucloud/common/storage_handlers.py

@@ -0,0 +1,158 @@
+import shutil
+import subprocess as sp
+import os
+import stat
+
+from abc import ABC
+from . import logger
+from os.path import join as join_path
+
+
+class ImageStorageHandler(ABC):
+    def __init__(self, image_base, vm_base):
+        self.image_base = image_base
+        self.vm_base = vm_base
+
+    def import_image(self, image_src, image_dest, protect=False):
+        """Put an image at the destination
+        :param src: An Image file
+        :param dest: A path where :param src: is to be put.
+        :param protect: If protect is true then the dest is protect (readonly etc)
+        The obj must exist on filesystem.
+        """
+
+        raise NotImplementedError()
+
+    def make_vm_image(self, image_path, path):
+        """Copy image from src to dest
+
+        :param src: A path
+        :param dest: A path
+
+        src and destination must be on same storage system i.e both on file system or both on CEPH etc.
+        """
+        raise NotImplementedError()
+
+    def resize_vm_image(self, path, size):
+        """Resize image located at :param path:
+        :param path: The file which is to be resized
+        :param size: Size must be in Megabytes
+        """
+        raise NotImplementedError()
+
+    def delete_vm_image(self, path):
+        raise NotImplementedError()
+
+    def execute_command(self, command, report=True):
+        command = list(map(str, command))
+        try:
+            output = sp.check_output(command, stderr=sp.PIPE)
+        except Exception as e:
+            if report:
+                print(e)
+                logger.exception(e)
+            return False
+        return True
+
+    def vm_path_string(self, path):
+        raise NotImplementedError()
+
+    def qemu_path_string(self, path):
+        raise NotImplementedError()
+
+    def is_vm_image_exists(self, path):
+        raise NotImplementedError()
+
+
+class FileSystemBasedImageStorageHandler(ImageStorageHandler):
+    def import_image(self, src, dest, protect=False):
+        dest = join_path(self.image_base, dest)
+        try:
+            shutil.copy(src, dest)
+            if protect:
+                os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+        except Exception as e:
+            logger.exception(e)
+            return False
+        return True
+
+    def make_vm_image(self, src, dest):
+        src = join_path(self.image_base, src)
+        dest = join_path(self.vm_base, dest)
+        try:
+            shutil.copy(src, dest)
+        except Exception as e:
+            logger.exception(e)
+            return False
+        return True
+
+    def resize_vm_image(self, path, size):
+        path = join_path(self.vm_base, path)
+        command = ["qemu-img", "resize", "-f", "raw", path, "{}M".format(size)]
+        if self.execute_command(command):
+            return True
+        else:
+            self.delete_vm_image(path)
+            return False
+
+    def delete_vm_image(self, path):
+        path = join_path(self.vm_base, path)
+        try:
+            os.remove(path)
+        except Exception as e:
+            logger.exception(e)
+            return False
+        return True
+
+    def vm_path_string(self, path):
+        return join_path(self.vm_base, path)
+
+    def qemu_path_string(self, path):
+        return self.vm_path_string(path)
+
+    def is_vm_image_exists(self, path):
+        path = join_path(self.vm_base, path)
+        command = ["ls", path]
+        return self.execute_command(command, report=False)
+
+
+class CEPHBasedImageStorageHandler(ImageStorageHandler):
+    def import_image(self, src, dest, protect=False):
+        dest = join_path(self.image_base, dest)
+        command = ["rbd", "import", src, dest]
+        if protect:
+            snap_create_command = ["rbd", "snap", "create", "{}@protected".format(dest)]
+            snap_protect_command = ["rbd", "snap", "protect", "{}@protected".format(dest)]
+
+            return self.execute_command(command) and self.execute_command(snap_create_command) and\
+                self.execute_command(snap_protect_command)
+
+        return self.execute_command(command)
+
+    def make_vm_image(self, src, dest):
+        src = join_path(self.image_base, src)
+        dest = join_path(self.vm_base, dest)
+
+        command = ["rbd", "clone", "{}@protected".format(src), dest]
+        return self.execute_command(command)
+
+    def resize_vm_image(self, path, size):
+        path = join_path(self.vm_base, path)
+        command = ["rbd", "resize", path, "--size", size]
+        return self.execute_command(command)
+
+    def delete_vm_image(self, path):
+        path = join_path(self.vm_base, path)
+        command = ["rbd", "rm", path]
+        return self.execute_command(command)
+
+    def vm_path_string(self, path):
+        return join_path(self.vm_base, path)
+
+    def qemu_path_string(self, path):
+        return "rbd:{}".format(self.vm_path_string(path))
+
+    def is_vm_image_exists(self, path):
+        path = join_path(self.vm_base, path)
+        command = ["rbd", "info", path]
+        return self.execute_command(command, report=False)

ucloud/common/vm.py

@@ -0,0 +1,90 @@
+from contextlib import contextmanager
+from datetime import datetime
+from os.path import join
+
+from .classes import SpecificEtcdEntryBase
+
+
+class VMStatus:
+    stopped = "STOPPED"  # After requested_shutdown
+    killed = "KILLED"  # either host died or vm died itself
+    running = "RUNNING"
+    error = "ERROR"  # An error occurred that cannot be resolved automatically
+
+
+class VMEntry(SpecificEtcdEntryBase):
+
+    def __init__(self, e):
+        self.owner = None  # type: str
+        self.specs = None  # type: dict
+        self.hostname = None  # type: str
+        self.status = None  # type: str
+        self.image_uuid = None  # type: str
+        self.log = None  # type: list
+        self.in_migration = None  # type: bool
+
+        super().__init__(e)
+
+    @property
+    def uuid(self):
+        return self.key.split("/")[-1]
+
+    def declare_killed(self):
+        self.hostname = ""
+        self.in_migration = False
+        if self.status == VMStatus.running:
+            self.status = VMStatus.killed
+
+    def declare_stopped(self):
+        self.hostname = ""
+        self.in_migration = False
+        self.status = VMStatus.stopped
+
+    def add_log(self, msg):
+        self.log = self.log[:5]
+        self.log.append("{} - {}".format(datetime.now().isoformat(), msg))
+
+
+class VmPool:
+    def __init__(self, etcd_client, vm_prefix):
+        self.client = etcd_client
+        self.prefix = vm_prefix
+
+    @property
+    def vms(self):
+        _vms = self.client.get_prefix(self.prefix, value_in_json=True)
+        return [VMEntry(vm) for vm in _vms]
+
+    def by_host(self, host, _vms=None):
+        if _vms is None:
+            _vms = self.vms
+        return list(filter(lambda x: x.hostname == host, _vms))
+
+    def by_status(self, status, _vms=None):
+        if _vms is None:
+            _vms = self.vms
+        return list(filter(lambda x: x.status == status, _vms))
+
+    def except_status(self, status, _vms=None):
+        if _vms is None:
+            _vms = self.vms
+        return list(filter(lambda x: x.status != status, _vms))
+
+    def get(self, key):
+        if not key.startswith(self.prefix):
+            key = join(self.prefix, key)
+        v = self.client.get(key, value_in_json=True)
+        if v:
+            return VMEntry(v)
+        return None
+
+    def put(self, obj: VMEntry):
+        self.client.put(obj.key, obj.value, value_in_json=True)
+
+    @contextmanager
+    def get_put(self, key) -> VMEntry:
+        # Updates object at key on exit
+        obj = self.get(key)
+        yield obj
+        if obj:
+            self.put(obj)