Refactoring, Removal of most global vars, config default path is ~/ucloud/

ucloud/common/etcd_wrapper.py

@@ -4,40 +4,63 @@ import queue
 import copy
 
 from collections import namedtuple
+from functools import wraps
+
+from . import logger
+
+PseudoEtcdMeta = namedtuple('PseudoEtcdMeta', ['key'])
 
-PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"])
 
 class EtcdEntry:
     # key: str
     # value: str
 
     def __init__(self, meta, value, value_in_json=False):
-        self.key = meta.key.decode("utf-8")
-        self.value = value.decode("utf-8")
+        self.key = meta.key.decode('utf-8')
+        self.value = value.decode('utf-8')
 
         if value_in_json:
             self.value = json.loads(self.value)
 
+
+def readable_errors(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            func(*args, **kwargs)
+        except etcd3.exceptions.ConnectionFailedError as err:
+            raise etcd3.exceptions.ConnectionFailedError('etcd connection failed') from err
+        except etcd3.exceptions.ConnectionTimeoutError as err:
+            raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout') from err
+        except Exception:
+            print('Some error occurred, most probably it is etcd that is erroring out.')
+            logger.exception('Some etcd error occurred')
+    return wrapper
+
+
 class Etcd3Wrapper:
     def __init__(self, *args, **kwargs):
         self.client = etcd3.client(*args, **kwargs)
 
+    @readable_errors
     def get(self, *args, value_in_json=False, **kwargs):
         _value, _key = self.client.get(*args, **kwargs)
         if _key is None or _value is None:
             return None
         return EtcdEntry(_key, _value, value_in_json=value_in_json)
 
+    @readable_errors
     def put(self, *args, value_in_json=False, **kwargs):
         _key, _value = args
         if value_in_json:
             _value = json.dumps(_value)
 
         if not isinstance(_key, str):
-            _key = _key.decode("utf-8")
+            _key = _key.decode('utf-8')
 
         return self.client.put(_key, _value, **kwargs)
 
+    @readable_errors
     def get_prefix(self, *args, value_in_json=False, **kwargs):
         r = self.client.get_prefix(*args, **kwargs)
         for entry in r:
@@ -45,10 +68,11 @@ class Etcd3Wrapper:
             if e.value:
                 yield e
 
+    @readable_errors
     def watch_prefix(self, key, timeout=0, value_in_json=False):
-        timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"),
-                                  value=str.encode(json.dumps({"status": "TIMEOUT",
-                                                               "type": "TIMEOUT"})),
+        timeout_event = EtcdEntry(PseudoEtcdMeta(key=b'TIMEOUT'),
+                                  value=str.encode(json.dumps({'status': 'TIMEOUT',
+                                                               'type': 'TIMEOUT'})),
                                   value_in_json=value_in_json)
 
         event_queue = queue.Queue()
@@ -71,4 +95,4 @@ class Etcd3Wrapper:
 
 class PsuedoEtcdEntry(EtcdEntry):
     def __init__(self, key, value, value_in_json=False):
-        super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json)
+        super().__init__(PseudoEtcdMeta(key=key.encode('utf-8')), value, value_in_json=value_in_json)

ucloud/common/network.py

@@ -0,0 +1,60 @@
+import subprocess as sp
+import random
+import logging
+import socket
+from contextlib import closing
+
+logger = logging.getLogger(__name__)
+
+
+def random_bytes(num=6):
+    return [random.randrange(256) for _ in range(num)]
+
+
+def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'):
+    mac = random_bytes()
+    if oui:
+        if type(oui) == str:
+            oui = [int(chunk) for chunk in oui.split(separator)]
+        mac = oui + random_bytes(num=6 - len(oui))
+    else:
+        if multicast:
+            mac[0] |= 1  # set bit 0
+        else:
+            mac[0] &= ~1  # clear bit 0
+        if uaa:
+            mac[0] &= ~(1 << 1)  # clear bit 1
+        else:
+            mac[0] |= 1 << 1  # set bit 1
+    return separator.join(byte_fmt % b for b in mac)
+
+
+def create_dev(script, _id, dev, ip=None):
+    command = [script, _id, dev]
+    if ip:
+        command.append(ip)
+    try:
+        output = sp.check_output(command, stderr=sp.PIPE)
+    except Exception as e:
+        print(e)
+        return None
+    else:
+        return output.decode('utf-8').strip()
+
+
+def delete_network_interface(iface):
+    try:
+        sp.check_output(['ip', 'link', 'del', iface])
+    except Exception:
+        logger.exception('Interface Deletion failed')
+
+
+def find_free_port():
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        try:
+            s.bind(('', 0))
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        except Exception:
+            return None
+        else:
+            return s.getsockname()[1]

ucloud/common/storage_handlers.py

@@ -7,6 +7,8 @@ from abc import ABC
 from . import logger
 from os.path import join as join_path
 
+from ucloud.settings import settings as config
+
 
 class ImageStorageHandler(ABC):
     def __init__(self, image_base, vm_base):
@@ -156,3 +158,19 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
         path = join_path(self.vm_base, path)
         command = ["rbd", "info", path]
         return self.execute_command(command, report=False)
+
+
+def get_storage_handler():
+    __storage_backend = config['storage']['storage_backend']
+    if __storage_backend == 'filesystem':
+        return FileSystemBasedImageStorageHandler(
+            vm_base=config['storage']['vm_dir'],
+            image_base=config['storage']['image_dir']
+        )
+    elif __storage_backend == 'ceph':
+        return CEPHBasedImageStorageHandler(
+            vm_base=config['storage']['ceph_vm_pool'],
+            image_base=config['storage']['ceph_image_pool']
+        )
+    else:
+        raise Exception('Unknown Image Storage Handler')