Merge branch 'master' of code.ungleich.ch:uncloud/uncloud

conf/uncloud.conf

@@ -1,6 +1,7 @@
 [etcd]
 url = localhost
 port = 2379
+base_prefix = /
 ca_cert
 cert_cert
 cert_key
@@ -9,3 +10,4 @@ cert_key
 name = replace_me
 realm = replace_me
 seed = replace_me
+api_server = http://localhost:5000

scripts/uncloud

@@ -3,10 +3,14 @@ import logging
 import sys
 import importlib
 import argparse
-import multiprocessing as mp
 
 from uncloud import UncloudException
-from contextlib import suppress
+
+# the components that use etcd
+ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
+
+ALL_COMPONENTS = ETCD_COMPONENTS.copy()
+ALL_COMPONENTS.append('cli')
 
 
 def exception_hook(exc_type, exc_value, exc_traceback):
@@ -27,32 +31,48 @@ if __name__ == '__main__':
     subparsers = arg_parser.add_subparsers(dest='command')
 
     parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
+    parent_parser.add_argument('--debug', '-d',
+                               action='store_true',
+                               default=False,
                                help='More verbose logging')
+    parent_parser.add_argument('--conf-dir', '-c',
+                               help='Configuration directory')
 
-    for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner',
+    etcd_parser = argparse.ArgumentParser(add_help=False)
-                      'metadata', 'configure', 'cli']:
+    etcd_parser.add_argument('--etcd-host')
+    etcd_parser.add_argument('--etcd-port')
+    etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
+    etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
+    etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
+
+    for component in ALL_COMPONENTS:
         mod = importlib.import_module('uncloud.{}.main'.format(component))
         parser = getattr(mod, 'arg_parser')
-        subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
+
+        if component in ETCD_COMPONENTS:
+            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser])
+        else:
+            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
 
     args = arg_parser.parse_args()
     if not args.command:
         arg_parser.print_help()
     else:
-
-        # if we start etcd in seperate process with default settings
-        # i.e inheriting few things from parent process etcd3 module
-        # errors out, so the following command configure multiprocessing
-        # module to not inherit anything from parent.
-        mp.set_start_method('spawn')
         arguments = vars(args)
+        name = arguments.pop('command')
+        mod = importlib.import_module('uncloud.{}.main'.format(name))
+        main = getattr(mod, 'main')
+
+        # If the component requires etcd3, we import it and catch the
+        # etcd3.exceptions.ConnectionFailedError
+        if name in ETCD_COMPONENTS:
+            import etcd3
+
         try:
-            name = arguments.pop('command')
+            main(arguments)
-            mod = importlib.import_module('uncloud.{}.main'.format(name))
-            main = getattr(mod, 'main')
-            main(**arguments)
         except UncloudException as err:
             logger.error(err)
+            sys.exit(1)
         except Exception as err:
             logger.exception(err)
+            sys.exit(1)

uncloud/api/common_fields.py

@@ -1,7 +1,7 @@
 import os
 
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 
 
 class Optional:

uncloud/api/create_image_store.py

@@ -3,18 +3,18 @@ import os
 
 from uuid import uuid4
 
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 
 data = {
-    "is_public": True,
+    'is_public': True,
-    "type": "ceph",
+    'type': 'ceph',
-    "name": "images",
+    'name': 'images',
-    "description": "first ever public image-store",
+    'description': 'first ever public image-store',
-    "attributes": {"list": [], "key": [], "pool": "images"},
+    'attributes': {'list': [], 'key': [], 'pool': 'images'},
 }
 
 shared.etcd_client.put(
-    os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex),
+    os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex),
     json.dumps(data),
 )

uncloud/api/helper.py

@@ -1,14 +1,13 @@
 import binascii
 import ipaddress
 import random
-import subprocess as sp
 import logging
 import requests
 
 from pyotp import TOTP
 
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 
 logger = logging.getLogger(__name__)
 

uncloud/api/main.py

@@ -10,11 +10,12 @@ from flask import Flask, request
 from flask_restful import Resource, Api
 from werkzeug.exceptions import HTTPException
 
+from uncloud.common.shared import shared
+
 from uncloud.common import counters
 from uncloud.common.vm import VMStatus
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.settings import settings
+from uncloud.common.settings import settings
-from uncloud.shared import shared
 from . import schemas
 from .helper import generate_mac, mac2ipv6
 from uncloud import UncloudException
@@ -41,7 +42,7 @@ def handle_exception(e):
 
 
 class CreateVM(Resource):
-    '''API Request to Handle Creation of VM'''
+    """API Request to Handle Creation of VM"""
 
     @staticmethod
     def post():
@@ -59,7 +60,7 @@ class CreateVM(Resource):
             macs = [generate_mac() for _ in range(len(data['network']))]
             tap_ids = [
                 counters.increment_etcd_counter(
-                    shared.etcd_client, '/v1/counter/tap'
+                    shared.etcd_client, settings['etcd']['tap_counter']
                 )
                 for _ in range(len(data['network']))
             ]
@@ -289,18 +290,16 @@ class ListUserFiles(Resource):
                 settings['etcd']['file_prefix'], value_in_json=True
             )
             return_files = []
-            user_files = list(
+            user_files = [f for f in files if f.value['owner'] == data['name']]
-                filter(
-                    lambda f: f.value['owner'] == data['name'], files
-                )
-            )
             for file in user_files:
-                return_files.append(
+                file_uuid = file.key.split('/')[-1]
-                    {
+                file = file.value
-                        'filename': file.value['filename'],
+                file['uuid'] = file_uuid
-                        'uuid': file.key.split('/')[-1],
+
-                    }
+                file.pop('sha512sum', None)
-                )
+                file.pop('owner', None)
+
+                return_files.append(file)
             return {'message': return_files}, 200
         else:
             return validator.get_errors(), 400
@@ -472,7 +471,7 @@ class CreateNetwork(Resource):
 
             network_entry = {
                 'id': counters.increment_etcd_counter(
-                    shared.etcd_client, '/v1/counter/vxlan'
+                    shared.etcd_client, settings['etcd']['vxlan_counter']
                 ),
                 'type': data['type'],
             }
@@ -564,7 +563,10 @@ api.add_resource(ListHost, '/host/list')
 api.add_resource(CreateNetwork, '/network/create')
 
 
-def main(debug=False, port=None):
+def main(arguments):
+    debug = arguments['debug']
+    port = arguments['port']
+
     try:
         image_stores = list(
             shared.etcd_client.get_prefix(
@@ -594,12 +596,6 @@ def main(debug=False, port=None):
     #     )
 
     try:
-        app.run(host='::',
+        app.run(host='::', port=port, debug=debug)
-                port=port,
-                debug=debug)
     except OSError as e:
         raise UncloudException('Failed to start Flask: {}'.format(e))
-
-
-if __name__ == '__main__':
-    main()

uncloud/api/schemas.py

@@ -21,8 +21,8 @@ import bitmath
 
 from uncloud.common.host import HostStatus
 from uncloud.common.vm import VMStatus
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 from . import helper, logger
 from .common_fields import Field, VmUUIDField
 from .helper import check_otp, resolve_vm_name

uncloud/cli/helper.py

@@ -5,15 +5,24 @@ import binascii
 
 from pyotp import TOTP
 from os.path import join as join_path
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 
 
 def get_otp_parser():
     otp_parser = argparse.ArgumentParser('otp')
-    otp_parser.add_argument('--name', default=settings['client']['name'])
+    try:
-    otp_parser.add_argument('--realm', default=settings['client']['realm'])
+        name = settings['client']['name']
-    otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'],
+        realm = settings['client']['realm']
-                            dest='token', metavar='SEED')
+        seed = settings['client']['seed']
+    except Exception:
+        otp_parser.add_argument('--name', required=True)
+        otp_parser.add_argument('--realm', required=True)
+        otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED')
+    else:
+        otp_parser.add_argument('--name', default=name)
+        otp_parser.add_argument('--realm', default=realm)
+        otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED')
+
     return otp_parser
 
 

uncloud/cli/image.py

@@ -12,7 +12,7 @@ class ImageParser(BaseParser):
         p = self.subparser.add_parser('create', **kwargs)
         p.add_argument('--name', required=True)
         p.add_argument('--uuid', required=True)
-        p.add_argument('--image-store-name', default='image_store')
+        p.add_argument('--image-store', required=True, dest='image_store')
 
     def list(self, **kwargs):
         self.subparser.add_parser('list', **kwargs)

uncloud/cli/main.py

@@ -12,12 +12,12 @@ for component in ['user', 'host', 'image', 'network', 'vm']:
     subparser.add_parser(name=parser.prog, parents=[parser])
 
 
-def main(**kwargs):
+def main(arguments):
-    if not kwargs['subcommand']:
+    if not arguments['subcommand']:
         arg_parser.print_help()
     else:
-        name = kwargs.pop('subcommand')
+        name = arguments.pop('subcommand')
-        kwargs.pop('debug')
+        arguments.pop('debug')
         mod = importlib.import_module('uncloud.cli.{}'.format(name))
         _main = getattr(mod, 'main')
-        _main(**kwargs)
+        _main(**arguments)

uncloud/common/etcd_wrapper.py

@@ -1,24 +1,21 @@
 import etcd3
 import json
-import queue
-import copy
-from uncloud import UncloudException
 
-from collections import namedtuple
 from functools import wraps
 
-from . import logger
+from uncloud import UncloudException
-
+from uncloud.common import logger
-PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"])
 
 
 class EtcdEntry:
-    # key: str
+    def __init__(self, meta_or_key, value, value_in_json=False):
-    # value: str
+        if hasattr(meta_or_key, 'key'):
-
+            # if meta has attr 'key' then get it
-    def __init__(self, meta, value, value_in_json=False):
+            self.key = meta_or_key.key.decode('utf-8')
-        self.key = meta.key.decode("utf-8")
+        else:
-        self.value = value.decode("utf-8")
+            # otherwise meta is the 'key'
+            self.key = meta_or_key
+        self.value = value.decode('utf-8')
 
         if value_in_json:
             self.value = json.loads(self.value)
@@ -29,18 +26,12 @@ def readable_errors(func):
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
-        except etcd3.exceptions.ConnectionFailedError as err:
+        except etcd3.exceptions.ConnectionFailedError:
-            raise UncloudException(
+            raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?')
-                "Cannot connect to etcd: is etcd running as configured in uncloud.conf?"
-            )
         except etcd3.exceptions.ConnectionTimeoutError as err:
-            raise etcd3.exceptions.ConnectionTimeoutError(
+            raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err
-                "etcd connection timeout."
-            ) from err
         except Exception:
-            logger.exception(
+            logger.exception('Some etcd error occured. See syslog for details.')
-                "Some etcd error occured. See syslog for details."
-            )
 
     return wrapper
 
@@ -64,55 +55,21 @@ class Etcd3Wrapper:
             _value = json.dumps(_value)
 
         if not isinstance(_key, str):
-            _key = _key.decode("utf-8")
+            _key = _key.decode('utf-8')
 
         return self.client.put(_key, _value, **kwargs)
 
     @readable_errors
-    def get_prefix(self, *args, value_in_json=False, **kwargs):
+    def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs):
-        r = self.client.get_prefix(*args, **kwargs)
+        event_iterator = self.client.get_prefix(*args, **kwargs)
-        for entry in r:
+        for e in event_iterator:
-            e = EtcdEntry(*entry[::-1], value_in_json=value_in_json)
+            yield EtcdEntry(*e[::-1], value_in_json=value_in_json)
-            if e.value:
-                yield e
 
     @readable_errors
-    def watch_prefix(self, key, timeout=0, value_in_json=False):
+    def watch_prefix(self, key, raise_exception=True, value_in_json=False):
-        timeout_event = EtcdEntry(
+        event_iterator, cancel = self.client.watch_prefix(key)
-            PseudoEtcdMeta(key=b"TIMEOUT"),
+        for e in event_iterator:
-            value=str.encode(
+            if hasattr(e, '_event'):
-                json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"})
+                e = e._event
-            ),
+                if e.type == e.PUT:
-            value_in_json=value_in_json,
+                    yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)
-        )
-
-        event_queue = queue.Queue()
-
-        def add_event_to_queue(event):
-            if hasattr(event, "events"):
-                for e in event.events:
-                    if e.value:
-                        event_queue.put(
-                            EtcdEntry(
-                                e, e.value, value_in_json=value_in_json
-                            )
-                        )
-
-        self.client.add_watch_prefix_callback(key, add_event_to_queue)
-
-        while True:
-            try:
-                while True:
-                    v = event_queue.get(timeout=timeout)
-                    yield v
-            except queue.Empty:
-                event_queue.put(copy.deepcopy(timeout_event))
-
-
-class PsuedoEtcdEntry(EtcdEntry):
-    def __init__(self, key, value, value_in_json=False):
-        super().__init__(
-            PseudoEtcdMeta(key=key.encode("utf-8")),
-            value,
-            value_in_json=value_in_json,
-        )

uncloud/common/network.py

@@ -1,8 +1,6 @@
 import subprocess as sp
 import random
 import logging
-import socket
-from contextlib import closing
 
 logger = logging.getLogger(__name__)
 

uncloud/common/request.py

@@ -2,8 +2,8 @@ import json
 from os.path import join
 from uuid import uuid4
 
-from .etcd_wrapper import PsuedoEtcdEntry
+from uncloud.common.etcd_wrapper import EtcdEntry
-from .classes import SpecificEtcdEntryBase
+from uncloud.common.classes import SpecificEtcdEntryBase
 
 
 class RequestType:
@@ -29,11 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase):
 
     @classmethod
     def from_scratch(cls, request_prefix, **kwargs):
-        e = PsuedoEtcdEntry(
+        e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex),
-            join(request_prefix, uuid4().hex),
+                      value=json.dumps(kwargs).encode('utf-8'), value_in_json=True)
-            value=json.dumps(kwargs).encode("utf-8"),
-            value_in_json=True,
-        )
         return cls(e)
 
 

uncloud/common/settings.py

@@ -4,8 +4,8 @@ import sys
 import os
 
 from datetime import datetime
-
 from uncloud.common.etcd_wrapper import Etcd3Wrapper
+from os.path import join as join_path
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +16,7 @@ class CustomConfigParser(configparser.RawConfigParser):
             result = super().__getitem__(key)
         except KeyError as err:
             raise KeyError(
-                "Key '{}' not found in configuration. Make sure you configure uncloud.".format(
+                'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format(
                     key
                 )
             ) from err
@@ -25,40 +25,41 @@ class CustomConfigParser(configparser.RawConfigParser):
 
 
 class Settings(object):
-    def __init__(self, config_key="/uncloud/config/"):
+    def __init__(self):
-        conf_name = "uncloud.conf"
+        conf_name = 'uncloud.conf'
-        conf_dir = os.environ.get(
+        conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
-            "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/")
+        self.config_file = join_path(conf_dir, conf_name)
-        )
-        self.config_file = os.path.join(conf_dir, conf_name)
-        self.config_parser = CustomConfigParser(allow_no_value=True)
-        self.config_key = config_key
 
         # this is used to cache config from etcd for 1 minutes. Without this we
         # would make a lot of requests to etcd which slows down everything.
         self.last_config_update = datetime.fromtimestamp(0)
 
-        self.read_internal_values()
+        self.config_parser = CustomConfigParser(allow_no_value=True)
+        self.config_parser.add_section('etcd')
+        self.config_parser.set('etcd', 'base_prefix', '/')
+
         try:
             self.config_parser.read(self.config_file)
         except Exception as err:
-            logger.error("%s", err)
+            logger.error('%s', err)
+
+        self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
+
+        self.read_internal_values()
 
     def get_etcd_client(self):
         args = tuple()
         try:
             kwargs = {
-                "host": self.config_parser.get("etcd", "url"),
+                'host': self.config_parser.get('etcd', 'url'),
-                "port": self.config_parser.get("etcd", "port"),
+                'port': self.config_parser.get('etcd', 'port'),
-                "ca_cert": self.config_parser.get("etcd", "ca_cert"),
+                'ca_cert': self.config_parser.get('etcd', 'ca_cert'),
-                "cert_cert": self.config_parser.get(
+                'cert_cert': self.config_parser.get('etcd', 'cert_cert'),
-                    "etcd", "cert_cert"
+                'cert_key': self.config_parser.get('etcd', 'cert_key'),
-                ),
-                "cert_key": self.config_parser.get("etcd", "cert_key"),
             }
         except configparser.Error as err:
             raise configparser.Error(
-                "{} in config file {}".format(
+                '{} in config file {}'.format(
                     err.message, self.config_file
                 )
             ) from err
@@ -67,8 +68,8 @@ class Settings(object):
                 wrapper = Etcd3Wrapper(*args, **kwargs)
             except Exception as err:
                 logger.error(
-                    "etcd connection not successfull. Please check your config file."
+                    'etcd connection not successfull. Please check your config file.'
-                    "\nDetails: %s\netcd connection parameters: %s",
+                    '\nDetails: %s\netcd connection parameters: %s',
                     err,
                     kwargs,
                 )
@@ -77,17 +78,20 @@ class Settings(object):
                 return wrapper
 
     def read_internal_values(self):
+        base_prefix = self['etcd']['base_prefix']
         self.config_parser.read_dict(
             {
-                "etcd": {
+                'etcd': {
-                    "file_prefix": "/files/",
+                    'file_prefix': join_path(base_prefix, 'files/'),
-                    "host_prefix": "/hosts/",
+                    'host_prefix': join_path(base_prefix, 'hosts/'),
-                    "image_prefix": "/images/",
+                    'image_prefix': join_path(base_prefix, 'images/'),
-                    "image_store_prefix": "/imagestore/",
+                    'image_store_prefix': join_path(base_prefix, 'imagestore/'),
-                    "network_prefix": "/networks/",
+                    'network_prefix': join_path(base_prefix, 'networks/'),
-                    "request_prefix": "/requests/",
+                    'request_prefix': join_path(base_prefix, 'requests/'),
-                    "user_prefix": "/users/",
+                    'user_prefix': join_path(base_prefix, 'users/'),
-                    "vm_prefix": "/vms/",
+                    'vm_prefix': join_path(base_prefix, 'vms/'),
+                    'vxlan_counter': join_path(base_prefix, 'counters/vxlan'),
+                    'tap_counter': join_path(base_prefix, 'counters/tap')
                 }
             }
         )
@@ -95,15 +99,13 @@ class Settings(object):
     def read_config_file_values(self, config_file):
         try:
             # Trying to read configuration file
-            with open(config_file, "r") as config_file_handle:
+            with open(config_file) as config_file_handle:
                 self.config_parser.read_file(config_file_handle)
         except FileNotFoundError:
-            sys.exit(
+            sys.exit('Configuration file {} not found!'.format(config_file))
-                "Configuration file {} not found!".format(config_file)
-            )
         except Exception as err:
             logger.exception(err)
-            sys.exit("Error occurred while reading configuration file")
+            sys.exit('Error occurred while reading configuration file')
 
     def read_values_from_etcd(self):
         etcd_client = self.get_etcd_client()
@@ -113,7 +115,7 @@ class Settings(object):
                 self.config_parser.read_dict(config_from_etcd.value)
                 self.last_config_update = datetime.utcnow()
             else:
-                raise KeyError("Key '{}' not found in etcd. Please configure uncloud.".format(self.config_key))
+                raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key))
 
     def __getitem__(self, key):
         # Allow failing to read from etcd if we have
@@ -121,9 +123,8 @@ class Settings(object):
         if key not in self.config_parser.sections():
             try:
                 self.read_values_from_etcd()
-            except KeyError as e:
+            except KeyError:
                 pass
-
         return self.config_parser[key]
 
 

uncloud/common/shared.py

@@ -1,4 +1,4 @@
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 from uncloud.common.vm import VmPool
 from uncloud.common.host import HostPool
 from uncloud.common.request import RequestPool

uncloud/common/storage_handlers.py

@@ -7,7 +7,7 @@ from abc import ABC
 from . import logger
 from os.path import join as join_path
 
-from uncloud.settings import settings as config
+from uncloud.common.settings import settings as config
 
 
 class ImageStorageHandler(ABC):

uncloud/configure/main.py

@@ -1,8 +1,8 @@
 import os
 import argparse
 
-from uncloud.settings import settings
+from uncloud.common.settings import settings
-from uncloud.shared import shared
+from uncloud.common.shared import shared
 
 arg_parser = argparse.ArgumentParser('configure', add_help=False)
 configure_subparsers = arg_parser.add_subparsers(dest='subcommand')
@@ -40,18 +40,14 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
 
 
 def update_config(section, kwargs):
-    uncloud_config = shared.etcd_client.get(
+    uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
-        settings.config_key, value_in_json=True
-    )
     if not uncloud_config:
         uncloud_config = {}
     else:
         uncloud_config = uncloud_config.value
 
     uncloud_config[section] = kwargs
-    shared.etcd_client.put(
+    shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
-        settings.config_key, uncloud_config, value_in_json=True
-    )
 
 
 def main(**kwargs):

uncloud/docs/README.md

@@ -0,0 +1,12 @@
+# uncloud docs
+
+## Requirements
+1. Python3
+2. Sphinx
+
+## Usage
+Run `make build` to build docs.
+
+Run `make clean` to remove build directory.
+
+Run `make publish` to push build dir to https://ungleich.ch/ucloud/

uncloud/filescanner/main.py

@@ -4,15 +4,16 @@ import pathlib
 import subprocess as sp
 import time
 import argparse
+import bitmath
 
 from uuid import uuid4
 
 from . import logger
-from uncloud.settings import settings
+from uncloud.common.settings import settings
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-
 
 arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
+arg_parser.add_argument('--hostname', required=True)
 
 
 def sha512sum(file: str):
@@ -28,66 +29,58 @@ def sha512sum(file: str):
     if not isinstance(file, str):
         raise TypeError
     try:
-        output = sp.check_output(["sha512sum", file], stderr=sp.PIPE)
+        output = sp.check_output(['sha512sum', file], stderr=sp.PIPE)
     except sp.CalledProcessError as e:
-        error = e.stderr.decode("utf-8")
+        error = e.stderr.decode('utf-8')
-        if "No such file or directory" in error:
+        if 'No such file or directory' in error:
             raise FileNotFoundError from None
     else:
-        output = output.decode("utf-8").strip()
+        output = output.decode('utf-8').strip()
-        output = output.split(" ")
+        output = output.split(' ')
         return output[0]
     return None
 
 
-def track_file(file, base_dir):
+def track_file(file, base_dir, host):
-    file_id = uuid4()
+    file_path = file.relative_to(base_dir)
-
+    file_str = str(file)
     # Get Username
-    owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)]
+    try:
+        owner = file_path.parts[0]
+    except IndexError:
+        pass
+    else:
+        file_path = file_path.relative_to(owner)
+        creation_date = time.ctime(os.stat(file_str).st_ctime)
 
-    # Get Creation Date of File
+        entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
-    # Here, we are assuming that ctime is creation time
+        entry_value = {
-    # which is mostly not true.
+            'filename': str(file_path),
-    creation_date = time.ctime(os.stat(file).st_ctime)
+            'owner': owner,
+            'sha512sum': sha512sum(file_str),
+            'creation_date': creation_date,
+            'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()),
+            'host': host
+        }
 
-    file_path = pathlib.Path(file).parts[-1]
+        logger.info('Tracking %s', file_str)
 
-    # Create Entry
+        shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
-    entry_key = os.path.join(
-        settings["etcd"]["file_prefix"], str(file_id)
-    )
-    entry_value = {
-        "filename": file_path,
-        "owner": owner,
-        "sha512sum": sha512sum(file),
-        "creation_date": creation_date,
-        "size": os.path.getsize(file),
-    }
-
-    logger.info("Tracking %s", file)
-
-    shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
-    os.setxattr(file, "user.utracked", b"True")
 
 
-def main(debug=False):
+def main(arguments):
-    base_dir = settings["storage"]["file_dir"]
+    hostname = arguments['hostname']
-
+    base_dir = settings['storage']['file_dir']
     # Recursively Get All Files and Folder below BASE_DIR
-    files = glob.glob("{}/**".format(base_dir), recursive=True)
+    files = glob.glob('{}/**'.format(base_dir), recursive=True)
+    files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
 
-    # Retain only Files
+    # Files that are already tracked
-    files = [file for file in files if os.path.isfile(file)]
+    tracked_files = [
-
+        pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
-    untracked_files = []
+        for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
-    for file in files:
+        if f.value['host'] == hostname
-        try:
+    ]
-            os.getxattr(file, "user.utracked")
+    untracked_files = set(files) - set(tracked_files)
-        except OSError:
+    for file in untracked_files:
-            track_file(file, base_dir)
+        track_file(file, base_dir, hostname)
-            untracked_files.append(file)
-
-
-if __name__ == "__main__":
-    main()

uncloud/hack/hackcloud/.gitignore

@@ -0,0 +1,3 @@
+*.iso
+radvdpid
+foo

uncloud/hack/hackcloud/ifup.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+echo $@ >> foo
+
+dev=$1; shift
+
+# bridge is setup from outside
+ip link set dev "$dev" master ${bridge}
+ip link set dev "$dev" up

uncloud/hack/hackcloud/net.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+netid=100
+dev=wlp2s0
+dev=wlp0s20f3
+dev=wlan0
+
+ip=2a0a:e5c1:111:888::42/64
+vxlandev=vxlan${netid}
+bridgedev=br${netid}
+
+ip -6 link add ${vxlandev} type vxlan \
+    id ${netid} \
+    dstport 4789 \
+    group ff05::${netid} \
+    dev ${dev} \
+    ttl 5
+
+ip link set ${vxlandev} up
+
+ip link add ${bridgedev} type bridge
+ip link set ${bridgedev} up
+
+ip addr add ${ip} dev ${bridgedev}

uncloud/hack/hackcloud/radvd.conf

@@ -0,0 +1,13 @@
+interface br100
+{
+  AdvSendAdvert on;
+  MinRtrAdvInterval 3;
+  MaxRtrAdvInterval 5;
+  AdvDefaultLifetime 3600;
+
+  prefix 2a0a:e5c1:111:888::/64 {
+  };
+
+  RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; };
+  DNSSL place7.ungleich.ch {  AdvDNSSLLifetime 6000; } ;
+};

uncloud/hack/hackcloud/radvd.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+radvd -C ./radvd.conf  -n -p ./radvdpid

uncloud/hack/hackcloud/vm.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+
+vmid=$1; shift
+
+qemu=/usr/bin/qemu-system-x86_64
+
+accel=kvm
+accel=tcg
+
+memory=1024
+cores=2
+uuid=732e08c7-84f8-4d43-9571-263db4f80080
+
+export bridge=br100
+
+$qemu -name uc${vmid} \
+      -machine pc,accel=${accel} \
+      -m ${memory} \
+      -smp ${cores} \
+      -uuid ${uuid} \
+      -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \
+      -netdev tap,id=netmain,script=./ifup.sh \
+      -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e
+
+
+
+exit 0
+
+-S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-17-one-24992/master-key.aes
+-machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off
+
+-m 2048
+-realtime mlock=off
+-smp 1,sockets=1,cores=1,threads=1
+-uuid 732e08c7-84f8-4d43-9571-263db4f80080 -no-user-config \
+      -nodefaults
+-chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-17-one-24992/monitor.sock,server,nowait
+-mon chardev=charmonitor,id=monitor,mode=control
+-rtc base=utc -no-shutdown
+-boot strict=on
+-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2
+-drive file=rbd:ssd/one-292-24992-0:id=libvirt:auth_supported=cephx\;none:mon_host=ceph1\:6789\;ceph2\:6789\;ceph3\:6789,format=raw,if=none,id=drive-virtio-disk0,cache=none
+-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
+-drive file=/var/lib/one//datastores/104/24992/disk.1,format=raw,if=none,id=drive-ide0-0-0,readonly=on
+-device ide-cd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0
+-netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38
+-device virtio-net-pci,netdev=hostnet0,id=net0,mac=02:00:f0:a9:c4:4e,bus=pci.0,addr=0x3
+-vnc [::]:4414 -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on

uncloud/host/main.py

@@ -5,8 +5,8 @@ import time
 from uuid import uuid4
 
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 from uncloud.common.vm import VMStatus
 from uncloud.vmm import VMM
 from os.path import join as join_path
@@ -33,10 +33,10 @@ def maintenance(host):
     vmm = VMM()
     running_vms = vmm.discover()
     for vm_uuid in running_vms:
-        if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running":
+        if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
             logger.debug('VM {} is running on {}'.format(vm_uuid, host))
             vm = shared.vm_pool.get(
-                join_path(settings["etcd"]["vm_prefix"], vm_uuid)
+                join_path(settings['etcd']['vm_prefix'], vm_uuid)
             )
             vm.status = VMStatus.running
             vm.vnc_socket = vmm.get_vnc(vm_uuid)
@@ -44,20 +44,21 @@ def maintenance(host):
             shared.vm_pool.put(vm)
 
 
-def main(hostname, debug=False):
+def main(arguments):
+    hostname = arguments['hostname']
     host_pool = shared.host_pool
     host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
 
     # Does not yet exist, create it
     if not host:
         host_key = join_path(
-            settings["etcd"]["host_prefix"], uuid4().hex
+            settings['etcd']['host_prefix'], uuid4().hex
         )
         host_entry = {
-            "specs": "",
+            'specs': '',
-            "hostname": hostname,
+            'hostname': hostname,
-            "status": "DEAD",
+            'status': 'DEAD',
-            "last_heartbeat": "",
+            'last_heartbeat': '',
         }
         shared.etcd_client.put(
             host_key, host_entry, value_in_json=True
@@ -70,54 +71,54 @@ def main(hostname, debug=False):
         heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,))
         heartbeat_updating_process.start()
     except Exception as e:
-        raise Exception("uncloud-host heartbeat updating mechanism is not working") from e
+        raise Exception('uncloud-host heartbeat updating mechanism is not working') from e
 
-    for events_iterator in [
+    # The below while True is neccessary for gracefully handling leadership transfer and temporary
-        shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True),
+    # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
-        shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True)
+    # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
-    ]:
+    # which ends the loop immediately. So, having it inside infinite loop we try again and again to
-        for request_event in events_iterator:
+    # get prefix until either success or deamon death comes.
-            request_event = RequestEntry(request_event)
+    while True:
+        for events_iterator in [
+            shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
+                                          raise_exception=False),
+            shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
+                                            raise_exception=False)
+        ]:
+            for request_event in events_iterator:
+                request_event = RequestEntry(request_event)
 
-            if request_event.type == "TIMEOUT":
                 maintenance(host.key)
 
-            elif request_event.hostname == host.key:
+                if request_event.hostname == host.key:
-                logger.debug("VM Request: %s on Host %s", request_event, host.hostname)
+                    logger.debug('VM Request: %s on Host %s', request_event, host.hostname)
-                shared.request_pool.client.client.delete(request_event.key)
-                vm_entry = shared.etcd_client.get(
-                    join_path(settings["etcd"]["vm_prefix"], request_event.uuid)
-                )
-                logger.debug("VM hostname: {}".format(vm_entry.value))
-                vm = virtualmachine.VM(vm_entry)
-                if request_event.type == RequestType.StartVM:
-                    vm.start()
 
-                elif request_event.type == RequestType.StopVM:
+                    shared.request_pool.client.client.delete(request_event.key)
-                    vm.stop()
+                    vm_entry = shared.etcd_client.get(
+                        join_path(settings['etcd']['vm_prefix'], request_event.uuid)
+                    )
 
-                elif request_event.type == RequestType.DeleteVM:
+                    logger.debug('VM hostname: {}'.format(vm_entry.value))
-                    vm.delete()
 
-                elif request_event.type == RequestType.InitVMMigration:
+                    vm = virtualmachine.VM(vm_entry)
-                    vm.start(destination_host_key=host.key)
+                    if request_event.type == RequestType.StartVM:
+                        vm.start()
 
-                elif request_event.type == RequestType.TransferVM:
+                    elif request_event.type == RequestType.StopVM:
-                    destination_host = host_pool.get(request_event.destination_host_key)
+                        vm.stop()
-                    if destination_host:
-                        vm.migrate(
-                            destination_host=destination_host.hostname,
-                            destination_sock_path=request_event.destination_sock_path,
-                        )
-                    else:
-                        logger.error("Host %s not found!", request_event.destination_host_key)
 
+                    elif request_event.type == RequestType.DeleteVM:
+                        vm.delete()
 
-if __name__ == "__main__":
+                    elif request_event.type == RequestType.InitVMMigration:
-    argparser = argparse.ArgumentParser()
+                        vm.start(destination_host_key=host.key)
-    argparser.add_argument(
+
-        "hostname", help="Name of this host. e.g uncloud1.ungleich.ch"
+                    elif request_event.type == RequestType.TransferVM:
-    )
+                        destination_host = host_pool.get(request_event.destination_host_key)
-    args = argparser.parse_args()
+                        if destination_host:
-    mp.set_start_method("spawn")
+                            vm.migrate(
-    main(args.hostname)
+                                destination_host=destination_host.hostname,
+                                destination_sock_path=request_event.destination_sock_path,
+                            )
+                        else:
+                            logger.error('Host %s not found!', request_event.destination_host_key)

uncloud/host/virtualmachine.py

@@ -16,8 +16,8 @@ from uncloud.common.vm import VMStatus, declare_stopped
 from uncloud.common.network import create_dev, delete_network_interface
 from uncloud.common.schemas import VMSchema, NetworkSchema
 from uncloud.host import logger
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 from uncloud.vmm import VMM
 
 from marshmallow import ValidationError
@@ -42,7 +42,7 @@ class VM:
 
     def get_qemu_args(self):
         command = (
-            "-drive file={file},format=raw,if=virtio,cache=none"
+            "-drive file={file},format=raw,if=virtio"
             " -device virtio-rng-pci"
             " -m {memory} -smp cores={cores},threads={threads}"
             " -name {owner}_{name}"
@@ -153,7 +153,10 @@ class VM:
                 )
             )
 
-        return command.split(" ")
+        if command:
+            command = command.split(' ')
+
+        return command
 
     def delete_network_dev(self):
         try:

uncloud/imagescanner/main.py

@@ -4,8 +4,8 @@ import argparse
 import subprocess as sp
 
 from os.path import join as join_path
-from uncloud.settings import settings
+from uncloud.common.settings import settings
-from uncloud.shared import shared
+from uncloud.common.shared import shared
 from uncloud.imagescanner import logger
 
 
@@ -30,7 +30,7 @@ def qemu_img_type(path):
         return qemu_img_info["format"]
 
 
-def main(debug=False):
+def main(arguments):
     # We want to get images entries that requests images to be created
     images = shared.etcd_client.get_prefix(
         settings["etcd"]["image_prefix"], value_in_json=True

uncloud/metadata/main.py

@@ -5,8 +5,8 @@ from flask import Flask, request
 from flask_restful import Resource, Api
 from werkzeug.exceptions import HTTPException
 
-from uncloud.settings import settings
+from uncloud.common.settings import settings
-from uncloud.shared import shared
+from uncloud.common.shared import shared
 
 app = Flask(__name__)
 api = Api(app)
@@ -84,40 +84,11 @@ class Root(Resource):
             data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys
             return data.value["metadata"], 200
 
-    @staticmethod
-    def post():
-        return {"message": "Previous Implementation is deprecated."}
-        # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True)
-        # print(data)
-        # if data:
-        #     for k in request.json:
-        #         if k not in data.value:
-        #             data.value[k] = request.json[k]
-        #             if k.endswith("-list"):
-        #                 data.value[k] = [request.json[k]]
-        #         else:
-        #             if k.endswith("-list"):
-        #                 data.value[k].append(request.json[k])
-        #             else:
-        #                 data.value[k] = request.json[k]
-        #     etcd_client.put("/v1/metadata/{}".format(request.remote_addr),
-        #                     data.value, value_in_json=True)
-        # else:
-        #     data = {}
-        #     for k in request.json:
-        #         data[k] = request.json[k]
-        #         if k.endswith("-list"):
-        #             data[k] = [request.json[k]]
-        #     etcd_client.put("/v1/metadata/{}".format(request.remote_addr),
-        #                     data, value_in_json=True)
-
 
 api.add_resource(Root, "/")
 
 
-def main(port=None, debug=False):
+def main(arguments):
+    port = arguments['port']
+    debug = arguments['debug']
     app.run(debug=debug, host="::", port=port)
-
-
-if __name__ == "__main__":
-    main()

uncloud/scheduler/helper.py

@@ -6,8 +6,8 @@ import bitmath
 from uncloud.common.host import HostStatus
 from uncloud.common.request import RequestEntry, RequestType
 from uncloud.common.vm import VMStatus
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.common.settings import settings
 
 
 def accumulated_specs(vms_specs):

uncloud/scheduler/main.py

@@ -6,59 +6,47 @@
 
 import argparse
 
+from uncloud.common.settings import settings
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.scheduler import logger
-from .helper import (dead_host_mitigation, dead_host_detection, assign_host, NoSuitableHostFound)
+from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection,
-from . import logger
+                                      assign_host, NoSuitableHostFound)
 
 arg_parser = argparse.ArgumentParser('scheduler', add_help=False)
 
 
-def main(debug=False):
+def main(arguments):
-    for request_iterator in [
+    # The below while True is neccessary for gracefully handling leadership transfer and temporary
-        shared.etcd_client.get_prefix(
+    # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
-            settings["etcd"]["request_prefix"], value_in_json=True
+    # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
-        ),
+    # which ends the loop immediately. So, having it inside infinite loop we try again and again to
-        shared.etcd_client.watch_prefix(
+    # get prefix until either success or deamon death comes.
-            settings["etcd"]["request_prefix"],
+    while True:
-            timeout=5,
+        for request_iterator in [
-            value_in_json=True,
+            shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
-        ),
+                                          raise_exception=False),
-    ]:
+            shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
-        for request_event in request_iterator:
+                                            raise_exception=False),
-            request_entry = RequestEntry(request_event)
+        ]:
-            # Never Run time critical mechanism inside timeout
+            for request_event in request_iterator:
-            # mechanism because timeout mechanism only comes
+                dead_host_mitigation(dead_host_detection())
-            # when no other event is happening. It means under
+                request_entry = RequestEntry(request_event)
-            # heavy load there would not be a timeout event.
-            if request_entry.type == "TIMEOUT":
 
-                # Detect hosts that are dead and set their status
+                if request_entry.type == RequestType.ScheduleVM:
-                # to "DEAD", and their VMs' status to "KILLED"
+                    logger.debug('%s, %s', request_entry.key, request_entry.value)
-                dead_hosts = dead_host_detection()
-                if dead_hosts:
-                    logger.debug("Dead hosts: %s", dead_hosts)
-                    dead_host_mitigation(dead_hosts)
 
-            elif request_entry.type == RequestType.ScheduleVM:
+                    vm_entry = shared.vm_pool.get(request_entry.uuid)
-                logger.debug("%s, %s", request_entry.key, request_entry.value)
+                    if vm_entry is None:
+                        logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid))
+                        continue
 
-                vm_entry = shared.vm_pool.get(request_entry.uuid)
+                    shared.etcd_client.client.delete(request_entry.key)  # consume Request
-                if vm_entry is None:
-                    logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid))
-                    continue
 
-                shared.etcd_client.client.delete(request_entry.key)  # consume Request
+                    try:
+                        assign_host(vm_entry)
+                    except NoSuitableHostFound:
+                        vm_entry.add_log('Can\'t schedule VM. No Resource Left.')
+                        shared.vm_pool.put(vm_entry)
 
-                try:
+                        logger.info('No Resource Left. Emailing admin....')
-                    assign_host(vm_entry)
-                except NoSuitableHostFound:
-                    vm_entry.add_log("Can't schedule VM. No Resource Left.")
-                    shared.vm_pool.put(vm_entry)
-
-                    logger.info("No Resource Left. Emailing admin....")
-
-
-if __name__ == "__main__":
-    main()

uncloud/vmm/__init__.py

@@ -190,18 +190,10 @@ class VMM:
                     err.stderr.decode("utf-8"),
                 )
             else:
-                with suppress(sp.CalledProcessError):
+                sp.check_output(
-                    sp.check_output(
+                    ["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory",
-                        [
+                     "chmod", "-R", "o=rwx,g=rwx", self.vmm_backend]
-                            "sudo",
+                )
-                            "-p",
-                            "Enter password to correct permission for uncloud-vmm's directory",
-                            "chmod",
-                            "-R",
-                            "o=rwx,g=rwx",
-                            self.vmm_backend,
-                        ]
-                    )
 
                 # TODO: Find some good way to check whether the virtual machine is up and
                 #       running without relying on non-guarenteed ways.