2nd commit

.gitignore

@@ -16,5 +16,3 @@ uncloud/version.py
 build/
 venv/
 dist/
-
-*.iso

bin/gen-version

@@ -1,22 +1,22 @@
 #!/bin/sh
 # -*- coding: utf-8 -*-
 #
-# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org)
+# 2019 Nico Schottelius (nico-ucloud at schottelius.org)
 #
-# This file is part of uncloud.
+# This file is part of ucloud.
 #
-# uncloud is free software: you can redistribute it and/or modify
+# ucloud is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# uncloud is distributed in the hope that it will be useful,
+# ucloud is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+# along with ucloud. If not, see <http://www.gnu.org/licenses/>.
 #
 #
 
@@ -26,4 +26,4 @@ dir=${0%/*}
 
 # Ensure version is present - the bundled/shipped version contains a static version,
 # the git version contains a dynamic version
-printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py
+printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py

bin/uncloud-run-reinstall

@@ -24,6 +24,6 @@
 dir=${0%/*}
 
 ${dir}/gen-version;
-pip uninstall -y uncloud >/dev/null
-python setup.py install  >/dev/null
+pip uninstall -y uncloud
+python setup.py install
 ${dir}/uncloud "$@"

docs/source/hacking.rst

@@ -1,36 +0,0 @@
-Hacking
-=======
-Using uncloud in hacking (aka development) mode.
-
-
-Get the code
-------------
-.. code-block:: sh
-    :linenos:
-
-     git clone https://code.ungleich.ch/uncloud/uncloud.git
-
-
-
-Install python requirements
----------------------------
-You need to have python3 installed.
-
-.. code-block:: sh
-    :linenos:
-
-    cd uncloud!
-    python -m venv venv
-    . ./venv/bin/activate
-    ./bin/uncloud-run-reinstall
-
-
-
-Install os requirements
------------------------
-Install the following software packages: **dnsmasq**.
-
-If you already have a working IPv6 SLAAC and DNS setup,
-this step can be skipped.
-
-Note that you need at least one /64 IPv6 network to run uncloud.

docs/source/vm-images.rst

@@ -1,66 +0,0 @@
-VM images
-==================================
-
-Overview
----------
-
-ucloud tries to be least invasise towards VMs and only require
-strictly necessary changes for running in a virtualised
-environment. This includes configurations for:
-
-* Configuring the network
-* Managing access via ssh keys
-* Resizing the attached disk(s)
-
-Upstream images
----------------
-
-The 'official' uncloud images are defined in the `uncloud/images
-<https://code.ungleich.ch/uncloud/images>`_ repository.
-
-How to make you own Uncloud images
-----------------------------------
-
-.. note::
-  It is fairly easy to create your own images for uncloud, as the common
-  operations (which are detailed below) can be automatically handled by the
-  `uncloud/uncloud-init <https://code.ungleich.ch/uncloud/uncloud-init>`_ tool.
-
-Network configuration
-~~~~~~~~~~~~~~~~~~~~~
-All VMs in ucloud are required to support IPv6. The primary network
-configuration is always done using SLAAC. A VM thus needs only to be
-configured to
-
-* accept router advertisements on all network interfaces
-* use the router advertisements to configure the network interfaces
-* accept the DNS entries from the router advertisements
-
-
-Configuring SSH keys
-~~~~~~~~~~~~~~~~~~~~
-
-To be able to access the VM, ucloud support provisioning SSH keys.
-
-To accept ssh keys in your VM, request the URL
-*http://metadata/ssh_keys*. Add the content to the appropriate user's
-**authorized_keys** file. Below you find sample code to accomplish
-this task:
-
-.. code-block:: sh
-
-    tmp=$(mktemp)
-    curl -s http://metadata/ssk_keys > "$tmp"
-    touch ~/.ssh/authorized_keys # ensure it exists
-    cat ~/.ssh/authorized_keys >> "$tmp"
-    sort "$tmp" | uniq > ~/.ssh/authorized_keys
-
-
-Disk resize
-~~~~~~~~~~~
-In virtualised environments, the disk sizes might grow. The operating
-system should detect disks that are bigger than the existing partition
-table and resize accordingly. This task is os specific.
-
-ucloud does not support shrinking disks due to the complexity and
-intra OS dependencies.

scripts/uncloud

@@ -3,86 +3,56 @@ import logging
 import sys
 import importlib
 import argparse
-import os
+import multiprocessing as mp
 
-from etcd3.exceptions import ConnectionFailedError
-
-from uncloud.common import settings
 from uncloud import UncloudException
-from uncloud.common.cli import resolve_otp_credentials
+from contextlib import suppress
 
-# Components that use etcd
-ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner',
-                   'imagescanner', 'metadata', 'configure', 'hack']
 
-ALL_COMPONENTS = ETCD_COMPONENTS.copy()
-ALL_COMPONENTS.append('oneshot')
-#ALL_COMPONENTS.append('cli')
+def exception_hook(exc_type, exc_value, exc_traceback):
+    logging.getLogger(__name__).error(
+        'Uncaught exception',
+        exc_info=(exc_type, exc_value, exc_traceback)
+    )
 
 
+sys.excepthook = exception_hook
+
 if __name__ == '__main__':
+    # Setting up root logger
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+
     arg_parser = argparse.ArgumentParser()
     subparsers = arg_parser.add_subparsers(dest='command')
 
     parent_parser = argparse.ArgumentParser(add_help=False)
     parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
                                help='More verbose logging')
-    parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
-                               default=os.path.expanduser('~/uncloud'))
 
-    etcd_parser = argparse.ArgumentParser(add_help=False)
-    etcd_parser.add_argument('--etcd-host')
-    etcd_parser.add_argument('--etcd-port')
-    etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
-    etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
-    etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
-
-    for component in ALL_COMPONENTS:
+    for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner',
+                      'metadata', 'configure', 'cli']:
         mod = importlib.import_module('uncloud.{}.main'.format(component))
         parser = getattr(mod, 'arg_parser')
+        subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
 
-        if component in ETCD_COMPONENTS:
-            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser])
-        else:
-            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
-
-    arguments = vars(arg_parser.parse_args())
-    etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
-    etcd_arguments = {
-        'etcd': {
-            key.replace('etcd_', ''): arguments[key]
-            for key in etcd_arguments
-        }
-    }
-    if not arguments['command']:
+    args = arg_parser.parse_args()
+    if not args.command:
         arg_parser.print_help()
     else:
-        # Initializing Settings and resolving otp_credentials
-        # It is neccessary to resolve_otp_credentials after argument parsing is done because
-        # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
-        # providing the default values for --name, --realm and --seed arguments from the values
-        # we read from file. But, now we are asking user about where the config file lives. So,
-        # to providing default value is not possible before parsing arguments. So, we are doing
-        # it after..
-#        settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
-#        resolve_otp_credentials(arguments)
-
-        name = arguments.pop('command')
-        mod = importlib.import_module('uncloud.{}.main'.format(name))
-        main = getattr(mod, 'main')
-
-        if arguments['debug']:
-            logging.basicConfig(level=logging.DEBUG)
-        else:
-            logging.basicConfig(level=logging.INFO)
-
-        log = logging.getLogger()
 
+        # if we start etcd in seperate process with default settings
+        # i.e inheriting few things from parent process etcd3 module
+        # errors out, so the following command configure multiprocessing
+        # module to not inherit anything from parent.
+        # mp.set_start_method('spawn')
+        arguments = vars(args)
         try:
-            main(arguments)
+            name = arguments.pop('command')
+            mod = importlib.import_module('uncloud.{}.main'.format(name))
+            main = getattr(mod, 'main')
+            main(**arguments)
         except UncloudException as err:
-            log.error(err)
-#        except ConnectionFailedError as err:
-#            log.error('Cannot connect to etcd: {}'.format(err))
+            logger.error(err)
         except Exception as err:
-            log.exception(err)
+            logger.exception(err)

uncloud/api/common_fields.py

@@ -1,59 +0,0 @@
-import os
-
-from uncloud.common.shared import shared
-
-
-class Optional:
-    pass
-
-
-class Field:
-    def __init__(self, _name, _type, _value=None):
-        self.name = _name
-        self.value = _value
-        self.type = _type
-        self.__errors = []
-
-    def validation(self):
-        return True
-
-    def is_valid(self):
-        if self.value == KeyError:
-            self.add_error(
-                "'{}' field is a required field".format(self.name)
-            )
-        else:
-            if isinstance(self.value, Optional):
-                pass
-            elif not isinstance(self.value, self.type):
-                self.add_error(
-                    "Incorrect Type for '{}' field".format(self.name)
-                )
-            else:
-                self.validation()
-
-        if self.__errors:
-            return False
-        return True
-
-    def get_errors(self):
-        return self.__errors
-
-    def add_error(self, error):
-        self.__errors.append(error)
-
-
-class VmUUIDField(Field):
-    def __init__(self, data):
-        self.uuid = data.get("uuid", KeyError)
-
-        super().__init__("uuid", str, self.uuid)
-
-        self.validation = self.vm_uuid_validation
-
-    def vm_uuid_validation(self):
-        r = shared.etcd_client.get(
-            os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
-        )
-        if not r:
-            self.add_error("VM with uuid {} does not exists".format(self.uuid))

uncloud/api/create_image_store.py

@@ -4,6 +4,7 @@ import os
 from uuid import uuid4
 
 from uncloud.common.shared import shared
+from uncloud.common.settings import settings
 
 data = {
     'is_public': True,
@@ -13,7 +14,4 @@ data = {
     'attributes': {'list': [], 'key': [], 'pool': 'images'},
 }
 
-shared.etcd_client.put(
-    os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
-    json.dumps(data),
-)
+shared.etcd_client.put(os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data))

uncloud/api/helper.py

@@ -1,12 +1,11 @@
 import binascii
-import ipaddress
-import random
 import logging
 import requests
 
 from pyotp import TOTP
 
 from uncloud.common.shared import shared
+from uncloud.common.settings import settings
 
 logger = logging.getLogger(__name__)
 
@@ -14,25 +13,19 @@ logger = logging.getLogger(__name__)
 def check_otp(name, realm, token):
     try:
         data = {
-            "auth_name": shared.settings["otp"]["auth_name"],
-            "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
-            "auth_realm": shared.settings["otp"]["auth_realm"],
-            "name": name,
-            "realm": realm,
-            "token": token,
+            'auth_name': settings['otp']['auth_name'],
+            'auth_token': TOTP(settings['otp']['auth_seed']).now(),
+            'auth_realm': settings['otp']['auth_realm'],
+            'name': name,
+            'realm': realm,
+            'token': token,
         }
-    except binascii.Error as err:
-        logger.error(
-            "Cannot compute OTP for seed: {}".format(
-                shared.settings["otp"]["auth_seed"]
-            )
-        )
+    except binascii.Error:
+        logger.error('Cannot compute OTP for seed: {}'.format(settings['otp']['auth_seed']))
         return 400
-
-    response = requests.post(
-        shared.settings["otp"]["verification_controller_url"], json=data
-    )
-    return response.status_code
+    else:
+        response = requests.post(settings['otp']['verification_controller_url'], json=data)
+        return response.status_code
 
 
 def resolve_vm_name(name, owner):
@@ -42,29 +35,24 @@ def resolve_vm_name(name, owner):
     Output: uuid of vm if found otherwise None
     """
     result = next(
-        filter(
-            lambda vm: vm.value["owner"] == owner
-            and vm.value["name"] == name,
-            shared.vm_pool.vms,
-        ),
-        None,
+        filter(lambda vm: vm.value['owner'] == owner and vm.value['name'] == name, shared.vm_pool.vms),
+        None
     )
     if result:
-        return result.key.split("/")[-1]
+        return result.key.split('/')[-1]
 
     return None
 
 
-def resolve_image_name(name, etcd_client):
+def resolve_image_name(name):
     """Return image uuid given its name and its store
 
        * If the provided name is not in correct format
          i.e {store_name}:{image_name} return ValueError
        * If no such image found then return KeyError
-
     """
 
-    seperator = ":"
+    seperator = ':'
 
     # Ensure, user/program passed valid name that is of type string
     try:
@@ -72,77 +60,35 @@ def resolve_image_name(name, etcd_client):
 
         """
         Examples, where it would work and where it would raise exception
-        "images:alpine" --> ["images", "alpine"]
+        'images:alpine' --> ['images', 'alpine']
 
-        "images" --> ["images"] it would raise Exception as non enough value to unpack
+        'images' --> ['images'] it would raise Exception as non enough value to unpack
 
-        "images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception
+        'images:alpine:meow' --> ['images', 'alpine', 'meow'] it would raise Exception
                                                               as too many values to unpack
         """
         store_name, image_name = store_name_and_image_name
     except Exception:
-        raise ValueError(
-            "Image name not in correct format i.e {store_name}:{image_name}"
-        )
+        raise ValueError('Image name not in correct format i.e {store_name}:{image_name}')
 
-    images = etcd_client.get_prefix(
-        shared.settings["etcd"]["image_prefix"], value_in_json=True
-    )
+    images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True)
 
     # Try to find image with name == image_name and store_name == store_name
     try:
         image = next(
             filter(
-                lambda im: im.value["name"] == image_name
-                and im.value["store_name"] == store_name,
+                lambda im: im.value['name'] == image_name
+                and im.value['store_name'] == store_name,
                 images,
             )
         )
     except StopIteration:
-        raise KeyError("No image with name {} found.".format(name))
+        raise KeyError('No image with name {} found.'.format(name))
     else:
-        image_uuid = image.key.split("/")[-1]
+        image_uuid = image.key.split('/')[-1]
 
     return image_uuid
 
 
-def random_bytes(num=6):
-    return [random.randrange(256) for _ in range(num)]
-
-
-def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
-    mac = random_bytes()
-    if oui:
-        if type(oui) == str:
-            oui = [int(chunk) for chunk in oui.split(separator)]
-        mac = oui + random_bytes(num=6 - len(oui))
-    else:
-        if multicast:
-            mac[0] |= 1  # set bit 0
-        else:
-            mac[0] &= ~1  # clear bit 0
-        if uaa:
-            mac[0] &= ~(1 << 1)  # clear bit 1
-        else:
-            mac[0] |= 1 << 1  # set bit 1
-    return separator.join(byte_fmt % b for b in mac)
-
-
-def mac2ipv6(mac, prefix):
-    # only accept MACs separated by a colon
-    parts = mac.split(":")
-
-    # modify parts to match IPv6 value
-    parts.insert(3, "ff")
-    parts.insert(4, "fe")
-    parts[0] = "%x" % (int(parts[0], 16) ^ 2)
-
-    # format output
-    ipv6_parts = [str(0)] * 4
-    for i in range(0, len(parts), 2):
-        ipv6_parts.append("".join(parts[i : i + 2]))
-
-    lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
-    prefix = ipaddress.IPv6Address(prefix)
-    return str(prefix + int(lower_part))
-
+def make_return_message(err, status_code=200):
+    return {'message': str(err)}, status_code

uncloud/api/main.py

@@ -14,9 +14,13 @@ from uncloud.common.shared import shared
 
 from uncloud.common import counters
 from uncloud.common.vm import VMStatus
+from uncloud.common.host import HostStatus
 from uncloud.common.request import RequestEntry, RequestType
+from uncloud.common.settings import settings
 from uncloud.api import schemas
-from uncloud.api.helper import generate_mac, mac2ipv6
+from uncloud.api.schemas import ValidationException
+from uncloud.common.network import generate_mac, mac2ipv6
+from uncloud.api.helper import make_return_message
 from uncloud import UncloudException
 
 logger = logging.getLogger(__name__)
@@ -29,51 +33,52 @@ arg_parser = argparse.ArgumentParser('api', add_help=False)
 arg_parser.add_argument('--port', '-p')
 
 
-@app.errorhandler(Exception)
-def handle_exception(e):
-    app.logger.error(e)
-    # pass through HTTP errors
-    if isinstance(e, HTTPException):
-        return e
-
-    # now you're handling non-HTTP exceptions only
-    return {'message': 'Server Error'}, 500
+# @app.errorhandler(Exception)
+# def handle_exception(e):
+#     app.logger.error(e)
+#
+#     # pass through HTTP errors
+#     if isinstance(e, HTTPException):
+#         return e
+#
+#     # now you're handling non-HTTP exceptions only
+#     return {'message': 'Server Error'}, 500
 
 
 class CreateVM(Resource):
-    """API Request to Handle Creation of VM"""
-
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.CreateVMSchema(data)
-        if validator.is_valid():
+        try:
+            validator = schemas.CreateVMSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
             vm_uuid = uuid4().hex
-            vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
+            vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid)
             specs = {
                 'cpu': validator.specs['cpu'],
                 'ram': validator.specs['ram'],
                 'os-ssd': validator.specs['os-ssd'],
                 'hdd': validator.specs['hdd'],
             }
-            macs = [generate_mac() for _ in range(len(data['network']))]
+            macs = [generate_mac() for _ in range(len(validator.network))]
             tap_ids = [
-                counters.increment_etcd_counter(
-                    shared.etcd_client, shared.settings['etcd']['tap_counter']
-                )
-                for _ in range(len(data['network']))
+                counters.increment_etcd_counter(settings['etcd']['tap_counter'])
+                for _ in range(len(validator.network))
             ]
             vm_entry = {
-                'name': data['vm_name'],
-                'owner': data['name'],
-                'owner_realm': data['realm'],
+                'name': validator.vm_name,
+                'owner': validator.name,
+                'owner_realm': validator.realm,
                 'specs': specs,
                 'hostname': '',
                 'status': VMStatus.stopped,
                 'image_uuid': validator.image_uuid,
                 'log': [],
                 'vnc_socket': '',
-                'network': list(zip(data['network'], macs, tap_ids)),
+                'network': list(zip(validator.network, macs, tap_ids)),
                 'metadata': {'ssh-keys': []},
                 'in_migration': False,
             }
@@ -83,89 +88,79 @@ class CreateVM(Resource):
             r = RequestEntry.from_scratch(
                 type=RequestType.ScheduleVM,
                 uuid=vm_uuid,
-                request_prefix=shared.settings['etcd']['request_prefix'],
+                request_prefix=settings['etcd']['request_prefix'],
             )
             shared.request_pool.put(r)
-
-            return {'message': 'VM Creation Queued'}, 200
-        return validator.get_errors(), 400
+            return make_return_message('VM Creation Queued')
 
 
-class VmStatus(Resource):
+class GetVMStatus(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.VMStatusSchema(data)
-        if validator.is_valid():
-            vm = shared.vm_pool.get(
-                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
-            )
+        try:
+            validator = schemas.VMStatusSchema(data)
+            validator.is_valid()
+        except (ValidationException, Exception) as err:
+            return make_return_message(err, 400)
+        else:
+            vm = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], validator.uuid))
             vm_value = vm.value.copy()
             vm_value['ip'] = []
             for network_mac_and_tap in vm.network:
                 network_name, mac, tap = network_mac_and_tap
                 network = shared.etcd_client.get(
-                    join_path(
-                        shared.settings['etcd']['network_prefix'],
-                        data['name'],
-                        network_name,
-                    ),
+                    join_path(settings['etcd']['network_prefix'], validator.name, network_name),
                     value_in_json=True,
                 )
-                ipv6_addr = (
-                    network.value.get('ipv6').split('::')[0] + '::'
-                )
+                ipv6_addr = (network.value.get('ipv6').split('::')[0] + '::')
                 vm_value['ip'].append(mac2ipv6(mac, ipv6_addr))
             vm.value = vm_value
-            return vm.value
-        else:
-            return validator.get_errors(), 400
+            return vm.value, 200
 
 
 class CreateImage(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.CreateImageSchema(data)
-        if validator.is_valid():
-            file_entry = shared.etcd_client.get(
-                join_path(shared.settings['etcd']['file_prefix'], data['uuid'])
-            )
-            file_entry_value = json.loads(file_entry.value)
+        try:
+            validator = schemas.CreateImageSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            try:
+                file_entry = shared.etcd_client.get(
+                    join_path(settings['etcd']['file_prefix'], validator.uuid), value_in_json=True
+                )
+            except KeyError:
+                # TODO: Add some message
+                pass
+            else:
+                image_entry_json = {
+                    'status': 'TO_BE_CREATED',
+                    'owner': file_entry.value['owner'],
+                    'filename': file_entry.value['filename'],
+                    'name': validator.name,
+                    'store_name': validator.image_store,
+                    'visibility': 'public',
+                }
+                shared.etcd_client.put(
+                    join_path(settings['etcd']['image_prefix'], validator.uuid),
+                    json.dumps(image_entry_json),
+                )
 
-            image_entry_json = {
-                'status': 'TO_BE_CREATED',
-                'owner': file_entry_value['owner'],
-                'filename': file_entry_value['filename'],
-                'name': data['name'],
-                'store_name': data['image_store'],
-                'visibility': 'public',
-            }
-            shared.etcd_client.put(
-                join_path(
-                    shared.settings['etcd']['image_prefix'], data['uuid']
-                ),
-                json.dumps(image_entry_json),
-            )
-
-            return {'message': 'Image queued for creation.'}
-        return validator.get_errors(), 400
+                return {'message': 'Image queued for creation.'}, 200
 
 
 class ListPublicImages(Resource):
     @staticmethod
     def get():
-        images = shared.etcd_client.get_prefix(
-            shared.settings['etcd']['image_prefix'], value_in_json=True
-        )
+        images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True)
         r = {'images': []}
         for image in images:
-            image_key = '{}:{}'.format(
-                image.value['store_name'], image.value['name']
-            )
-            r['images'].append(
-                {'name': image_key, 'status': image.value['status']}
-            )
+            image_key = '{}:{}'.format(image.value['store_name'], image.value['name'])
+            r['images'].append({'name': image_key, 'status': image.value['status']})
         return r, 200
 
 
@@ -173,92 +168,79 @@ class VMAction(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.VmActionSchema(data)
-
-        if validator.is_valid():
-            vm_entry = shared.vm_pool.get(
-                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
-            )
-            action = data['action']
+        try:
+            validator = schemas.VmActionSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            vm_entry = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], validator.uuid))
+            action = validator.action
 
             if action == 'start':
                 action = 'schedule'
 
             if action == 'delete' and vm_entry.hostname == '':
-                if shared.storage_handler.is_vm_image_exists(
-                    vm_entry.uuid
-                ):
-                    r_status = shared.storage_handler.delete_vm_image(
-                        vm_entry.uuid
-                    )
+                if shared.storage_handler.is_vm_image_exists(vm_entry.uuid):
+                    r_status = shared.storage_handler.delete_vm_image(vm_entry.uuid)
                     if r_status:
                         shared.etcd_client.client.delete(vm_entry.key)
-                        return {'message': 'VM successfully deleted'}
+                        return make_return_message('VM successfully deleted')
                     else:
-                        logger.error(
-                            'Some Error Occurred while deleting VM'
-                        )
-                        return {'message': 'VM deletion unsuccessfull'}
+                        logger.error('Some Error Occurred while deleting VM')
+                        return make_return_message('VM deletion unsuccessfull')
                 else:
                     shared.etcd_client.client.delete(vm_entry.key)
-                    return {'message': 'VM successfully deleted'}
+                    return make_return_message('VM successfully deleted')
 
             r = RequestEntry.from_scratch(
                 type='{}VM'.format(action.title()),
-                uuid=data['uuid'],
+                uuid=validator.uuid,
                 hostname=vm_entry.hostname,
-                request_prefix=shared.settings['etcd']['request_prefix'],
+                request_prefix=settings['etcd']['request_prefix'],
             )
             shared.request_pool.put(r)
-            return (
-                {'message': 'VM {} Queued'.format(action.title())},
-                200,
-            )
-        else:
-            return validator.get_errors(), 400
+            return make_return_message('VM {} Queued'.format(action.title()))
 
 
 class VMMigration(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.VmMigrationSchema(data)
-
-        if validator.is_valid():
-            vm = shared.vm_pool.get(data['uuid'])
+        try:
+            validator = schemas.VmMigrationSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err), 400
+        else:
+            vm = shared.vm_pool.get(validator.uuid)
             r = RequestEntry.from_scratch(
                 type=RequestType.InitVMMigration,
                 uuid=vm.uuid,
                 hostname=join_path(
-                    shared.settings['etcd']['host_prefix'],
-                    validator.destination.value,
+                    settings['etcd']['host_prefix'],
+                    validator.destination,
                 ),
-                request_prefix=shared.settings['etcd']['request_prefix'],
+                request_prefix=settings['etcd']['request_prefix'],
             )
 
             shared.request_pool.put(r)
-            return (
-                {'message': 'VM Migration Initialization Queued'},
-                200,
-            )
-        else:
-            return validator.get_errors(), 400
+            return make_return_message('VM Migration Initialization Queued')
 
 
 class ListUserVM(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.OTPSchema(data)
-
-        if validator.is_valid():
-            vms = shared.etcd_client.get_prefix(
-                shared.settings['etcd']['vm_prefix'], value_in_json=True
-            )
+        try:
+            validator = schemas.OTPSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            vms = shared.etcd_client.get_prefix(settings['etcd']['vm_prefix'], value_in_json=True)
             return_vms = []
-            user_vms = filter(
-                lambda v: v.value['owner'] == data['name'], vms
-            )
+            user_vms = filter(lambda v: v.value['owner'] == validator.name, vms)
             for vm in user_vms:
                 return_vms.append(
                     {
@@ -270,26 +252,22 @@ class ListUserVM(Resource):
                         'vnc_socket': vm.value.get('vnc_socket', None),
                     }
                 )
-            if return_vms:
-                return {'message': return_vms}, 200
-            return {'message': 'No VM found'}, 404
-
-        else:
-            return validator.get_errors(), 400
+            return make_return_message(return_vms)
 
 
 class ListUserFiles(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.OTPSchema(data)
-
-        if validator.is_valid():
-            files = shared.etcd_client.get_prefix(
-                shared.settings['etcd']['file_prefix'], value_in_json=True
-            )
+        try:
+            validator = schemas.OTPSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            files = shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
             return_files = []
-            user_files = [f for f in files if f.value['owner'] == data['name']]
+            user_files = [f for f in files if f.value['owner'] == validator.name]
             for file in user_files:
                 file_uuid = file.key.split('/')[-1]
                 file = file.value
@@ -299,33 +277,28 @@ class ListUserFiles(Resource):
                 file.pop('owner', None)
 
                 return_files.append(file)
-            return {'message': return_files}, 200
-        else:
-            return validator.get_errors(), 400
+            return make_return_message(return_files)
 
 
 class CreateHost(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.CreateHostSchema(data)
-        if validator.is_valid():
-            host_key = join_path(
-                shared.settings['etcd']['host_prefix'], uuid4().hex
-            )
+        try:
+            validator = schemas.CreateHostSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            host_key = join_path(settings['etcd']['host_prefix'], uuid4().hex)
             host_entry = {
-                'specs': data['specs'],
-                'hostname': data['hostname'],
-                'status': 'DEAD',
+                'specs': validator.specs,
+                'hostname': validator.hostname,
+                'status': HostStatus.dead,
                 'last_heartbeat': '',
             }
-            shared.etcd_client.put(
-                host_key, host_entry, value_in_json=True
-            )
-
-            return {'message': 'Host Created'}, 200
-
-        return validator.get_errors(), 400
+            shared.etcd_client.put(host_key, host_entry, value_in_json=True)
+            return make_return_message('Host Created.')
 
 
 class ListHost(Resource):
@@ -347,200 +320,142 @@ class GetSSHKeys(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.GetSSHSchema(data)
-        if validator.is_valid():
-            if not validator.key_name.value:
-
-                # {user_prefix}/{realm}/{name}/key/
-                etcd_key = join_path(
-                    shared.settings['etcd']['user_prefix'],
-                    data['realm'],
-                    data['name'],
-                    'key',
-                )
-                etcd_entry = shared.etcd_client.get_prefix(
-                    etcd_key, value_in_json=True
-                )
-
+        try:
+            validator = schemas.GetSSHSchema(data)
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm,
+                                 validator.name, 'key')
+            if not validator.key_name:
+                etcd_entry = shared.etcd_client.get_prefix(etcd_key, value_in_json=True)
                 keys = {
                     key.key.split('/')[-1]: key.value
                     for key in etcd_entry
                 }
                 return {'keys': keys}
             else:
-
-                # {user_prefix}/{realm}/{name}/key/{key_name}
-                etcd_key = join_path(
-                    shared.settings['etcd']['user_prefix'],
-                    data['realm'],
-                    data['name'],
-                    'key',
-                    data['key_name'],
-                )
-                etcd_entry = shared.etcd_client.get(
-                    etcd_key, value_in_json=True
-                )
-
-                if etcd_entry:
-                    return {
-                        'keys': {
-                            etcd_entry.key.split('/')[
-                                -1
-                            ]: etcd_entry.value
-                        }
-                    }
+                etcd_key = join_path(validator.key_name)
+                try:
+                    etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True)
+                except KeyError:
+                    return make_return_message('No such key found.', 400)
                 else:
-                    return {'keys': {}}
-        else:
-            return validator.get_errors(), 400
+                    return {
+                        'keys': {etcd_entry.key.split('/')[-1]: etcd_entry.value}
+                    }
 
 
 class AddSSHKey(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.AddSSHSchema(data)
-        if validator.is_valid():
-
+        try:
+            validator = schemas.AddSSHSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                shared.settings['etcd']['user_prefix'],
-                data['realm'],
-                data['name'],
-                'key',
-                data['key_name'],
+                settings['etcd']['user_prefix'], validator.realm,
+                validator.name, 'key', validator.key_name
             )
-            etcd_entry = shared.etcd_client.get(
-                etcd_key, value_in_json=True
-            )
-            if etcd_entry:
-                return {
-                    'message': 'Key with name "{}" already exists'.format(
-                        data['key_name']
-                    )
-                }
-            else:
+            try:
+                shared.etcd_client.get(etcd_key, value_in_json=True)
+            except KeyError:
                 # Key Not Found. It implies user' haven't added any key yet.
-                shared.etcd_client.put(
-                    etcd_key, data['key'], value_in_json=True
-                )
-                return {'message': 'Key added successfully'}
-        else:
-            return validator.get_errors(), 400
+                shared.etcd_client.put(etcd_key, validator.key, value_in_json=True)
+                return make_return_message('Key added successfully')
+            else:
+                return make_return_message('Key "{}" already exists'.format(validator.key_name))
 
 
 class RemoveSSHKey(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.RemoveSSHSchema(data)
-        if validator.is_valid():
-
+        try:
+            validator = schemas.RemoveSSHSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
             # {user_prefix}/{realm}/{name}/key/{key_name}
-            etcd_key = join_path(
-                shared.settings['etcd']['user_prefix'],
-                data['realm'],
-                data['name'],
-                'key',
-                data['key_name'],
-            )
-            etcd_entry = shared.etcd_client.get(
-                etcd_key, value_in_json=True
-            )
+            etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm,
+                                 validator.name, 'key', validator.key_name)
+            try:
+                etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True)
+            except KeyError:
+                return make_return_message('No Key "{}" exists.'.format(validator.key_name))
             if etcd_entry:
                 shared.etcd_client.client.delete(etcd_key)
                 return {'message': 'Key successfully removed.'}
-            else:
-                return {
-                    'message': 'No Key with name "{}" Exists at all.'.format(
-                        data['key_name']
-                    )
-                }
-        else:
-            return validator.get_errors(), 400
 
 
 class CreateNetwork(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.CreateNetwork(data)
-
-        if validator.is_valid():
-
+        try:
+            validator = schemas.CreateNetwork(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
             network_entry = {
-                'id': counters.increment_etcd_counter(
-                    shared.etcd_client, shared.settings['etcd']['vxlan_counter']
-                ),
-                'type': data['type'],
+                'id': counters.increment_etcd_counter(settings['etcd']['vxlan_counter']),
+                'type': validator.type,
             }
-            if validator.user.value:
+            if validator.user:
                 try:
-                    nb = pynetbox.api(
-                        url=shared.settings['netbox']['url'],
-                        token=shared.settings['netbox']['token'],
-                    )
-                    nb_prefix = nb.ipam.prefixes.get(
-                        prefix=shared.settings['network']['prefix']
-                    )
+                    nb = pynetbox.api(url=settings['netbox']['url'], token=settings['netbox']['token'])
+                    nb_prefix = nb.ipam.prefixes.get(prefix=settings['network']['prefix'])
                     prefix = nb_prefix.available_prefixes.create(
                         data={
-                            'prefix_length': int(
-                                shared.settings['network']['prefix_length']
-                            ),
+                            'prefix_length': int(settings['network']['prefix_length']),
                             'description': '{}\'s network "{}"'.format(
-                                data['name'], data['network_name']
+                                validator.name,
+                                validator.network_name
                             ),
                             'is_pool': True,
                         }
                     )
                 except Exception as err:
                     app.logger.error(err)
-                    return {
-                        'message': 'Error occured while creating network.'
-                    }
+                    return make_return_message('Error occured while creating network.', 400)
                 else:
                     network_entry['ipv6'] = prefix['prefix']
             else:
                 network_entry['ipv6'] = 'fd00::/64'
 
-            network_key = join_path(
-                shared.settings['etcd']['network_prefix'],
-                data['name'],
-                data['network_name'],
-            )
-            shared.etcd_client.put(
-                network_key, network_entry, value_in_json=True
-            )
-            return {'message': 'Network successfully added.'}
-        else:
-            return validator.get_errors(), 400
+            network_key = join_path(settings['etcd']['network_prefix'], validator.name,
+                                    validator.network_name)
+            shared.etcd_client.put(network_key, network_entry, value_in_json=True)
+            return make_return_message('Network successfully added.')
 
 
 class ListUserNetwork(Resource):
     @staticmethod
     def post():
         data = request.json
-        validator = schemas.OTPSchema(data)
-
-        if validator.is_valid():
-            prefix = join_path(
-                shared.settings['etcd']['network_prefix'], data['name']
-            )
-            networks = shared.etcd_client.get_prefix(
-                prefix, value_in_json=True
-            )
+        try:
+            validator = schemas.OTPSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            prefix = join_path(settings['etcd']['network_prefix'], validator.name)
+            networks = shared.etcd_client.get_prefix(prefix, value_in_json=True)
             user_networks = []
             for net in networks:
                 net.value['name'] = net.key.split('/')[-1]
                 user_networks.append(net.value)
             return {'networks': user_networks}, 200
-        else:
-            return validator.get_errors(), 400
 
 
 api.add_resource(CreateVM, '/vm/create')
-api.add_resource(VmStatus, '/vm/status')
+api.add_resource(GetVMStatus, '/vm/status')
 
 api.add_resource(VMAction, '/vm/action')
 api.add_resource(VMMigration, '/vm/migrate')
@@ -562,39 +477,12 @@ api.add_resource(ListHost, '/host/list')
 api.add_resource(CreateNetwork, '/network/create')
 
 
-def main(arguments):
-    debug = arguments['debug']
-    port = arguments['port']
-
-    try:
-        image_stores = list(
-            shared.etcd_client.get_prefix(
-                shared.settings['etcd']['image_store_prefix'], value_in_json=True
-            )
-        )
-    except KeyError:
-        image_stores = False
-
-    # Do not inject default values that might be very wrong
-    # fail when required, not before
-    #
-    # if not image_stores:
-    #     data = {
-    #         'is_public': True,
-    #         'type': 'ceph',
-    #         'name': 'images',
-    #         'description': 'first ever public image-store',
-    #         'attributes': {'list': [], 'key': [], 'pool': 'images'},
-    #     }
-
-    #     shared.etcd_client.put(
-    #         join_path(
-    #             shared.settings['etcd']['image_store_prefix'], uuid4().hex
-    #         ),
-    #         json.dumps(data),
-    #     )
-
+def main(debug=False, port=None):
     try:
         app.run(host='::', port=port, debug=debug)
     except OSError as e:
         raise UncloudException('Failed to start Flask: {}'.format(e))
+
+
+if __name__ == '__main__':
+    main()

uncloud/api/schemas.py

@@ -1,19 +1,3 @@
-"""
-This module contain classes thats validates and intercept/modify
-data coming from uncloud-cli (user)
-
-It was primarily developed as an alternative to argument parser
-of Flask_Restful which is going to be deprecated. I also tried
-marshmallow for that purpose but it was an overkill (because it
-do validation + serialization + deserialization) and little
-inflexible for our purpose.
-"""
-
-# TODO: Fix error message when user's mentioned VM (referred by name)
-#       does not exists.
-#
-#       Currently, it says uuid is a required field.
-
 import json
 import os
 
@@ -22,19 +6,54 @@ import bitmath
 from uncloud.common.host import HostStatus
 from uncloud.common.vm import VMStatus
 from uncloud.common.shared import shared
-from . import helper, logger
-from .common_fields import Field, VmUUIDField
-from .helper import check_otp, resolve_vm_name
+from uncloud.common.settings import settings
+from uncloud.api import helper
+from uncloud.api.helper import check_otp, resolve_vm_name
+
+
+class ValidationException(Exception):
+    """Validation Error"""
+
+
+class Field:
+    def __init__(self, _name, _type, _value=None, validators=None):
+        if validators is None:
+            validators = []
+
+        assert isinstance(validators, list)
+
+        self.name = _name
+        self.value = _value
+        self.type = _type
+        self.validators = validators
+
+    def is_valid(self):
+        if not isinstance(self.value, self.type):
+            raise ValidationException("Incorrect Type for '{}' field".format(self.name))
+
+        for validator in self.validators:
+            validator()
+
+    def __repr__(self):
+        return self.name
+
+
+class VmUUIDField(Field):
+    def __init__(self, data):
+        self.uuid = data.get('uuid', KeyError)
+
+        super().__init__('uuid', str, self.uuid, validators=[self.vm_uuid_validation])
+
+    def vm_uuid_validation(self):
+        try:
+            shared.etcd_client.get(os.path.join(settings['etcd']['vm_prefix'], self.uuid))
+        except KeyError:
+            raise ValidationException('VM with uuid {} does not exists'.format(self.uuid))
 
 
 class BaseSchema:
-    def __init__(self, data, fields=None):
-        _ = data  # suppress linter warning
-        self.__errors = []
-        if fields is None:
-            self.fields = []
-        else:
-            self.fields = fields
+    def __init__(self):
+        self.fields = [getattr(self, field) for field in dir(self) if isinstance(getattr(self, field), Field)]
 
     def validation(self):
         # custom validation is optional
@@ -43,515 +62,340 @@ class BaseSchema:
     def is_valid(self):
         for field in self.fields:
             field.is_valid()
-            self.add_field_errors(field)
 
         for parent in self.__class__.__bases__:
-            try:
-                parent.validation(self)
-            except AttributeError:
-                pass
-        if not self.__errors:
-            self.validation()
+            parent.validation(self)
 
-        if self.__errors:
-            return False
-        return True
+        self.validation()
 
-    def get_errors(self):
-        return {"message": self.__errors}
+        for field in self.fields:
+            setattr(self, field.name, field.value)
 
-    def add_field_errors(self, field: Field):
-        self.__errors += field.get_errors()
 
-    def add_error(self, error):
-        self.__errors.append(error)
+def get(dictionary: dict, key: str, return_default=False, default=None):
+    if dictionary is None:
+        raise ValidationException('No data provided at all.')
+    try:
+        value = dictionary[key]
+    except KeyError:
+        if return_default:
+            return default
+        raise ValidationException("Missing data for '{}' field.".format(key))
+    else:
+        return value
 
 
 class OTPSchema(BaseSchema):
-    def __init__(self, data: dict, fields=None):
-        self.name = Field("name", str, data.get("name", KeyError))
-        self.realm = Field("realm", str, data.get("realm", KeyError))
-        self.token = Field("token", str, data.get("token", KeyError))
-
-        _fields = [self.name, self.realm, self.token]
-        if fields:
-            _fields += fields
-        super().__init__(data=data, fields=_fields)
+    def __init__(self, data: dict):
+        self.name = Field('name', str, get(data, 'name'))
+        self.realm = Field('realm', str, get(data, 'realm'))
+        self.token = Field('token', str, get(data, 'token'))
+        super().__init__()
 
     def validation(self):
-        if (
-            check_otp(
-                self.name.value, self.realm.value, self.token.value
-            )
-            != 200
-        ):
-            self.add_error("Wrong Credentials")
-
-
-########################## Image Operations ###############################################
+        if check_otp(self.name.value, self.realm.value, self.token.value) != 200:
+            raise ValidationException('Wrong Credentials')
 
 
 class CreateImageSchema(BaseSchema):
     def __init__(self, data):
-        # Fields
-        self.uuid = Field("uuid", str, data.get("uuid", KeyError))
-        self.name = Field("name", str, data.get("name", KeyError))
-        self.image_store = Field(
-            "image_store", str, data.get("image_store", KeyError)
-        )
-
-        # Validations
-        self.uuid.validation = self.file_uuid_validation
-        self.image_store.validation = self.image_store_name_validation
-
-        # All Fields
-        fields = [self.uuid, self.name, self.image_store]
-        super().__init__(data, fields)
+        self.uuid = Field('uuid', str, get(data, 'uuid'), validators=[self.file_uuid_validation])
+        self.name = Field('name', str, get(data, 'name'))
+        self.image_store = Field('image_store', str, get(data, 'image_store'),
+                                 validators=[self.image_store_name_validation])
+        super().__init__()
 
     def file_uuid_validation(self):
-        file_entry = shared.etcd_client.get(
-            os.path.join(
-                shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
-            )
-        )
-        if file_entry is None:
-            self.add_error(
-                "Image File with uuid '{}' Not Found".format(
-                    self.uuid.value
-                )
-            )
+        try:
+            shared.etcd_client.get(os.path.join(settings['etcd']['file_prefix'], self.uuid.value))
+        except KeyError:
+            raise ValidationException("Image File with uuid '{}' Not Found".format(self.uuid.value))
 
     def image_store_name_validation(self):
-        image_stores = list(
-            shared.etcd_client.get_prefix(
-                shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
-            )
-        )
-
-        image_store = next(
-            filter(
-                lambda s: json.loads(s.value)["name"]
-                == self.image_store.value,
-                image_stores,
-            ),
-            None,
-        )
-        if not image_store:
-            self.add_error(
-                "Store '{}' does not exists".format(
-                    self.image_store.value
-                )
-            )
-
-
-# Host Operations
+        image_stores = list(shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix']))
+        try:
+            next(filter(lambda s: json.loads(s.value)['name'] == self.image_store.value, image_stores))
+        except StopIteration:
+            raise ValidationException("Store '{}' does not exists".format(self.image_store.value))
 
 
 class CreateHostSchema(OTPSchema):
     def __init__(self, data):
-        # Fields
-        self.specs = Field("specs", dict, data.get("specs", KeyError))
-        self.hostname = Field(
-            "hostname", str, data.get("hostname", KeyError)
-        )
+        self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation])
+        self.hostname = Field('hostname', str, get(data, 'hostname'))
 
-        # Validation
-        self.specs.validation = self.specs_validation
-
-        fields = [self.hostname, self.specs]
-
-        super().__init__(data=data, fields=fields)
+        super().__init__(data)
 
     def specs_validation(self):
-        ALLOWED_BASE = 10
+        allowed_base = 10
 
-        _cpu = self.specs.value.get("cpu", KeyError)
-        _ram = self.specs.value.get("ram", KeyError)
-        _os_ssd = self.specs.value.get("os-ssd", KeyError)
-        _hdd = self.specs.value.get("hdd", KeyError)
+        _cpu = self.specs.value.get('cpu', KeyError)
+        _ram = self.specs.value.get('ram', KeyError)
+        _os_ssd = self.specs.value.get('os-ssd', KeyError)
+        _hdd = self.specs.value.get('hdd', KeyError)
 
-        if KeyError in [_cpu, _ram, _os_ssd, _hdd]:
-            self.add_error(
-                "You must specify CPU, RAM and OS-SSD in your specs"
-            )
-            return None
+        if KeyError in [_cpu, _ram, _os_ssd]:
+            raise ValidationException('You must specify CPU, RAM and OS-SSD in your specs')
         try:
             parsed_ram = bitmath.parse_string_unsafe(_ram)
             parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd)
 
-            if parsed_ram.base != ALLOWED_BASE:
-                self.add_error(
-                    "Your specified RAM is not in correct units"
-                )
-            if parsed_os_ssd.base != ALLOWED_BASE:
-                self.add_error(
-                    "Your specified OS-SSD is not in correct units"
-                )
+            if parsed_ram.base != allowed_base:
+                raise ValidationException('Your specified RAM is not in correct units')
+
+            if parsed_os_ssd.base != allowed_base:
+                raise ValidationException('Your specified OS-SSD is not in correct units')
 
             if _cpu < 1:
-                self.add_error("CPU must be atleast 1")
+                raise ValidationException('CPU must be atleast 1')
 
             if parsed_ram < bitmath.GB(1):
-                self.add_error("RAM must be atleast 1 GB")
+                raise ValidationException('RAM must be atleast 1 GB')
 
             if parsed_os_ssd < bitmath.GB(10):
-                self.add_error("OS-SSD must be atleast 10 GB")
+                raise ValidationException('OS-SSD must be atleast 10 GB')
 
             parsed_hdd = []
             for hdd in _hdd:
                 _parsed_hdd = bitmath.parse_string_unsafe(hdd)
-                if _parsed_hdd.base != ALLOWED_BASE:
-                    self.add_error(
-                        "Your specified HDD is not in correct units"
-                    )
-                    break
+                if _parsed_hdd.base != allowed_base:
+                    raise ValidationException('Your specified HDD is not in correct units')
                 else:
                     parsed_hdd.append(str(_parsed_hdd))
 
         except ValueError:
-            # TODO: Find some good error message
-            self.add_error("Specs are not correct.")
+            raise ValidationException('Specs are not correct.')
         else:
-            if self.get_errors():
-                self.specs = {
-                    "cpu": _cpu,
-                    "ram": str(parsed_ram),
-                    "os-ssd": str(parsed_os_ssd),
-                    "hdd": parsed_hdd,
-                }
+            self.specs = {
+                'cpu': _cpu,
+                'ram': str(parsed_ram),
+                'os-ssd': str(parsed_os_ssd),
+                'hdd': parsed_hdd,
+            }
 
     def validation(self):
-        if self.realm.value != "ungleich-admin":
-            self.add_error(
-                "Invalid Credentials/Insufficient Permission"
-            )
-
-
-# VM Operations
+        if self.realm.value != 'ungleich-admin':
+            raise ValidationException('Invalid Credentials/Insufficient Permission')
 
 
 class CreateVMSchema(OTPSchema):
     def __init__(self, data):
-        # Fields
-        self.specs = Field("specs", dict, data.get("specs", KeyError))
-        self.vm_name = Field(
-            "vm_name", str, data.get("vm_name", KeyError)
-        )
-        self.image = Field("image", str, data.get("image", KeyError))
-        self.network = Field(
-            "network", list, data.get("network", KeyError)
-        )
+        self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation])
+        self.vm_name = Field('vm_name', str, get(data, 'vm_name'), validators=[self.vm_name_validation])
+        self.image = Field('image', str, get(data, 'image'), validators=[self.image_validation])
+        self.network = Field('network', list, get(data, 'network', return_default=True, default=[]),
+                             validators=[self.network_validation])
+        self.image_uuid = None
 
-        # Validation
-        self.image.validation = self.image_validation
-        self.vm_name.validation = self.vm_name_validation
-        self.specs.validation = self.specs_validation
-        self.network.validation = self.network_validation
-
-        fields = [self.vm_name, self.image, self.specs, self.network]
-
-        super().__init__(data=data, fields=fields)
+        super().__init__(data=data)
 
     def image_validation(self):
         try:
-            image_uuid = helper.resolve_image_name(
-                self.image.value, shared.etcd_client
-            )
-        except Exception as e:
-            logger.exception(
-                "Cannot resolve image name = %s", self.image.value
-            )
-            self.add_error(str(e))
+            image_uuid = helper.resolve_image_name(self.image.value)
+        except Exception:
+            raise ValidationException('No image of name \'{}\' found'.format(self.image.value))
         else:
             self.image_uuid = image_uuid
 
     def vm_name_validation(self):
-        if resolve_vm_name(
-            name=self.vm_name.value, owner=self.name.value
-        ):
-            self.add_error(
-                'VM with same name "{}" already exists'.format(
-                    self.vm_name.value
-                )
-            )
+        if resolve_vm_name(name=self.vm_name.value, owner=self.name.value):
+            raise ValidationException("VM with same name '{}' already exists".format(self.vm_name.value))
 
     def network_validation(self):
         _network = self.network.value
 
         if _network:
             for net in _network:
-                network = shared.etcd_client.get(
-                    os.path.join(
-                        shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
-                        self.name.value,
-                        net,
-                    ),
-                    value_in_json=True,
-                )
-                if not network:
-                    self.add_error(
-                        "Network with name {} does not exists".format(
-                            net
-                        )
+                try:
+                    shared.etcd_client.get(
+                        os.path.join(settings['etcd']['network_prefix'], self.name.value, net),
+                        value_in_json=True
                     )
+                except KeyError:
+                    raise ValidationException('Network with name {} does not exists'.format(net))
 
     def specs_validation(self):
-        ALLOWED_BASE = 10
+        allowed_base = 10
 
-        _cpu = self.specs.value.get("cpu", KeyError)
-        _ram = self.specs.value.get("ram", KeyError)
-        _os_ssd = self.specs.value.get("os-ssd", KeyError)
-        _hdd = self.specs.value.get("hdd", KeyError)
-
-        if KeyError in [_cpu, _ram, _os_ssd, _hdd]:
-            self.add_error(
-                "You must specify CPU, RAM and OS-SSD in your specs"
-            )
-            return None
         try:
-            parsed_ram = bitmath.parse_string_unsafe(_ram)
-            parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd)
-
-            if parsed_ram.base != ALLOWED_BASE:
-                self.add_error(
-                    "Your specified RAM is not in correct units"
-                )
-            if parsed_os_ssd.base != ALLOWED_BASE:
-                self.add_error(
-                    "Your specified OS-SSD is not in correct units"
-                )
-
-            if int(_cpu) < 1:
-                self.add_error("CPU must be atleast 1")
-
-            if parsed_ram < bitmath.GB(1):
-                self.add_error("RAM must be atleast 1 GB")
-
-            if parsed_os_ssd < bitmath.GB(1):
-                self.add_error("OS-SSD must be atleast 1 GB")
-
-            parsed_hdd = []
-            for hdd in _hdd:
-                _parsed_hdd = bitmath.parse_string_unsafe(hdd)
-                if _parsed_hdd.base != ALLOWED_BASE:
-                    self.add_error(
-                        "Your specified HDD is not in correct units"
-                    )
-                    break
-                else:
-                    parsed_hdd.append(str(_parsed_hdd))
-
-        except ValueError:
-            # TODO: Find some good error message
-            self.add_error("Specs are not correct.")
+            _cpu = get(self.specs.value, 'cpu')
+            _ram = get(self.specs.value, 'ram')
+            _os_ssd = get(self.specs.value, 'os-ssd')
+            _hdd = get(self.specs.value, 'hdd', return_default=True, default=[])
+        except (KeyError, Exception):
+            raise ValidationException('You must specify CPU, RAM and OS-SSD in your specs')
         else:
-            if self.get_errors():
+            try:
+                parsed_ram = bitmath.parse_string_unsafe(_ram)
+                parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd)
+
+                if parsed_ram.base != allowed_base:
+                    raise ValidationException('Your specified RAM is not in correct units')
+
+                if parsed_os_ssd.base != allowed_base:
+                    raise ValidationException('Your specified OS-SSD is not in correct units')
+
+                if int(_cpu) < 1:
+                    raise ValidationException('CPU must be atleast 1')
+
+                if parsed_ram < bitmath.GB(1):
+                    raise ValidationException('RAM must be atleast 1 GB')
+
+                if parsed_os_ssd < bitmath.GB(1):
+                    raise ValidationException('OS-SSD must be atleast 1 GB')
+
+                parsed_hdd = []
+                for hdd in _hdd:
+                    _parsed_hdd = bitmath.parse_string_unsafe(hdd)
+                    if _parsed_hdd.base != allowed_base:
+                        raise ValidationException('Your specified HDD is not in correct units')
+                    else:
+                        parsed_hdd.append(str(_parsed_hdd))
+
+            except ValueError:
+                raise ValidationException('Specs are not correct.')
+            else:
                 self.specs = {
-                    "cpu": _cpu,
-                    "ram": str(parsed_ram),
-                    "os-ssd": str(parsed_os_ssd),
-                    "hdd": parsed_hdd,
+                    'cpu': _cpu,
+                    'ram': str(parsed_ram),
+                    'os-ssd': str(parsed_os_ssd),
+                    'hdd': parsed_hdd,
                 }
 
 
 class VMStatusSchema(OTPSchema):
     def __init__(self, data):
-        data["uuid"] = (
+        data['uuid'] = (
             resolve_vm_name(
-                name=data.get("vm_name", None),
+                name=get(data, 'vm_name', return_default=True),
                 owner=(
-                    data.get("in_support_of", None)
-                    or data.get("name", None)
-                ),
+                        get(data, 'in_support_of', return_default=True) or
+                        get(data, 'name', return_default=True)
+                )
             )
             or KeyError
         )
         self.uuid = VmUUIDField(data)
 
-        fields = [self.uuid]
-
-        super().__init__(data, fields)
+        super().__init__(data)
 
     def validation(self):
         vm = shared.vm_pool.get(self.uuid.value)
-        if not (
-            vm.value["owner"] == self.name.value
-            or self.realm.value == "ungleich-admin"
-        ):
-            self.add_error("Invalid User")
+        if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'):
+            raise ValidationException('Invalid User')
 
 
 class VmActionSchema(OTPSchema):
     def __init__(self, data):
-        data["uuid"] = (
+        data['uuid'] = (
             resolve_vm_name(
-                name=data.get("vm_name", None),
+                name=get(data, 'vm_name', return_default=True),
                 owner=(
-                    data.get("in_support_of", None)
-                    or data.get("name", None)
-                ),
+                        get(data, 'in_support_of', return_default=True) or
+                        get(data, 'name', return_default=True)
+                )
             )
             or KeyError
         )
         self.uuid = VmUUIDField(data)
-        self.action = Field("action", str, data.get("action", KeyError))
+        self.action = Field('action', str, get(data, 'action'), validators=[self.action_validation])
 
-        self.action.validation = self.action_validation
-
-        _fields = [self.uuid, self.action]
-
-        super().__init__(data=data, fields=_fields)
+        super().__init__(data=data)
 
     def action_validation(self):
-        allowed_actions = ["start", "stop", "delete"]
+        allowed_actions = ['start', 'stop', 'delete']
         if self.action.value not in allowed_actions:
-            self.add_error(
-                "Invalid Action. Allowed Actions are {}".format(
-                    allowed_actions
-                )
-            )
+            raise ValidationException('Invalid Action. Allowed Actions are {}'.format(allowed_actions))
 
     def validation(self):
         vm = shared.vm_pool.get(self.uuid.value)
-        if not (
-            vm.value["owner"] == self.name.value
-            or self.realm.value == "ungleich-admin"
-        ):
-            self.add_error("Invalid User")
+        if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'):
+            raise ValidationException('Invalid User.')
 
-        if (
-            self.action.value == "start"
-            and vm.status == VMStatus.running
-            and vm.hostname != ""
-        ):
-            self.add_error("VM Already Running")
+        if self.action.value == 'start' and vm.status == VMStatus.running and vm.hostname != '':
+            raise ValidationException('VM Already Running')
 
-        if self.action.value == "stop":
+        if self.action.value == 'stop':
             if vm.status == VMStatus.stopped:
-                self.add_error("VM Already Stopped")
+                raise ValidationException('VM Already Stopped')
             elif vm.status != VMStatus.running:
-                self.add_error("Cannot stop non-running VM")
+                raise ValidationException('Cannot stop non-running VM')
 
 
 class VmMigrationSchema(OTPSchema):
     def __init__(self, data):
-        data["uuid"] = (
+        data['uuid'] = (
             resolve_vm_name(
-                name=data.get("vm_name", None),
+                name=get(data, 'vm_name', return_default=True),
                 owner=(
-                    data.get("in_support_of", None)
-                    or data.get("name", None)
-                ),
-            )
-            or KeyError
+                        get(data, 'in_support_of', return_default=True) or
+                        get(data, 'name', return_default=True)
+                )
+            ) or KeyError
         )
 
         self.uuid = VmUUIDField(data)
-        self.destination = Field(
-            "destination", str, data.get("destination", KeyError)
-        )
+        self.destination = Field('destination', str, get(data, 'destination'),
+                                 validators=[self.destination_validation])
 
-        self.destination.validation = self.destination_validation
-
-        fields = [self.destination]
-        super().__init__(data=data, fields=fields)
+        super().__init__(data=data)
 
     def destination_validation(self):
         hostname = self.destination.value
-        host = next(
-            filter(
-                lambda h: h.hostname == hostname, shared.host_pool.hosts
-            ),
-            None,
-        )
+        host = next(filter(lambda h: h.hostname == hostname, shared.host_pool.hosts), None,)
         if not host:
-            self.add_error(
-                "No Such Host ({}) exists".format(
-                    self.destination.value
-                )
-            )
+            raise ValidationException('No Such Host ({}) exists'.format(self.destination.value))
         elif host.status != HostStatus.alive:
-            self.add_error("Destination Host is dead")
+            raise ValidationException('Destination Host is dead')
         else:
             self.destination.value = host.key
 
     def validation(self):
         vm = shared.vm_pool.get(self.uuid.value)
-        if not (
-            vm.value["owner"] == self.name.value
-            or self.realm.value == "ungleich-admin"
-        ):
-            self.add_error("Invalid User")
+        if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'):
+            raise ValidationException('Invalid User')
 
         if vm.status != VMStatus.running:
-            self.add_error("Can't migrate non-running VM")
+            raise ValidationException("Can't migrate non-running VM")
 
-        if vm.hostname == os.path.join(
-            shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
-        ):
-            self.add_error(
-                "Destination host couldn't be same as Source Host"
-            )
+        if vm.hostname == os.path.join(settings['etcd']['host_prefix'], self.destination.value):
+            raise ValidationException("Destination host couldn't be same as Source Host")
 
 
 class AddSSHSchema(OTPSchema):
     def __init__(self, data):
-        self.key_name = Field(
-            "key_name", str, data.get("key_name", KeyError)
-        )
-        self.key = Field("key", str, data.get("key_name", KeyError))
-
-        fields = [self.key_name, self.key]
-        super().__init__(data=data, fields=fields)
+        self.key_name = Field('key_name', str, get(data, 'key_name'))
+        self.key = Field('key', str, get(data, 'key'))
+        super().__init__(data=data)
 
 
 class RemoveSSHSchema(OTPSchema):
     def __init__(self, data):
-        self.key_name = Field(
-            "key_name", str, data.get("key_name", KeyError)
-        )
-
-        fields = [self.key_name]
-        super().__init__(data=data, fields=fields)
+        self.key_name = Field('key_name', str, get(data, 'key_name'))
+        super().__init__(data=data)
 
 
 class GetSSHSchema(OTPSchema):
     def __init__(self, data):
-        self.key_name = Field(
-            "key_name", str, data.get("key_name", None)
-        )
-
-        fields = [self.key_name]
-        super().__init__(data=data, fields=fields)
+        self.key_name = Field('key_name', str, get(data, 'key_name', return_default=True))
+        super().__init__(data=data)
 
 
 class CreateNetwork(OTPSchema):
     def __init__(self, data):
-        self.network_name = Field("network_name", str, data.get("network_name", KeyError))
-        self.type = Field("type", str, data.get("type", KeyError))
-        self.user = Field("user", bool, bool(data.get("user", False)))
-
-        self.network_name.validation = self.network_name_validation
-        self.type.validation = self.network_type_validation
-
-        fields = [self.network_name, self.type, self.user]
-        super().__init__(data, fields=fields)
+        self.network_name = Field('network_name', str, get(data, 'name'),
+                                  validators=[self.network_name_validation])
+        self.type = Field('type', str, get(data, 'type'), validators=[self.network_type_validation])
+        self.user = Field('user', bool, bool(get(data, 'user', return_default=True, default=False)))
+        super().__init__(data)
 
     def network_name_validation(self):
-        key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
+        key = os.path.join(settings['etcd']['network_prefix'], self.name.value, self.network_name.value)
         network = shared.etcd_client.get(key, value_in_json=True)
         if network:
-            self.add_error(
-                "Network with name {} already exists".format(
-                    self.network_name.value
-                )
-            )
+            raise ValidationException('Network with name {} already exists'.format(self.network_name.value))
 
     def network_type_validation(self):
-        supported_network_types = ["vxlan"]
+        supported_network_types = ['vxlan']
         if self.type.value not in supported_network_types:
-            self.add_error(
-                "Unsupported Network Type. Supported network types are {}".format(
-                    supported_network_types
-                )
-            )
+            raise ValidationException('Unsupported Network Type. Supported network types are {}'.format(supported_network_types))

uncloud/cli/helper.py

@@ -5,15 +5,15 @@ import binascii
 
 from pyotp import TOTP
 from os.path import join as join_path
-from uncloud.common.shared import shared
+from uncloud.common.settings import settings
 
 
 def get_otp_parser():
     otp_parser = argparse.ArgumentParser('otp')
-    otp_parser.add_argument('--name')
-    otp_parser.add_argument('--realm')
-    otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
-
+    otp_parser.add_argument('--name', default=settings['client']['name'])
+    otp_parser.add_argument('--realm', default=settings['client']['realm'])
+    otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'],
+                            dest='token', metavar='SEED')
     return otp_parser
 
 
@@ -25,15 +25,11 @@ def load_dump_pretty(content):
 
 
 def make_request(*args, data=None, request_method=requests.post):
+    r = request_method(join_path(settings['client']['api_server'], *args), json=data)
     try:
-        r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
-    except requests.exceptions.RequestException:
-        print('Error occurred while connecting to API server.')
-    else:
-        try:
-            print(load_dump_pretty(r.content))
-        except Exception:
-            print('Error occurred while getting output from api server.')
+        print(load_dump_pretty(r.content))
+    except Exception:
+        print('Error occurred while getting output from api server.')
 
 
 def get_token(seed):

uncloud/cli/main.py

@@ -12,12 +12,12 @@ for component in ['user', 'host', 'image', 'network', 'vm']:
     subparser.add_parser(name=parser.prog, parents=[parser])
 
 
-def main(arguments):
-    if not arguments['subcommand']:
+def main(**kwargs):
+    if not kwargs['subcommand']:
         arg_parser.print_help()
     else:
-        name = arguments.pop('subcommand')
-        arguments.pop('debug')
+        name = kwargs.pop('subcommand')
+        kwargs.pop('debug')
         mod = importlib.import_module('uncloud.cli.{}'.format(name))
         _main = getattr(mod, 'main')
-        _main(**arguments)
+        _main(**kwargs)

uncloud/common/cli.py

@@ -1,26 +0,0 @@
-from uncloud.common.shared import shared
-from pyotp import TOTP
-
-
-def get_token(seed):
-    if seed is not None:
-        try:
-            token = TOTP(seed).now()
-        except Exception:
-            raise Exception('Invalid seed')
-        else:
-            return token
-
-
-def resolve_otp_credentials(kwargs):
-    d = {
-        'name': shared.settings['client']['name'],
-        'realm': shared.settings['client']['realm'],
-        'token': get_token(shared.settings['client']['seed'])
-    }
-
-    for k, v in d.items():
-        if k in kwargs and kwargs[k] is None:
-            kwargs.update({k: v})
-
-    return d

uncloud/common/counters.py

@@ -1,8 +1,8 @@
-from .etcd_wrapper import Etcd3Wrapper
+from uncloud.common.shared import shared
 
 
-def increment_etcd_counter(etcd_client: Etcd3Wrapper, key):
-    kv = etcd_client.get(key)
+def increment_etcd_counter(key):
+    kv = shared.etcd_client.get(key)
 
     if kv:
         counter = int(kv.value)
@@ -10,12 +10,12 @@ def increment_etcd_counter(etcd_client: Etcd3Wrapper, key):
     else:
         counter = 1
 
-    etcd_client.put(key, str(counter))
+    shared.etcd_client.put(key, str(counter))
     return counter
 
 
-def get_etcd_counter(etcd_client: Etcd3Wrapper, key):
-    kv = etcd_client.get(key)
+def get_etcd_counter(key):
+    kv = shared.etcd_client.get(key)
     if kv:
         return int(kv.value)
     return None

uncloud/common/etcd_wrapper.py

@@ -5,6 +5,7 @@ from functools import wraps
 
 from uncloud import UncloudException
 from uncloud.common import logger
+from typing import Iterator
 
 
 class EtcdEntry:
@@ -42,14 +43,30 @@ class Etcd3Wrapper:
         self.client = etcd3.client(*args, **kwargs)
 
     @readable_errors
-    def get(self, *args, value_in_json=False, **kwargs):
+    def get(self, *args, value_in_json=False, **kwargs) -> EtcdEntry:
+        """Get a key/value pair from etcd
+
+        :return:
+            EtcdEntry: if a key/value pair is found in etcd
+        :raises:
+            KeyError: If key is not found in etcd
+            Exception: Different type of exception can be raised depending on
+                       situation
+        """
         _value, _key = self.client.get(*args, **kwargs)
         if _key is None or _value is None:
-            return None
+            raise KeyError
         return EtcdEntry(_key, _value, value_in_json=value_in_json)
 
     @readable_errors
     def put(self, *args, value_in_json=False, **kwargs):
+        """Put key/value pair in etcd
+
+        :return: a response containing a header and the prev_kv
+        :raises:
+            Exception:  Different type of exception can be raised depending on
+                        situation
+        """
         _key, _value = args
         if value_in_json:
             _value = json.dumps(_value)
@@ -60,16 +77,28 @@ class Etcd3Wrapper:
         return self.client.put(_key, _value, **kwargs)
 
     @readable_errors
-    def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs):
+    def get_prefix(self, *args, value_in_json=False, **kwargs) -> \
+            Iterator[EtcdEntry]:
         event_iterator = self.client.get_prefix(*args, **kwargs)
         for e in event_iterator:
             yield EtcdEntry(*e[::-1], value_in_json=value_in_json)
 
     @readable_errors
-    def watch_prefix(self, key, raise_exception=True, value_in_json=False):
-        event_iterator, cancel = self.client.watch_prefix(key)
-        for e in event_iterator:
-            if hasattr(e, '_event'):
-                e = e._event
-                if e.type == e.PUT:
-                    yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)
+    def watch_prefix(self, key, raise_exception=True, value_in_json=False) -> Iterator[EtcdEntry]:
+        try:
+            event_iterator, cancel = self.client.watch_prefix(key)
+            for e in event_iterator:
+                if hasattr(e, '_event'):
+                    e = e._event
+                    if e.type == e.PUT:
+                        yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)
+        except Exception as err:
+            if raise_exception:
+                raise Exception('Exception in etcd_wrapper.get_prefix') from err
+            else:
+                logger.exception('Error in etcd_wrapper.watch_prefix')
+                try:
+                    cancel()
+                except Exception:
+                    pass
+                return iter([])

uncloud/common/network.py

@@ -1,6 +1,7 @@
 import subprocess as sp
 import random
 import logging
+import ipaddress
 
 logger = logging.getLogger(__name__)
 
@@ -9,9 +10,7 @@ def random_bytes(num=6):
     return [random.randrange(256) for _ in range(num)]
 
 
-def generate_mac(
-    uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
-):
+def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
     mac = random_bytes()
     if oui:
         if type(oui) == str:
@@ -68,3 +67,21 @@ def delete_network_interface(iface):
     except Exception:
         logger.exception("Interface %s Deletion failed", iface)
 
+
+def mac2ipv6(mac, prefix):
+    # only accept MACs separated by a colon
+    parts = mac.split(':')
+
+    # modify parts to match IPv6 value
+    parts.insert(3, 'ff')
+    parts.insert(4, 'fe')
+    parts[0] = '%x' % (int(parts[0], 16) ^ 2)
+
+    # format output
+    ipv6_parts = [str(0)] * 4
+    for i in range(0, len(parts), 2):
+        ipv6_parts.append(''.join(parts[i: i + 2]))
+
+    lower_part = ipaddress.IPv6Address(':'.join(ipv6_parts))
+    prefix = ipaddress.IPv6Address(prefix)
+    return str(prefix + int(lower_part))

uncloud/common/settings.py

@@ -8,7 +8,6 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper
 from os.path import join as join_path
 
 logger = logging.getLogger(__name__)
-settings = None
 
 
 class CustomConfigParser(configparser.RawConfigParser):
@@ -26,8 +25,9 @@ class CustomConfigParser(configparser.RawConfigParser):
 
 
 class Settings(object):
-    def __init__(self, conf_dir, seed_value=None):
+    def __init__(self):
         conf_name = 'uncloud.conf'
+        conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
         self.config_file = join_path(conf_dir, conf_name)
 
         # this is used to cache config from etcd for 1 minutes. Without this we
@@ -38,19 +38,15 @@ class Settings(object):
         self.config_parser.add_section('etcd')
         self.config_parser.set('etcd', 'base_prefix', '/')
 
-        if os.access(self.config_file, os.R_OK):
+        try:
             self.config_parser.read(self.config_file)
-        else:
-            raise FileNotFoundError('Config file %s not found!', self.config_file)
+        except Exception as err:
+            logger.error('%s', err)
+
         self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
 
         self.read_internal_values()
 
-        if seed_value is None:
-            seed_value = dict()
-
-        self.config_parser.read_dict(seed_value)
-
     def get_etcd_client(self):
         args = tuple()
         try:
@@ -103,10 +99,12 @@ class Settings(object):
     def read_config_file_values(self, config_file):
         try:
             # Trying to read configuration file
-            with open(config_file) as config_file_handle:
+            with open(config_file, 'r') as config_file_handle:
                 self.config_parser.read_file(config_file_handle)
         except FileNotFoundError:
-            sys.exit('Configuration file {} not found!'.format(config_file))
+            sys.exit(
+                'Configuration file {} not found!'.format(config_file)
+            )
         except Exception as err:
             logger.exception(err)
             sys.exit('Error occurred while reading configuration file')
@@ -132,5 +130,4 @@ class Settings(object):
         return self.config_parser[key]
 
 
-def get_settings():
-    return settings
+settings = Settings()

uncloud/common/shared.py

@@ -1,34 +1,34 @@
-from uncloud.common.settings import get_settings
+from uncloud.common.settings import settings
 from uncloud.common.vm import VmPool
 from uncloud.common.host import HostPool
 from uncloud.common.request import RequestPool
-import uncloud.common.storage_handlers as storage_handlers
+from uncloud.common.storage_handlers import get_storage_handler
 
 
 class Shared:
-    @property
-    def settings(self):
-        return get_settings()
-
     @property
     def etcd_client(self):
-        return self.settings.get_etcd_client()
+        return settings.get_etcd_client()
 
     @property
     def host_pool(self):
-        return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
+        return HostPool(
+            self.etcd_client, settings["etcd"]["host_prefix"]
+        )
 
     @property
     def vm_pool(self):
-        return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
+        return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"])
 
     @property
     def request_pool(self):
-        return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
+        return RequestPool(
+            self.etcd_client, settings["etcd"]["request_prefix"]
+        )
 
     @property
     def storage_handler(self):
-        return storage_handlers.get_storage_handler()
+        return get_storage_handler()
 
 
 shared = Shared()

uncloud/common/storage_handlers.py

@@ -6,7 +6,8 @@ import stat
 from abc import ABC
 from . import logger
 from os.path import join as join_path
-import uncloud.common.shared as shared
+
+from uncloud.common.settings import settings as config
 
 
 class ImageStorageHandler(ABC):
@@ -192,16 +193,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
 
 
 def get_storage_handler():
-    __storage_backend = shared.shared.settings["storage"]["storage_backend"]
+    __storage_backend = config["storage"]["storage_backend"]
     if __storage_backend == "filesystem":
         return FileSystemBasedImageStorageHandler(
-            vm_base=shared.shared.settings["storage"]["vm_dir"],
-            image_base=shared.shared.settings["storage"]["image_dir"],
+            vm_base=config["storage"]["vm_dir"],
+            image_base=config["storage"]["image_dir"],
         )
     elif __storage_backend == "ceph":
         return CEPHBasedImageStorageHandler(
-            vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
-            image_base=shared.shared.settings["storage"]["ceph_image_pool"],
+            vm_base=config["storage"]["ceph_vm_pool"],
+            image_base=config["storage"]["ceph_image_pool"],
         )
     else:
-        raise Exception("Unknown Image Storage Handler")
+        raise Exception("Unknown Image Storage Handler")

uncloud/configure/main.py

@@ -1,6 +1,7 @@
 import os
 import argparse
 
+from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 
 arg_parser = argparse.ArgumentParser('configure', add_help=False)
@@ -39,19 +40,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
 
 
 def update_config(section, kwargs):
-    uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
+    uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
     if not uncloud_config:
         uncloud_config = {}
     else:
         uncloud_config = uncloud_config.value
 
     uncloud_config[section] = kwargs
-    shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
+    shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
 
 
-def main(arguments):
-    subcommand = arguments['subcommand']
+def main(**kwargs):
+    subcommand = kwargs.pop('subcommand')
     if not subcommand:
         arg_parser.print_help()
     else:
-        update_config(subcommand, arguments)
+        update_config(subcommand, kwargs)

uncloud/docs/Makefile

@@ -7,7 +7,7 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source/
 BUILDDIR      = build/
-DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/
+DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/
 
 .PHONY: all build clean
 

uncloud/docs/source/admin-guide

@@ -56,13 +56,40 @@ To start host we created earlier, execute the following command
 
     ucloud host ungleich.ch
 
-File & image scanners
---------------------------
+Create OS Image
+---------------
 
-Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our
-uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by
-ucloud. We can only make images from tracked files. So, we need to track the
-file by running File Scanner
+Create ucloud-init ready OS image (Optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+This step is optional if you just want to test ucloud. However, sooner or later
+you want to create OS images with ucloud-init to properly
+contexualize VMs.
+
+1. Start a VM with OS image on which you want to install ucloud-init
+2. Execute the following command on the started VM
+
+   .. code-block:: sh
+
+    apk add git
+    git clone https://code.ungleich.ch/ucloud/ucloud-init.git
+    cd ucloud-init
+    sh ./install.sh
+3. Congratulations. Your image is now ucloud-init ready.
+
+
+Upload Sample OS Image
+~~~~~~~~~~~~~~~~~~~~~~
+Execute the following to get the sample OS image file.
+
+.. code-block:: sh
+
+    mkdir /var/www/admin
+    (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download)
+
+Run File Scanner and Image Scanner
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make
+images from tracked files. So, we need to track the file by running File Scanner
 
 .. code-block:: sh
 

uncloud/docs/source/hacking.rst

@@ -0,0 +1,17 @@
+Hacking
+=======
+How to hack on the code.
+
+[ to be done by Balazs:
+
+* make nice
+* indent with shell script mode
+
+]
+
+* git clone the repo
+* cd to the repo
+* Setup your venv: python -m venv venv
+* . ./venv/bin/activate # you need the leading dot for sourcing!
+* Run ./bin/ucloud-run-reinstall - it should print you an error
+  message on how to use ucloud

uncloud/docs/source/index.rst

@@ -11,13 +11,14 @@ Welcome to ucloud's documentation!
    :caption: Contents:
 
    introduction
-   setup-install
-   vm-images
    user-guide
+   setup-install
    admin-guide
+   user-guide/how-to-create-an-os-image-for-ucloud
    troubleshooting
    hacking
 
+
 Indices and tables
 ==================
 

uncloud/filescanner/main.py

@@ -9,6 +9,7 @@ import bitmath
 from uuid import uuid4
 
 from . import logger
+from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 
 arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
@@ -52,7 +53,7 @@ def track_file(file, base_dir, host):
         file_path = file_path.relative_to(owner)
         creation_date = time.ctime(os.stat(file_str).st_ctime)
 
-        entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
+        entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
         entry_value = {
             'filename': str(file_path),
             'owner': owner,
@@ -67,9 +68,8 @@ def track_file(file, base_dir, host):
         shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
 
 
-def main(arguments):
-    hostname = arguments['hostname']
-    base_dir = shared.settings['storage']['file_dir']
+def main(hostname, debug=False):
+    base_dir = settings['storage']['file_dir']
     # Recursively Get All Files and Folder below BASE_DIR
     files = glob.glob('{}/**'.format(base_dir), recursive=True)
     files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
@@ -77,7 +77,7 @@ def main(arguments):
     # Files that are already tracked
     tracked_files = [
         pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
-        for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
+        for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
         if f.value['host'] == hostname
     ]
     untracked_files = set(files) - set(tracked_files)

uncloud/hack/config.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
-#
-# This file is part of uncloud.
-#
-# uncloud is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# uncloud is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-class Config(object):
-    def __init__(self, arguments):
-        """ read arguments dicts as a base """
-
-        self.arguments = arguments
-
-        # Split them so *etcd_args can be used and we can
-        # iterate over etcd_hosts
-        self.etcd_hosts = [ arguments['etcd_host'] ]
-        self.etcd_args = {
-            'ca_cert': arguments['etcd_ca_cert'],
-            'cert_cert': arguments['etcd_cert_cert'],
-            'cert_key': arguments['etcd_cert_key'],
-#            'user': None,
-#            'password': None
-        }
-        self.etcd_prefix = '/nicohack/'

uncloud/hack/db.py

@@ -1,113 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
-#
-# This file is part of uncloud.
-#
-# uncloud is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# uncloud is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import etcd3
-import json
-import logging
-
-from functools import wraps
-from uncloud import UncloudException
-
-log = logging.getLogger(__name__)
-
-
-def readable_errors(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except etcd3.exceptions.ConnectionFailedError as e:
-            raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
-        except etcd3.exceptions.ConnectionTimeoutError as e:
-            raise UncloudException('etcd connection timeout. {}'.format(e))
-
-    return wrapper
-
-
-class DB(object):
-    def __init__(self, config, prefix="/"):
-        self.config = config
-
-        # Root for everything
-        self.base_prefix= '/nicohack'
-
-        # Can be set from outside
-        self.prefix = prefix
-
-        self.connect()
-
-    @readable_errors
-    def connect(self):
-        self._db_clients = []
-        for endpoint in self.config.etcd_hosts:
-            client = etcd3.client(host=endpoint, **self.config.etcd_args)
-            self._db_clients.append(client)
-
-    def realkey(self, key):
-        return "{}{}/{}".format(self.base_prefix,
-                                self.prefix,
-                                key)
-
-    @readable_errors
-    def get(self, key, as_json=False, **kwargs):
-        value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
-
-        if as_json:
-            value = json.loads(value)
-
-        return value
-
-
-    @readable_errors
-    def set(self, key, value, as_json=False, **kwargs):
-        if as_json:
-            value = json.dumps(value)
-
-        # FIXME: iterate over clients in case of failure ?
-        return self._db_clients[0].put(self.realkey(key), value, **kwargs)
-
-    @readable_errors
-    def increment(self, key, **kwargs):
-        print(self.realkey(key))
-
-
-        print("prelock")
-        lock = self._db_clients[0].lock('/nicohack/foo')
-        print("prelockacq")
-        lock.acquire()
-        print("prelockrelease")
-        lock.release()
-
-        with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock:
-            print("in lock")
-            pass
-
-#        with self._db_clients[0].lock(self.realkey(key)) as lock:#            value = int(self.get(self.realkey(key), **kwargs))
-#            self.set(self.realkey(key), str(value + 1), **kwargs)
-
-
-if __name__ == '__main__':
-    endpoints = [ "https://etcd1.ungleich.ch:2379",
-                  "https://etcd2.ungleich.ch:2379",
-                  "https://etcd3.ungleich.ch:2379" ]
-
-    db = DB(url=endpoints)

uncloud/hack/hackcloud/.gitignore

@@ -1,3 +0,0 @@
-*.iso
-radvdpid
-foo

uncloud/hack/hackcloud/__init__.py

@@ -1 +0,0 @@
-

uncloud/hack/hackcloud/etcd-client.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-etcdctl  --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \
-         --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \
-         --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \
-         --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@"

uncloud/hack/hackcloud/ifdown.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-echo $@

uncloud/hack/hackcloud/ifup.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-dev=$1; shift
-
-# bridge is setup from outside
-ip link set dev "$dev" master ${bridge}
-ip link set dev "$dev" up

uncloud/hack/hackcloud/mac-last

@@ -1 +0,0 @@
-000000000252

uncloud/hack/hackcloud/mac-prefix

@@ -1 +0,0 @@
-02:00

uncloud/hack/hackcloud/net.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-set -x
-
-netid=100
-dev=wlp2s0
-dev=wlp0s20f3
-#dev=wlan0
-
-ip=2a0a:e5c1:111:888::48/64
-vxlandev=vxlan${netid}
-bridgedev=br${netid}
-
-ip -6 link add ${vxlandev} type vxlan \
-    id ${netid} \
-    dstport 4789 \
-    group ff05::${netid} \
-    dev ${dev} \
-    ttl 5
-
-ip link set ${vxlandev} up
-
-
-ip link add ${bridgedev} type bridge
-ip link set ${bridgedev} up
-
-ip link set ${vxlandev} master ${bridgedev} up
-
-ip addr add ${ip} dev ${bridgedev}

uncloud/hack/hackcloud/nftrules

@@ -1,31 +0,0 @@
-flush ruleset
-
-table bridge filter {
-    chain prerouting {
-        type filter hook prerouting priority 0;
-        policy accept;
-
-        ibrname br100 jump br100
-    }
-
-    chain br100 {
-        # Allow all incoming traffic from outside
-        iifname vxlan100 accept
-
-        # Default blocks: router advertisements, dhcpv6, dhcpv4
-        icmpv6 type nd-router-advert drop
-        ip6 version 6 udp sport 547 drop
-        ip  version 4 udp sport 67 drop
-
-        jump br100_vmlist
-        drop
-    }
-    chain br100_vmlist {
-        # VM1
-        iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept
-
-        # VM2
-        iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept
-        iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept
-    }
-}

uncloud/hack/hackcloud/radvd.conf

@@ -1,13 +0,0 @@
-interface br100
-{
-  AdvSendAdvert on;
-  MinRtrAdvInterval 3;
-  MaxRtrAdvInterval 5;
-  AdvDefaultLifetime 3600;
-
-  prefix 2a0a:e5c1:111:888::/64 {
-  };
-
-  RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; };
-  DNSSL place7.ungleich.ch {  AdvDNSSLLifetime 6000; } ;
-};

uncloud/hack/hackcloud/radvd.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-radvd -C ./radvd.conf  -n -p ./radvdpid

uncloud/hack/hackcloud/vm.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-# if [ $# -ne 1 ]; then
-#     echo "$0: owner"
-#     exit 1
-# fi
-
-qemu=/usr/bin/qemu-system-x86_64
-
-accel=kvm
-#accel=tcg
-
-memory=1024
-cores=2
-uuid=$(uuidgen)
-mac=$(./mac-gen.py)
-owner=nico
-
-export bridge=br100
-
-set -x
-$qemu -name "uncloud-${uuid}" \
-      -machine pc,accel=${accel} \
-      -m ${memory} \
-      -smp ${cores} \
-      -uuid ${uuid} \
-      -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \
-      -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \
-      -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac}

uncloud/hack/mac.py

@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2012 Nico Schottelius (nico-cinv at schottelius.org)
-#
-# This file is part of cinv.
-#
-# cinv is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cinv is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cinv. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import argparse
-import logging
-import os.path
-import os
-import re
-import json
-
-from uncloud import UncloudException
-from uncloud.hack.db import DB
-
-log = logging.getLogger(__name__)
-
-
-class MAC(object):
-    def __init__(self, config):
-        self.config = config
-        self.no_db = self.config.arguments['no_db']
-        if not self.no_db:
-            self.db = DB(config, prefix="/mac")
-
-        self.prefix = 0x420000000000
-        self._number = 0 # Not set by default
-
-    @staticmethod
-    def validate_mac(mac):
-        if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I):
-            raise Error("Not a valid mac address: %s" % mac)
-
-    def last_used_index(self):
-        if not self.no_db:
-            value = self.db.get("last_used_index")
-            if not value:
-                self.db.set("last_used_index", "0")
-                value = self.db.get("last_used_index")
-
-        else:
-            value = "0"
-
-        return int(value)
-
-    def last_used_mac(self):
-        return self.int_to_mac(self.prefix + self.last_used_index())
-
-    def to_colon_format(self):
-        b = self._number.to_bytes(6, byteorder="big")
-        return ':'.join(format(s, '02x') for s in b)
-
-    def to_str_format(self):
-        b = self._number.to_bytes(6, byteorder="big")
-        return ''.join(format(s, '02x') for s in b)
-
-    def create(self):
-        last_number = self.last_used_index()
-
-        if last_number == int('0xffffffff', 16):
-            raise UncloudException("Exhausted all possible mac addresses - try to free some")
-
-        next_number = last_number + 1
-        self._number = self.prefix + next_number
-
-        #next_number_string = "{:012x}".format(next_number)
-        #next_mac = self.int_to_mac(next_mac_number)
-        # db_entry = {}
-        # db_entry['vm_uuid'] = vmuuid
-        # db_entry['index'] = next_number
-        # db_entry['mac_address'] = next_mac
-
-        # should be one transaction
-        # self.db.increment("last_used_index")
-        # self.db.set("used/{}".format(next_mac),
-        #             db_entry, as_json=True)
-
-    def __int__(self):
-        return self._number
-
-    def __repr__(self):
-        return self.to_str_format()
-
-    def __str__(self):
-        return self.to_colon_format()

uncloud/hack/main.py

@@ -1,92 +0,0 @@
-import argparse
-import logging
-
-from uncloud.hack.vm import VM
-from uncloud.hack.config import Config
-from uncloud.hack.mac import MAC
-from uncloud.hack.net import VXLANBridge, DNSRA
-
-from uncloud import UncloudException
-
-arg_parser = argparse.ArgumentParser('hack', add_help=False)
-                                     #description="Commands that are unfinished - use at own risk")
-arg_parser.add_argument('--last-used-mac', action='store_true')
-arg_parser.add_argument('--get-new-mac', action='store_true')
-
-arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true')
-arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true')
-arg_parser.add_argument('--network', help="/64 IPv6 network")
-arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0")
-arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int)
-arg_parser.add_argument('--run-dns-ra', action='store_true',
-                        help="Provide router advertisements and DNS resolution via dnsmasq")
-arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true')
-
-arg_parser.add_argument('--create-vm', action='store_true')
-arg_parser.add_argument('--destroy-vm', action='store_true')
-arg_parser.add_argument('--get-vm-status', action='store_true')
-arg_parser.add_argument('--get-vm-vnc', action='store_true')
-arg_parser.add_argument('--list-vms', action='store_true')
-arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int)
-arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int)
-arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image")
-arg_parser.add_argument('--uuid', help="VM UUID")
-
-arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true')
-arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh")
-
-
-log = logging.getLogger(__name__)
-
-
-def main(arguments):
-    config = Config(arguments)
-
-    if arguments['create_vm']:
-        vm = VM(config)
-        vm.create()
-
-    if arguments['destroy_vm']:
-        vm = VM(config)
-        vm.stop()
-
-    if arguments['get_vm_status']:
-        vm = VM(config)
-        vm.status()
-
-    if arguments['get_vm_vnc']:
-        vm = VM(config)
-        vm.vnc_addr()
-
-    if arguments['list_vms']:
-        vm = VM(config)
-        vm.list()
-
-    if arguments['last_used_mac']:
-        m = MAC(config)
-        print(m.last_used_mac())
-
-    if arguments['get_new_mac']:
-        print(MAC(config).get_next())
-
-    #if arguments['init_network']:
-    if arguments['create_vxlan']:
-        if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']:
-            raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)")
-        vb = VXLANBridge(vni=arguments['vni'],
-                         route=arguments['network'],
-                         uplinkdev=arguments['vxlan_uplink_device'],
-                         use_sudo=arguments['use_sudo'])
-        vb._setup_vxlan()
-        vb._setup_bridge()
-        vb._add_vxlan_to_bridge()
-        vb._route_network()
-
-    if arguments['run_dns_ra']:
-        if not arguments['network'] or not arguments['vni']:
-            raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)")
-
-        dnsra = DNSRA(route=arguments['network'],
-                      vni=arguments['vni'],
-                      use_sudo=arguments['use_sudo'])
-        dnsra._setup_dnsmasq()

uncloud/hack/net.py

@@ -1,116 +0,0 @@
-import subprocess
-import ipaddress
-import logging
-
-
-from uncloud import UncloudException
-
-log = logging.getLogger(__name__)
-
-
-class VXLANBridge(object):
-    cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5"
-    cmd_up_dev = "{sudo}ip link set {dev} up"
-    cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge"
-    cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up"
-    cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}"
-    cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}"
-
-    # VXLAN ids are at maximum 24 bit - use a /104
-    multicast_network = ipaddress.IPv6Network("ff05::/104")
-    max_vni = (2**24)-1
-
-    def __init__(self,
-                 vni,
-                 uplinkdev,
-                 route=None,
-                 use_sudo=False):
-        self.config = {}
-
-        if vni > self.max_vni:
-            raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni))
-
-        if use_sudo:
-            self.config['sudo'] = 'sudo '
-        else:
-            self.config['sudo'] = ''
-
-        self.config['vni_dec'] = vni
-        self.config['vni_hex'] = "{:x}".format(vni)
-        self.config['multicast_address'] = self.multicast_network[vni]
-
-        self.config['route_network'] = ipaddress.IPv6Network(route)
-        self.config['route'] = route
-
-        self.config['uplinkdev'] = uplinkdev
-        self.config['vxlandev'] = "vx{}".format(self.config['vni_hex'])
-        self.config['bridgedev'] = "br{}".format(self.config['vni_hex'])
-
-
-    def setup_networking(self):
-        pass
-
-    def _setup_vxlan(self):
-        self._execute_cmd(self.cmd_create_vxlan)
-        self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev'])
-
-    def _setup_bridge(self):
-        self._execute_cmd(self.cmd_create_bridge)
-        self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev'])
-
-    def _route_network(self):
-        self._execute_cmd(self.cmd_add_route_dev)
-
-    def _add_vxlan_to_bridge(self):
-        self._execute_cmd(self.cmd_add_to_bridge)
-
-    def _execute_cmd(self, cmd_string, **kwargs):
-        cmd = cmd_string.format(**self.config, **kwargs)
-        log.info("Executing: {}".format(cmd))
-        subprocess.run(cmd.split())
-
-class ManagementBridge(VXLANBridge):
-    pass
-
-
-class DNSRA(object):
-    # VXLAN ids are at maximum 24 bit
-    max_vni = (2**24)-1
-
-
-    # Command to start dnsmasq
-    cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra"
-
-    def __init__(self,
-                 vni,
-                 route=None,
-                 use_sudo=False):
-        self.config = {}
-
-        if vni > self.max_vni:
-            raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni))
-
-        if use_sudo:
-            self.config['sudo'] = 'sudo '
-        else:
-            self.config['sudo'] = ''
-
-        #TODO: remove if not needed
-        #self.config['vni_dec'] = vni
-        self.config['vni_hex'] = "{:x}".format(vni)
-
-        # dnsmasq only wants the network without the prefix, therefore, cut it off
-        self.config['route'] = ipaddress.IPv6Network(route).network_address
-        self.config['bridgedev'] = "br{}".format(self.config['vni_hex'])
-
-    def _setup_dnsmasq(self):
-         self._execute_cmd(self.cmd_start_dnsmasq)
-
-    def _execute_cmd(self, cmd_string, **kwargs):
-        cmd = cmd_string.format(**self.config, **kwargs)
-        log.info("Executing: {}".format(cmd))
-        print("Executing: {}".format(cmd))
-        subprocess.run(cmd.split())
-
-class Firewall(object):
-    pass

uncloud/hack/uncloud-hack-init-host

@@ -1,26 +0,0 @@
-id=100
-rawdev=eth0
-
-# create vxlan
-ip -6 link add vxlan${id} type vxlan \
-    id ${id} \
-    dstport 4789 \
-    group ff05::${id} \
-    dev ${rawdev} \
-    ttl 5
-
-ip link set vxlan${id} up
-
-# create bridge
-ip link set vxlan${id} up
-ip link set br${id} up
-
-# Add vxlan into bridge
-ip link set vxlan${id} master br${id}
-
-
-#  useradd -m uncloud
-# [18:05] tablett.place10:~# id uncloud
-# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu)
-# apk add qemu-system-x86_64
-# also needs group netdev

uncloud/hack/uncloud-run-vm

@@ -1,25 +0,0 @@
-#!/bin/sh
-
-if [ $# -ne 1 ]; then
-    echo $0 vmid
-    exit 1
-fi
-
-id=$1; shift
-
-memory=512
-macaddress=02:00:b9:cb:70:${id}
-netname=net${id}-1
-
-qemu-system-x86_64 \
-    -name uncloud-${id} \
-    -accel kvm \
-    -m ${memory} \
-    -smp 2,sockets=2,cores=1,threads=1 \
-    -device virtio-net-pci,netdev=net0,mac=$macaddress \
-    -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \
-    -vnc [::]:0
-
-# To be changed:
-# -vnc to unix path
-# or -spice

uncloud/hack/vm.py

@@ -1,136 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
-#
-# This file is part of uncloud.
-#
-# uncloud is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# uncloud is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
-
-# This module is directly called from the hack module, and can be used as follow:
-#
-#   Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix.
-#   `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2`
-#
-#   List running VMs (returns a list of UUIDs).
-#   `uncloud hack --hackprefix /tmp/hackcloud --list-vms
-#
-#   Get VM status:
-#   `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid`
-#
-#   Stop a VM:
-#   `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid`
-#    ``
-
-import subprocess
-import uuid
-import os
-import logging
-
-from uncloud.hack.db import DB
-from uncloud.hack.mac import MAC
-from uncloud.vmm import VMM
-
-log = logging.getLogger(__name__)
-log.setLevel(logging.DEBUG)
-
-class VM(object):
-    def __init__(self, config):
-        self.config = config
-
-        #TODO: Enable etcd lookup
-        self.no_db = self.config.arguments['no_db']
-        if not self.no_db:
-             self.db = DB(self.config, prefix="/vm")
-
-        # General CLI arguments.
-        self.hackprefix = self.config.arguments['hackprefix']
-        self.uuid = self.config.arguments['uuid']
-        self.memory = self.config.arguments['memory'] or '1024M'
-        self.cores = self.config.arguments['cores'] or 1
-        if self.config.arguments['image']:
-            self.image = os.path.join(self.hackprefix, self.config.arguments['image'])
-        else:
-            self.image = None
-
-        # External components.
-        self.vmm = VMM(vmm_backend=self.hackprefix)
-        self.mac = MAC(self.config)
-
-        # Harcoded & generated values.
-        self.owner = 'uncoud'
-        self.image_format='qcow2'
-        self.accel = 'kvm'
-        self.threads = 1
-        self.ifup = os.path.join(self.hackprefix, "ifup.sh")
-        self.ifdown = os.path.join(self.hackprefix, "ifdown.sh")
-        self.ifname = "uc{}".format(self.mac.to_str_format())
-
-    def get_qemu_args(self):
-        command = (
-            "-name {owner}-{name}"
-            " -machine pc,accel={accel}"
-            " -drive file={image},format={image_format},if=virtio"
-            " -device virtio-rng-pci"
-            " -m {memory} -smp cores={cores},threads={threads}"
-            " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}"
-            " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}"
-        ).format(
-            owner=self.owner, name=self.uuid,
-            accel=self.accel,
-            image=self.image, image_format=self.image_format,
-            memory=self.memory, cores=self.cores, threads=self.threads,
-            ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname,
-            mac=self.mac
-        )
-
-        return command.split(" ")
-
-    def create(self):
-        # New VM: new UUID, new MAC.
-        self.uuid = str(uuid.uuid4())
-        self.mac.create()
-
-        qemu_args = self.get_qemu_args()
-        log.debug("QEMU args passed to VMM: {}".format(qemu_args))
-        self.vmm.start(
-                uuid=self.uuid,
-                migration=False,
-                *qemu_args
-                )
-
-    def stop(self):
-        if not self.uuid:
-            print("Please specific an UUID with the --uuid flag.")
-            exit(1)
-
-        self.vmm.stop(self.uuid)
-
-    def status(self):
-        if not self.uuid:
-            print("Please specific an UUID with the --uuid flag.")
-            exit(1)
-
-        print(self.vmm.get_status(self.uuid))
-
-    def vnc_addr(self):
-        if not self.uuid:
-            print("Please specific an UUID with the --uuid flag.")
-            exit(1)
-
-        print(self.vmm.get_vnc(self.uuid))
-
-    def list(self):
-        print(self.vmm.discover())
-

uncloud/host/main.py

@@ -6,6 +6,7 @@ from uuid import uuid4
 
 from uncloud.common.request import RequestEntry, RequestType
 from uncloud.common.shared import shared
+from uncloud.common.settings import settings
 from uncloud.common.vm import VMStatus
 from uncloud.vmm import VMM
 from os.path import join as join_path
@@ -35,7 +36,7 @@ def maintenance(host):
         if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
             logger.debug('VM {} is running on {}'.format(vm_uuid, host))
             vm = shared.vm_pool.get(
-                join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
+                join_path(settings['etcd']['vm_prefix'], vm_uuid)
             )
             vm.status = VMStatus.running
             vm.vnc_socket = vmm.get_vnc(vm_uuid)
@@ -43,15 +44,14 @@ def maintenance(host):
             shared.vm_pool.put(vm)
 
 
-def main(arguments):
-    hostname = arguments['hostname']
+def main(hostname, debug=False):
     host_pool = shared.host_pool
     host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
 
     # Does not yet exist, create it
     if not host:
         host_key = join_path(
-            shared.settings['etcd']['host_prefix'], uuid4().hex
+            settings['etcd']['host_prefix'], uuid4().hex
         )
         host_entry = {
             'specs': '',
@@ -79,9 +79,9 @@ def main(arguments):
     # get prefix until either success or deamon death comes.
     while True:
         for events_iterator in [
-            shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
+            shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
                                           raise_exception=False),
-            shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
+            shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
                                             raise_exception=False)
         ]:
             for request_event in events_iterator:
@@ -94,7 +94,7 @@ def main(arguments):
 
                     shared.request_pool.client.client.delete(request_event.key)
                     vm_entry = shared.etcd_client.get(
-                        join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid)
+                        join_path(settings['etcd']['vm_prefix'], request_event.uuid)
                     )
 
                     logger.debug('VM hostname: {}'.format(vm_entry.value))

uncloud/host/virtualmachine.py

@@ -17,6 +17,7 @@ from uncloud.common.network import create_dev, delete_network_interface
 from uncloud.common.schemas import VMSchema, NetworkSchema
 from uncloud.host import logger
 from uncloud.common.shared import shared
+from uncloud.common.settings import settings
 from uncloud.vmm import VMM
 
 from marshmallow import ValidationError
@@ -90,7 +91,7 @@ class VM:
                         self.vmm.socket_dir, self.uuid
                     ),
                     destination_host_key=destination_host_key,  # Where source host transfer VM
-                    request_prefix=shared.settings["etcd"]["request_prefix"],
+                    request_prefix=settings["etcd"]["request_prefix"],
                 )
                 shared.request_pool.put(r)
             else:
@@ -118,7 +119,7 @@ class VM:
             network_name, mac, tap = network_mac_and_tap
 
             _key = os.path.join(
-                shared.settings["etcd"]["network_prefix"],
+                settings["etcd"]["network_prefix"],
                 self.vm["owner"],
                 network_name,
             )
@@ -132,13 +133,13 @@ class VM:
             if network["type"] == "vxlan":
                 tap = create_vxlan_br_tap(
                     _id=network["id"],
-                    _dev=shared.settings["network"]["vxlan_phy_dev"],
+                    _dev=settings["network"]["vxlan_phy_dev"],
                     tap_id=tap,
                     ip=network["ipv6"],
                 )
 
                 all_networks = shared.etcd_client.get_prefix(
-                    shared.settings["etcd"]["network_prefix"],
+                    settings["etcd"]["network_prefix"],
                     value_in_json=True,
                 )
 
@@ -228,7 +229,7 @@ class VM:
 def resolve_network(network_name, network_owner):
     network = shared.etcd_client.get(
         join_path(
-            shared.settings["etcd"]["network_prefix"],
+            settings["etcd"]["network_prefix"],
             network_owner,
             network_name,
         ),

uncloud/imagescanner/main.py

@@ -4,6 +4,7 @@ import argparse
 import subprocess as sp
 
 from os.path import join as join_path
+from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 from uncloud.imagescanner import logger
 
@@ -29,10 +30,10 @@ def qemu_img_type(path):
         return qemu_img_info["format"]
 
 
-def main(arguments):
+def main(debug=False):
     # We want to get images entries that requests images to be created
     images = shared.etcd_client.get_prefix(
-        shared.settings["etcd"]["image_prefix"], value_in_json=True
+        settings["etcd"]["image_prefix"], value_in_json=True
     )
     images_to_be_created = list(
         filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
@@ -45,13 +46,13 @@ def main(arguments):
             image_filename = image.value["filename"]
             image_store_name = image.value["store_name"]
             image_full_path = join_path(
-                shared.settings["storage"]["file_dir"],
+                settings["storage"]["file_dir"],
                 image_owner,
                 image_filename,
             )
 
             image_stores = shared.etcd_client.get_prefix(
-                shared.settings["etcd"]["image_store_prefix"],
+                settings["etcd"]["image_store_prefix"],
                 value_in_json=True,
             )
             user_image_store = next(

uncloud/metadata/main.py

@@ -5,6 +5,7 @@ from flask import Flask, request
 from flask_restful import Resource, Api
 from werkzeug.exceptions import HTTPException
 
+from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 
 app = Flask(__name__)
@@ -12,10 +13,8 @@ api = Api(app)
 
 app.logger.handlers.clear()
 
-DEFAULT_PORT=1234
-
 arg_parser = argparse.ArgumentParser('metadata', add_help=False)
-arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT))
+arg_parser.add_argument('--port', '-p', default=80, help='By default bind to port 80')
 
 
 @app.errorhandler(Exception)
@@ -73,7 +72,7 @@ class Root(Resource):
             )
         else:
             etcd_key = os.path.join(
-                shared.settings["etcd"]["user_prefix"],
+                settings["etcd"]["user_prefix"],
                 data.value["owner_realm"],
                 data.value["owner"],
                 "key",
@@ -89,7 +88,9 @@ class Root(Resource):
 api.add_resource(Root, "/")
 
 
-def main(arguments):
-    port = arguments['port']
-    debug = arguments['debug']
+def main(port=None, debug=False):
     app.run(debug=debug, host="::", port=port)
+
+
+if __name__ == "__main__":
+    main()

uncloud/oneshot/__init__.py

@@ -1,3 +0,0 @@
-import logging
-
-logger = logging.getLogger(__name__)

uncloud/oneshot/main.py

@@ -1,123 +0,0 @@
-import argparse
-import os
-
-
-from pathlib import Path
-from uncloud.vmm import VMM
-from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap
-
-from . import virtualmachine, logger
-
-###
-# Argument parser loaded by scripts/uncloud.
-arg_parser = argparse.ArgumentParser('oneshot', add_help=False)
-
-# Actions.
-arg_parser.add_argument('--list', action='store_true',
-        help='list UUID and name of running VMs')
-arg_parser.add_argument('--start', nargs=3,
-        metavar=('IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'),
-        help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix')
-arg_parser.add_argument('--stop', metavar='UUID',
-        help='stop a VM')
-arg_parser.add_argument('--get-status', metavar='UUID',
-        help='return the status of the VM')
-arg_parser.add_argument('--get-vnc', metavar='UUID',
-        help='return the path of the VNC socket of the VM')
-arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK',
-        help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix')
-
-# Arguments.
-arg_parser.add_argument('--workdir', default=Path.home(),
-        help='Working directory, defaulting to $HOME')
-arg_parser.add_argument('--mac',
-        help='MAC address of the VM to create (--start)')
-arg_parser.add_argument('--memory', type=int,
-        help='Memory (MB) to allocate (--start)')
-arg_parser.add_argument('--cores', type=int,
-        help='Number of cores to allocate (--start)')
-arg_parser.add_argument('--threads', type=int,
-        help='Number of threads to allocate (--start)')
-arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'],
-        help='Format of OS image (--start)')
-arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm',
-        help='QEMU acceleration to use (--start)')
-arg_parser.add_argument('--upstream-interface', default='eth0',
-        help='Name of upstream interface (--start)')
-
-###
-# Helpers.
-
-# XXX: check if it is possible to use the type returned by ETCD queries.
-class UncloudEntryWrapper:
-    def __init__(self, value):
-        self.value = value
-
-    def value(self):
-        return self.value
-
-def status_line(vm):
-     return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status())
-
-###
-# Entrypoint.
-
-def main(arguments):
-    # Initialize VMM.
-    workdir = arguments['workdir']
-    vmm = VMM(vmm_backend=workdir)
-
-    # Harcoded debug values.
-    net_id = 0
-
-    # Build VM configuration.
-    vm_config = {}
-    vm_options = [
-            'mac', 'memory', 'cores', 'threads', 'image', 'image_format',
-            '--upstream_interface', 'upstream_interface', 'network', 'accel'
-            ]
-    for option in vm_options:
-        if arguments.get(option):
-            vm_config[option] = arguments[option]
-
-    vm_config['net_id'] = net_id
-
-    # Execute requested VM action.
-    if arguments['reconfigure_radvd']:
-        # TODO: check that RADVD is available.
-        prefix = arguments['reconfigure_radvd']
-        network = UncloudEntryWrapper({
-                'id': net_id,
-                'ipv6': prefix
-                })
-
-        # Make use of uncloud.host.virtualmachine for network configuration.
-        update_radvd_conf([network])
-    elif arguments['start']:
-        # Extract from --start positional arguments. Quite fragile.
-        vm_config['image'] = arguments['start'][0]
-        vm_config['network'] = arguments['start'][1]
-        vm_config['upstream_interface'] = arguments['start'][2]
-
-        vm_config['tap_interface'] = "uc{}".format(len(vmm.discover()))
-        vm = virtualmachine.VM(vmm, vm_config)
-        vm.start()
-    elif arguments['stop']:
-        vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']})
-        vm = virtualmachine.VM(vmm, vm_config)
-        vm.stop()
-    elif arguments['get_status']:
-        vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']})
-        print(status_line(vm))
-    elif arguments['get_vnc']:
-        vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']})
-        print(vm.get_vnc_addr())
-    elif arguments['list']:
-        vms = vmm.discover()
-        print("Found {} VMs.".format(len(vms)))
-        for uuid in vms:
-            vm = virtualmachine.VM(vmm, {'uuid': uuid})
-            print(status_line(vm))
-    else:
-        print('Please specify an action: --start, --stop, --list,\
---get-status, --get-vnc, --reconfigure-radvd')

uncloud/oneshot/virtualmachine.py

@@ -1,81 +0,0 @@
-import uuid
-import os
-
-from uncloud.host.virtualmachine import create_vxlan_br_tap
-from uncloud.oneshot import logger
-
-class VM(object):
-    def __init__(self, vmm, config):
-        self.config = config
-        self.vmm = vmm
-
-        # Extract VM specs/metadata from configuration.
-        self.name         = config.get('name', 'no-name')
-        self.memory       = config.get('memory', 1024)
-        self.cores        = config.get('cores', 1)
-        self.threads      = config.get('threads', 1)
-        self.image_format = config.get('image_format', 'qcow2')
-        self.image        = config.get('image')
-        self.uuid         = config.get('uuid', str(uuid.uuid4()))
-        self.mac          = config.get('mac')
-        self.accel        = config.get('accel', 'kvm')
-
-        self.net_id             = config.get('net_id', 0)
-        self.upstream_interface = config.get('upstream_interface', 'eth0')
-        self.tap_interface      = config.get('tap_interface', 'uc0')
-        self.network            = config.get('network')
-
-    def get_qemu_args(self):
-        command = (
-            "-uuid {uuid} -name {name} -machine pc,accel={accel}"
-            " -drive file={image},format={image_format},if=virtio"
-            " -device virtio-rng-pci"
-            " -m {memory} -smp cores={cores},threads={threads}"
-            " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no"
-            " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}"
-        ).format(
-            uuid=self.uuid, name=self.name, accel=self.accel,
-            image=self.image, image_format=self.image_format,
-            memory=self.memory, cores=self.cores, threads=self.threads,
-            net_id=self.net_id, tap=self.tap_interface, mac=self.mac
-        )
-
-        return command.split(" ")
-
-    def start(self):
-        # Check that VM image is available.
-        if not os.path.isfile(self.image):
-            logger.error("Image {} does not exist. Aborting.".format(self.image))
-
-        # Create Bridge, VXLAN and tap interface for VM.
-        create_vxlan_br_tap(
-                self.net_id, self.upstream_interface, self.tap_interface, self.network
-                )
-
-        # Generate config for and run QEMU.
-        qemu_args = self.get_qemu_args()
-        logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args))
-        self.vmm.start(
-                uuid=self.uuid,
-                migration=False,
-                *qemu_args
-                )
-
-    def stop(self):
-        self.vmm.stop(self.uuid)
-
-    def get_status(self):
-        return self.vmm.get_status(self.uuid)
-
-    def get_vnc_addr(self):
-        return self.vmm.get_vnc(self.uuid)
-
-    def get_uuid(self):
-        return self.uuid
-
-    def get_name(self):
-        success, json = self.vmm.execute_command(uuid, 'query-name')
-        if success:
-            return json['return']['name']
-
-        return None

uncloud/scheduler/helper.py

@@ -7,6 +7,7 @@ from uncloud.common.host import HostStatus
 from uncloud.common.request import RequestEntry, RequestType
 from uncloud.common.vm import VMStatus
 from uncloud.common.shared import shared
+from uncloud.common.settings import settings
 
 
 def accumulated_specs(vms_specs):
@@ -129,7 +130,7 @@ def assign_host(vm):
         type=RequestType.StartVM,
         uuid=vm.uuid,
         hostname=vm.hostname,
-        request_prefix=shared.settings["etcd"]["request_prefix"],
+        request_prefix=settings["etcd"]["request_prefix"],
     )
     shared.request_pool.put(r)
 

uncloud/scheduler/main.py

@@ -6,6 +6,7 @@
 
 import argparse
 
+from uncloud.common.settings import settings
 from uncloud.common.request import RequestEntry, RequestType
 from uncloud.common.shared import shared
 from uncloud.scheduler import logger
@@ -15,7 +16,7 @@ from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection,
 arg_parser = argparse.ArgumentParser('scheduler', add_help=False)
 
 
-def main(arguments):
+def main(debug=False):
     # The below while True is neccessary for gracefully handling leadership transfer and temporary
     # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
     # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
@@ -23,9 +24,9 @@ def main(arguments):
     # get prefix until either success or deamon death comes.
     while True:
         for request_iterator in [
-            shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
+            shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
                                           raise_exception=False),
-            shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
+            shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
                                             raise_exception=False),
         ]:
             for request_event in request_iterator:
@@ -49,3 +50,7 @@ def main(arguments):
                         shared.vm_pool.put(vm_entry)
 
                         logger.info('No Resource Left. Emailing admin....')
+
+
+if __name__ == '__main__':
+    main()

uncloud/vmm/__init__.py

@@ -100,9 +100,9 @@ class TransferVM(Process):
 class VMM:
     # Virtual Machine Manager
     def __init__(
-            self,
-            qemu_path="/usr/bin/qemu-system-x86_64",
-            vmm_backend=os.path.expanduser("~/uncloud/vmm/"),
+        self,
+        qemu_path="/usr/bin/qemu-system-x86_64",
+        vmm_backend=os.path.expanduser("~/uncloud/vmm/"),
     ):
         self.qemu_path = qemu_path
         self.vmm_backend = vmm_backend
@@ -125,7 +125,7 @@ class VMM:
             os.makedirs(self.socket_dir, exist_ok=True)
 
     def is_running(self, uuid):
-        sock_path = os.path.join(self.socket_dir, uuid)
+        sock_path = os.path.join(self.vmm_backend, uuid)
         try:
             sock = socket.socket(socket.AF_UNIX)
             sock.connect(sock_path)
@@ -163,7 +163,7 @@ class VMM:
             qmp_arg = (
                 "-qmp",
                 "unix:{},server,nowait".format(
-                    join_path(self.socket_dir, uuid)
+                    join_path(self.vmm_backend, uuid)
                 ),
             )
             vnc_arg = (
@@ -212,7 +212,7 @@ class VMM:
     def execute_command(self, uuid, command, **kwargs):
         # execute_command -> sucess?, output
         try:
-            with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as (
+            with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as (
                 sock_handle,
                 file_handle,
             ):
@@ -255,8 +255,8 @@ class VMM:
     def discover(self):
         vms = [
             uuid
-            for uuid in os.listdir(self.socket_dir)
-            if not isdir(join_path(self.socket_dir, uuid))
+            for uuid in os.listdir(self.vmm_backend)
+            if not isdir(join_path(self.vmm_backend, uuid))
         ]
         return vms