.gitignore

@@ -16,5 +16,3 @@ uncloud/version.py
 build/
 venv/
 dist/
-
-*.iso

bin/gen-version

@@ -1,22 +1,22 @@
 #!/bin/sh
 # -*- coding: utf-8 -*-
 #
-# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org)
+# 2019 Nico Schottelius (nico-ucloud at schottelius.org)
 #
-# This file is part of uncloud.
+# This file is part of ucloud.
 #
-# uncloud is free software: you can redistribute it and/or modify
+# ucloud is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# uncloud is distributed in the hope that it will be useful,
+# ucloud is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+# along with ucloud. If not, see <http://www.gnu.org/licenses/>.
 #
 #
 
@@ -26,4 +26,4 @@ dir=${0%/*}
 
 # Ensure version is present - the bundled/shipped version contains a static version,
 # the git version contains a dynamic version
-printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py
+printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py

bin/uncloud-run-reinstall

@@ -24,6 +24,6 @@
 dir=${0%/*}
 
 ${dir}/gen-version;
-pip uninstall -y uncloud >/dev/null
-python setup.py install  >/dev/null
+pip uninstall -y uncloud
+python setup.py install
 ${dir}/uncloud "$@"

conf/uncloud.conf

@@ -1,13 +1,7 @@
 [etcd]
 url = localhost
 port = 2379
-base_prefix = /
+
 ca_cert
 cert_cert
 cert_key
-
-[client]
-name = replace_me
-realm = replace_me
-seed = replace_me
-api_server = http://localhost:5000

docs/README.md

@@ -1,12 +0,0 @@
-# uncloud docs
-
-## Requirements
-1. Python3
-2. Sphinx
-
-## Usage
-Run `make build` to build docs.
-
-Run `make clean` to remove build directory.
-
-Run `make publish` to push build dir to https://ungleich.ch/ucloud/

docs/source/hacking.rst

@@ -1,36 +0,0 @@
-Hacking
-=======
-Using uncloud in hacking (aka development) mode.
-
-
-Get the code
-------------
-.. code-block:: sh
-    :linenos:
-
-     git clone https://code.ungleich.ch/uncloud/uncloud.git
-
-
-
-Install python requirements
----------------------------
-You need to have python3 installed.
-
-.. code-block:: sh
-    :linenos:
-
-    cd uncloud!
-    python -m venv venv
-    . ./venv/bin/activate
-    ./bin/uncloud-run-reinstall
-
-
-
-Install os requirements
------------------------
-Install the following software packages: **dnsmasq**.
-
-If you already have a working IPv6 SLAAC and DNS setup,
-this step can be skipped.
-
-Note that you need at least one /64 IPv6 network to run uncloud.

docs/source/vm-images.rst

@@ -1,66 +0,0 @@
-VM images
-==================================
-
-Overview
----------
-
-ucloud tries to be least invasise towards VMs and only require
-strictly necessary changes for running in a virtualised
-environment. This includes configurations for:
-
-* Configuring the network
-* Managing access via ssh keys
-* Resizing the attached disk(s)
-
-Upstream images
----------------
-
-The 'official' uncloud images are defined in the `uncloud/images
-<https://code.ungleich.ch/uncloud/images>`_ repository.
-
-How to make you own Uncloud images
-----------------------------------
-
-.. note::
-  It is fairly easy to create your own images for uncloud, as the common
-  operations (which are detailed below) can be automatically handled by the
-  `uncloud/uncloud-init <https://code.ungleich.ch/uncloud/uncloud-init>`_ tool.
-
-Network configuration
-~~~~~~~~~~~~~~~~~~~~~
-All VMs in ucloud are required to support IPv6. The primary network
-configuration is always done using SLAAC. A VM thus needs only to be
-configured to
-
-* accept router advertisements on all network interfaces
-* use the router advertisements to configure the network interfaces
-* accept the DNS entries from the router advertisements
-
-
-Configuring SSH keys
-~~~~~~~~~~~~~~~~~~~~
-
-To be able to access the VM, ucloud support provisioning SSH keys.
-
-To accept ssh keys in your VM, request the URL
-*http://metadata/ssh_keys*. Add the content to the appropriate user's
-**authorized_keys** file. Below you find sample code to accomplish
-this task:
-
-.. code-block:: sh
-
-    tmp=$(mktemp)
-    curl -s http://metadata/ssk_keys > "$tmp"
-    touch ~/.ssh/authorized_keys # ensure it exists
-    cat ~/.ssh/authorized_keys >> "$tmp"
-    sort "$tmp" | uniq > ~/.ssh/authorized_keys
-
-
-Disk resize
-~~~~~~~~~~~
-In virtualised environments, the disk sizes might grow. The operating
-system should detect disks that are bigger than the existing partition
-table and resize accordingly. This task is os specific.
-
-ucloud does not support shrinking disks due to the complexity and
-intra OS dependencies.

scripts/uncloud

@@ -1,88 +1,70 @@
 #!/usr/bin/env python3
-import logging
-import sys
-import importlib
 import argparse
-import os
+import logging
+import importlib
+import multiprocessing as mp
+import sys
 
-from etcd3.exceptions import ConnectionFailedError
+from logging.handlers import SysLogHandler
+from uncloud.configure.main import configure_parser
 
-from uncloud.common import settings
 from uncloud import UncloudException
-from uncloud.common.cli import resolve_otp_credentials
 
-# Components that use etcd
-ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner',
-                   'imagescanner', 'metadata', 'configure', 'hack']
+def exception_hook(exc_type, exc_value, exc_traceback):
+    logging.getLogger(__name__).error(
+        'Uncaught exception',
+        exc_info=(exc_type, exc_value, exc_traceback)
+    )
 
-ALL_COMPONENTS = ETCD_COMPONENTS.copy()
-ALL_COMPONENTS.append('oneshot')
-#ALL_COMPONENTS.append('cli')
+
+sys.excepthook = exception_hook
 
 
 if __name__ == '__main__':
-    arg_parser = argparse.ArgumentParser()
-    subparsers = arg_parser.add_subparsers(dest='command')
+    # Setting up root logger
+    logger = logging.getLogger()
+
+    logger.setLevel(logging.DEBUG)
 
     parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
-                               help='More verbose logging')
-    parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
-                               default=os.path.expanduser('~/uncloud'))
+    parent_parser.add_argument("--debug", "-d", action='store_true')
 
-    etcd_parser = argparse.ArgumentParser(add_help=False)
-    etcd_parser.add_argument('--etcd-host')
-    etcd_parser.add_argument('--etcd-port')
-    etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
-    etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
-    etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
+    arg_parser = argparse.ArgumentParser()
 
-    for component in ALL_COMPONENTS:
-        mod = importlib.import_module('uncloud.{}.main'.format(component))
-        parser = getattr(mod, 'arg_parser')
+    subparsers = arg_parser.add_subparsers(dest="command")
 
-        if component in ETCD_COMPONENTS:
-            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser])
-        else:
-            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
+    api_parser = subparsers.add_parser("api", parents=[parent_parser])
+    api_parser.add_argument("--port", "-p")
 
-    arguments = vars(arg_parser.parse_args())
-    etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
-    etcd_arguments = {
-        'etcd': {
-            key.replace('etcd_', ''): arguments[key]
-            for key in etcd_arguments
-        }
-    }
-    if not arguments['command']:
+    host_parser = subparsers.add_parser("host")
+    host_parser.add_argument("--hostname", required=True)
+
+    scheduler_parser = subparsers.add_parser("scheduler")
+    filescanner_parser = subparsers.add_parser("filescanner")
+    imagescanner_parser = subparsers.add_parser("imagescanner")
+    metadata_parser = subparsers.add_parser("metadata")
+    config_parser = subparsers.add_parser("configure")
+
+    configure_parser(config_parser)
+    args = arg_parser.parse_args()
+
+    if not args.command:
         arg_parser.print_help()
     else:
-        # Initializing Settings and resolving otp_credentials
-        # It is neccessary to resolve_otp_credentials after argument parsing is done because
-        # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
-        # providing the default values for --name, --realm and --seed arguments from the values
-        # we read from file. But, now we are asking user about where the config file lives. So,
-        # to providing default value is not possible before parsing arguments. So, we are doing
-        # it after..
-#        settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
-#        resolve_otp_credentials(arguments)
 
-        name = arguments.pop('command')
-        mod = importlib.import_module('uncloud.{}.main'.format(name))
-        main = getattr(mod, 'main')
-
-        if arguments['debug']:
-            logging.basicConfig(level=logging.DEBUG)
-        else:
-            logging.basicConfig(level=logging.INFO)
-
-        log = logging.getLogger()
+        # if we start etcd in seperate process with default settings
+        # i.e inheriting few things from parent process etcd3 module
+        # errors out, so the following command configure multiprocessing
+        # module to not inherit anything from parent.
+        mp.set_start_method('spawn')
 
+        arguments = vars(args)
         try:
-            main(arguments)
+            name = arguments.pop('command')
+            mod = importlib.import_module("uncloud.{}.main".format(name))
+            main = getattr(mod, "main")
+            main(**arguments)
         except UncloudException as err:
-            log.error(err)
-#        except ConnectionFailedError as err:
-#            log.error('Cannot connect to etcd: {}'.format(err))
+            logger.error(err)
         except Exception as err:
-            log.exception(err)
+            logger.exception(err)

setup.py

@@ -40,7 +40,7 @@ setup(
         "pynetbox",
         "colorama",
         "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3",
-        "marshmallow"
+        "marshmallow",
     ],
     scripts=["scripts/uncloud"],
     data_files=[

uncloud/api/common_fields.py

@@ -1,6 +1,7 @@
 import os
 
-from uncloud.common.shared import shared
+from uncloud.shared import shared
+from uncloud.settings import settings
 
 
 class Optional:
@@ -53,7 +54,9 @@ class VmUUIDField(Field):
 
     def vm_uuid_validation(self):
         r = shared.etcd_client.get(
-            os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
+            os.path.join(settings["etcd"]["vm_prefix"], self.uuid)
         )
         if not r:
-            self.add_error("VM with uuid {} does not exists".format(self.uuid))
+            self.add_error(
+                "VM with uuid {} does not exists".format(self.uuid)
+            )

uncloud/api/create_image_store.py

@@ -3,17 +3,18 @@ import os
 
 from uuid import uuid4
 
-from uncloud.common.shared import shared
+from uncloud.shared import shared
+from uncloud.settings import settings
 
 data = {
-    'is_public': True,
-    'type': 'ceph',
-    'name': 'images',
-    'description': 'first ever public image-store',
-    'attributes': {'list': [], 'key': [], 'pool': 'images'},
+    "is_public": True,
+    "type": "ceph",
+    "name": "images",
+    "description": "first ever public image-store",
+    "attributes": {"list": [], "key": [], "pool": "images"},
 }
 
 shared.etcd_client.put(
-    os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
+    os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex),
     json.dumps(data),
 )

uncloud/api/helper.py

@@ -1,12 +1,14 @@
 import binascii
 import ipaddress
 import random
+import subprocess as sp
 import logging
 import requests
 
 from pyotp import TOTP
 
-from uncloud.common.shared import shared
+from uncloud.shared import shared
+from uncloud.settings import settings
 
 logger = logging.getLogger(__name__)
 
@@ -14,9 +16,9 @@ logger = logging.getLogger(__name__)
 def check_otp(name, realm, token):
     try:
         data = {
-            "auth_name": shared.settings["otp"]["auth_name"],
-            "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
-            "auth_realm": shared.settings["otp"]["auth_realm"],
+            "auth_name": settings["otp"]["auth_name"],
+            "auth_token": TOTP(settings["otp"]["auth_seed"]).now(),
+            "auth_realm": settings["otp"]["auth_realm"],
             "name": name,
             "realm": realm,
             "token": token,
@@ -24,13 +26,13 @@ def check_otp(name, realm, token):
     except binascii.Error as err:
         logger.error(
             "Cannot compute OTP for seed: {}".format(
-                shared.settings["otp"]["auth_seed"]
+                settings["otp"]["auth_seed"]
             )
         )
         return 400
 
     response = requests.post(
-        shared.settings["otp"]["verification_controller_url"], json=data
+        settings["otp"]["verification_controller_url"], json=data
     )
     return response.status_code
 
@@ -86,7 +88,7 @@ def resolve_image_name(name, etcd_client):
         )
 
     images = etcd_client.get_prefix(
-        shared.settings["etcd"]["image_prefix"], value_in_json=True
+        settings["etcd"]["image_prefix"], value_in_json=True
     )
 
     # Try to find image with name == image_name and store_name == store_name
@@ -110,7 +112,9 @@ def random_bytes(num=6):
     return [random.randrange(256) for _ in range(num)]
 
 
-def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
+def generate_mac(
+    uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
+):
     mac = random_bytes()
     if oui:
         if type(oui) == str:
@@ -145,4 +149,3 @@ def mac2ipv6(mac, prefix):
     lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
     prefix = ipaddress.IPv6Address(prefix)
     return str(prefix + int(lower_part))
-

uncloud/api/main.py

@@ -1,7 +1,6 @@
 import json
 import pynetbox
 import logging
-import argparse
 
 from uuid import uuid4
 from os.path import join as join_path
@@ -10,13 +9,14 @@ from flask import Flask, request
 from flask_restful import Resource, Api
 from werkzeug.exceptions import HTTPException
 
-from uncloud.common.shared import shared
-
 from uncloud.common import counters
 from uncloud.common.vm import VMStatus
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.api import schemas
-from uncloud.api.helper import generate_mac, mac2ipv6
+from uncloud.settings import settings
+from uncloud.shared import shared
+
+from . import schemas
+from .helper import generate_mac, mac2ipv6
 from uncloud import UncloudException
 
 logger = logging.getLogger(__name__)
@@ -25,9 +25,6 @@ app = Flask(__name__)
 api = Api(app)
 app.logger.handlers.clear()
 
-arg_parser = argparse.ArgumentParser('api', add_help=False)
-arg_parser.add_argument('--port', '-p')
-
 
 @app.errorhandler(Exception)
 def handle_exception(e):
@@ -37,7 +34,7 @@ def handle_exception(e):
         return e
 
     # now you're handling non-HTTP exceptions only
-    return {'message': 'Server Error'}, 500
+    return {"message": "Server Error"}, 500
 
 
 class CreateVM(Resource):
@@ -49,33 +46,33 @@ class CreateVM(Resource):
         validator = schemas.CreateVMSchema(data)
         if validator.is_valid():
             vm_uuid = uuid4().hex
-            vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
+            vm_key = join_path(settings["etcd"]["vm_prefix"], vm_uuid)
             specs = {
-                'cpu': validator.specs['cpu'],
-                'ram': validator.specs['ram'],
-                'os-ssd': validator.specs['os-ssd'],
-                'hdd': validator.specs['hdd'],
+                "cpu": validator.specs["cpu"],
+                "ram": validator.specs["ram"],
+                "os-ssd": validator.specs["os-ssd"],
+                "hdd": validator.specs["hdd"],
             }
-            macs = [generate_mac() for _ in range(len(data['network']))]
+            macs = [generate_mac() for _ in range(len(data["network"]))]
             tap_ids = [
                 counters.increment_etcd_counter(
-                    shared.etcd_client, shared.settings['etcd']['tap_counter']
+                    shared.etcd_client, "/v1/counter/tap"
                 )
-                for _ in range(len(data['network']))
+                for _ in range(len(data["network"]))
             ]
             vm_entry = {
-                'name': data['vm_name'],
-                'owner': data['name'],
-                'owner_realm': data['realm'],
-                'specs': specs,
-                'hostname': '',
-                'status': VMStatus.stopped,
-                'image_uuid': validator.image_uuid,
-                'log': [],
-                'vnc_socket': '',
-                'network': list(zip(data['network'], macs, tap_ids)),
-                'metadata': {'ssh-keys': []},
-                'in_migration': False,
+                "name": data["vm_name"],
+                "owner": data["name"],
+                "owner_realm": data["realm"],
+                "specs": specs,
+                "hostname": "",
+                "status": VMStatus.stopped,
+                "image_uuid": validator.image_uuid,
+                "log": [],
+                "vnc_socket": "",
+                "network": list(zip(data["network"], macs, tap_ids)),
+                "metadata": {"ssh-keys": []},
+                "in_migration": False,
             }
             shared.etcd_client.put(vm_key, vm_entry, value_in_json=True)
 
@@ -83,39 +80,39 @@ class CreateVM(Resource):
             r = RequestEntry.from_scratch(
                 type=RequestType.ScheduleVM,
                 uuid=vm_uuid,
-                request_prefix=shared.settings['etcd']['request_prefix'],
+                request_prefix=settings["etcd"]["request_prefix"],
             )
             shared.request_pool.put(r)
 
-            return {'message': 'VM Creation Queued'}, 200
+            return {"message": "VM Creation Queued"}, 200
         return validator.get_errors(), 400
 
 
 class VmStatus(Resource):
     @staticmethod
-    def post():
+    def get():
         data = request.json
         validator = schemas.VMStatusSchema(data)
         if validator.is_valid():
             vm = shared.vm_pool.get(
-                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
+                join_path(settings["etcd"]["vm_prefix"], data["uuid"])
             )
             vm_value = vm.value.copy()
-            vm_value['ip'] = []
+            vm_value["ip"] = []
             for network_mac_and_tap in vm.network:
                 network_name, mac, tap = network_mac_and_tap
                 network = shared.etcd_client.get(
                     join_path(
-                        shared.settings['etcd']['network_prefix'],
-                        data['name'],
+                        settings["etcd"]["network_prefix"],
+                        data["name"],
                         network_name,
                     ),
                     value_in_json=True,
                 )
                 ipv6_addr = (
-                    network.value.get('ipv6').split('::')[0] + '::'
+                    network.value.get("ipv6").split("::")[0] + "::"
                 )
-                vm_value['ip'].append(mac2ipv6(mac, ipv6_addr))
+                vm_value["ip"].append(mac2ipv6(mac, ipv6_addr))
             vm.value = vm_value
             return vm.value
         else:
@@ -129,26 +126,26 @@ class CreateImage(Resource):
         validator = schemas.CreateImageSchema(data)
         if validator.is_valid():
             file_entry = shared.etcd_client.get(
-                join_path(shared.settings['etcd']['file_prefix'], data['uuid'])
+                join_path(settings["etcd"]["file_prefix"], data["uuid"])
             )
             file_entry_value = json.loads(file_entry.value)
 
             image_entry_json = {
-                'status': 'TO_BE_CREATED',
-                'owner': file_entry_value['owner'],
-                'filename': file_entry_value['filename'],
-                'name': data['name'],
-                'store_name': data['image_store'],
-                'visibility': 'public',
+                "status": "TO_BE_CREATED",
+                "owner": file_entry_value["owner"],
+                "filename": file_entry_value["filename"],
+                "name": data["name"],
+                "store_name": data["image_store"],
+                "visibility": "public",
             }
             shared.etcd_client.put(
                 join_path(
-                    shared.settings['etcd']['image_prefix'], data['uuid']
+                    settings["etcd"]["image_prefix"], data["uuid"]
                 ),
                 json.dumps(image_entry_json),
             )
 
-            return {'message': 'Image queued for creation.'}
+            return {"message": "Image queued for creation."}
         return validator.get_errors(), 400
 
 
@@ -156,15 +153,15 @@ class ListPublicImages(Resource):
     @staticmethod
     def get():
         images = shared.etcd_client.get_prefix(
-            shared.settings['etcd']['image_prefix'], value_in_json=True
+            settings["etcd"]["image_prefix"], value_in_json=True
         )
-        r = {'images': []}
+        r = {"images": []}
         for image in images:
-            image_key = '{}:{}'.format(
-                image.value['store_name'], image.value['name']
+            image_key = "{}:{}".format(
+                image.value["store_name"], image.value["name"]
             )
-            r['images'].append(
-                {'name': image_key, 'status': image.value['status']}
+            r["images"].append(
+                {"name": image_key, "status": image.value["status"]}
             )
         return r, 200
 
@@ -177,14 +174,14 @@ class VMAction(Resource):
 
         if validator.is_valid():
             vm_entry = shared.vm_pool.get(
-                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
+                join_path(settings["etcd"]["vm_prefix"], data["uuid"])
             )
-            action = data['action']
+            action = data["action"]
 
-            if action == 'start':
-                action = 'schedule'
+            if action == "start":
+                action = "schedule"
 
-            if action == 'delete' and vm_entry.hostname == '':
+            if action == "delete" and vm_entry.hostname == "":
                 if shared.storage_handler.is_vm_image_exists(
                     vm_entry.uuid
                 ):
@@ -193,25 +190,25 @@ class VMAction(Resource):
                     )
                     if r_status:
                         shared.etcd_client.client.delete(vm_entry.key)
-                        return {'message': 'VM successfully deleted'}
+                        return {"message": "VM successfully deleted"}
                     else:
                         logger.error(
-                            'Some Error Occurred while deleting VM'
+                            "Some Error Occurred while deleting VM"
                         )
-                        return {'message': 'VM deletion unsuccessfull'}
+                        return {"message": "VM deletion unsuccessfull"}
                 else:
                     shared.etcd_client.client.delete(vm_entry.key)
-                    return {'message': 'VM successfully deleted'}
+                    return {"message": "VM successfully deleted"}
 
             r = RequestEntry.from_scratch(
-                type='{}VM'.format(action.title()),
-                uuid=data['uuid'],
+                type="{}VM".format(action.title()),
+                uuid=data["uuid"],
                 hostname=vm_entry.hostname,
-                request_prefix=shared.settings['etcd']['request_prefix'],
+                request_prefix=settings["etcd"]["request_prefix"],
             )
             shared.request_pool.put(r)
             return (
-                {'message': 'VM {} Queued'.format(action.title())},
+                {"message": "VM {} Queued".format(action.title())},
                 200,
             )
         else:
@@ -225,20 +222,20 @@ class VMMigration(Resource):
         validator = schemas.VmMigrationSchema(data)
 
         if validator.is_valid():
-            vm = shared.vm_pool.get(data['uuid'])
+            vm = shared.vm_pool.get(data["uuid"])
             r = RequestEntry.from_scratch(
                 type=RequestType.InitVMMigration,
                 uuid=vm.uuid,
                 hostname=join_path(
-                    shared.settings['etcd']['host_prefix'],
+                    settings["etcd"]["host_prefix"],
                     validator.destination.value,
                 ),
-                request_prefix=shared.settings['etcd']['request_prefix'],
+                request_prefix=settings["etcd"]["request_prefix"],
             )
 
             shared.request_pool.put(r)
             return (
-                {'message': 'VM Migration Initialization Queued'},
+                {"message": "VM Migration Initialization Queued"},
                 200,
             )
         else:
@@ -247,32 +244,32 @@ class VMMigration(Resource):
 
 class ListUserVM(Resource):
     @staticmethod
-    def post():
+    def get():
         data = request.json
         validator = schemas.OTPSchema(data)
 
         if validator.is_valid():
             vms = shared.etcd_client.get_prefix(
-                shared.settings['etcd']['vm_prefix'], value_in_json=True
+                settings["etcd"]["vm_prefix"], value_in_json=True
             )
             return_vms = []
             user_vms = filter(
-                lambda v: v.value['owner'] == data['name'], vms
+                lambda v: v.value["owner"] == data["name"], vms
             )
             for vm in user_vms:
                 return_vms.append(
                     {
-                        'name': vm.value['name'],
-                        'vm_uuid': vm.key.split('/')[-1],
-                        'specs': vm.value['specs'],
-                        'status': vm.value['status'],
-                        'hostname': vm.value['hostname'],
-                        'vnc_socket': vm.value.get('vnc_socket', None),
+                        "name": vm.value["name"],
+                        "vm_uuid": vm.key.split("/")[-1],
+                        "specs": vm.value["specs"],
+                        "status": vm.value["status"],
+                        "hostname": vm.value["hostname"],
+                        "vnc_socket": vm.value.get("vnc_socket", None),
                     }
                 )
             if return_vms:
-                return {'message': return_vms}, 200
-            return {'message': 'No VM found'}, 404
+                return {"message": return_vms}, 200
+            return {"message": "No VM found"}, 404
 
         else:
             return validator.get_errors(), 400
@@ -280,26 +277,28 @@ class ListUserVM(Resource):
 
 class ListUserFiles(Resource):
     @staticmethod
-    def post():
+    def get():
         data = request.json
         validator = schemas.OTPSchema(data)
 
         if validator.is_valid():
             files = shared.etcd_client.get_prefix(
-                shared.settings['etcd']['file_prefix'], value_in_json=True
+                settings["etcd"]["file_prefix"], value_in_json=True
             )
             return_files = []
-            user_files = [f for f in files if f.value['owner'] == data['name']]
+            user_files = list(
+                filter(
+                    lambda f: f.value["owner"] == data["name"], files
+                )
+            )
             for file in user_files:
-                file_uuid = file.key.split('/')[-1]
-                file = file.value
-                file['uuid'] = file_uuid
-
-                file.pop('sha512sum', None)
-                file.pop('owner', None)
-
-                return_files.append(file)
-            return {'message': return_files}, 200
+                return_files.append(
+                    {
+                        "filename": file.value["filename"],
+                        "uuid": file.key.split("/")[-1],
+                    }
+                )
+            return {"message": return_files}, 200
         else:
             return validator.get_errors(), 400
 
@@ -311,19 +310,19 @@ class CreateHost(Resource):
         validator = schemas.CreateHostSchema(data)
         if validator.is_valid():
             host_key = join_path(
-                shared.settings['etcd']['host_prefix'], uuid4().hex
+                settings["etcd"]["host_prefix"], uuid4().hex
             )
             host_entry = {
-                'specs': data['specs'],
-                'hostname': data['hostname'],
-                'status': 'DEAD',
-                'last_heartbeat': '',
+                "specs": data["specs"],
+                "hostname": data["hostname"],
+                "status": "DEAD",
+                "last_heartbeat": "",
             }
             shared.etcd_client.put(
                 host_key, host_entry, value_in_json=True
             )
 
-            return {'message': 'Host Created'}, 200
+            return {"message": "Host Created"}, 200
 
         return validator.get_errors(), 400
 
@@ -334,9 +333,9 @@ class ListHost(Resource):
         hosts = shared.host_pool.hosts
         r = {
             host.key: {
-                'status': host.status,
-                'specs': host.specs,
-                'hostname': host.hostname,
+                "status": host.status,
+                "specs": host.specs,
+                "hostname": host.hostname,
             }
             for host in hosts
         }
@@ -345,7 +344,7 @@ class ListHost(Resource):
 
 class GetSSHKeys(Resource):
     @staticmethod
-    def post():
+    def get():
         data = request.json
         validator = schemas.GetSSHSchema(data)
         if validator.is_valid():
@@ -353,29 +352,29 @@ class GetSSHKeys(Resource):
 
                 # {user_prefix}/{realm}/{name}/key/
                 etcd_key = join_path(
-                    shared.settings['etcd']['user_prefix'],
-                    data['realm'],
-                    data['name'],
-                    'key',
+                    settings["etcd"]["user_prefix"],
+                    data["realm"],
+                    data["name"],
+                    "key",
                 )
                 etcd_entry = shared.etcd_client.get_prefix(
                     etcd_key, value_in_json=True
                 )
 
                 keys = {
-                    key.key.split('/')[-1]: key.value
+                    key.key.split("/")[-1]: key.value
                     for key in etcd_entry
                 }
-                return {'keys': keys}
+                return {"keys": keys}
             else:
 
                 # {user_prefix}/{realm}/{name}/key/{key_name}
                 etcd_key = join_path(
-                    shared.settings['etcd']['user_prefix'],
-                    data['realm'],
-                    data['name'],
-                    'key',
-                    data['key_name'],
+                    settings["etcd"]["user_prefix"],
+                    data["realm"],
+                    data["name"],
+                    "key",
+                    data["key_name"],
                 )
                 etcd_entry = shared.etcd_client.get(
                     etcd_key, value_in_json=True
@@ -383,14 +382,14 @@ class GetSSHKeys(Resource):
 
                 if etcd_entry:
                     return {
-                        'keys': {
-                            etcd_entry.key.split('/')[
+                        "keys": {
+                            etcd_entry.key.split("/")[
                                 -1
                             ]: etcd_entry.value
                         }
                     }
                 else:
-                    return {'keys': {}}
+                    return {"keys": {}}
         else:
             return validator.get_errors(), 400
 
@@ -404,56 +403,56 @@ class AddSSHKey(Resource):
 
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                shared.settings['etcd']['user_prefix'],
-                data['realm'],
-                data['name'],
-                'key',
-                data['key_name'],
+                settings["etcd"]["user_prefix"],
+                data["realm"],
+                data["name"],
+                "key",
+                data["key_name"],
             )
             etcd_entry = shared.etcd_client.get(
                 etcd_key, value_in_json=True
             )
             if etcd_entry:
                 return {
-                    'message': 'Key with name "{}" already exists'.format(
-                        data['key_name']
+                    "message": "Key with name '{}' already exists".format(
+                        data["key_name"]
                     )
                 }
             else:
                 # Key Not Found. It implies user' haven't added any key yet.
                 shared.etcd_client.put(
-                    etcd_key, data['key'], value_in_json=True
+                    etcd_key, data["key"], value_in_json=True
                 )
-                return {'message': 'Key added successfully'}
+                return {"message": "Key added successfully"}
         else:
             return validator.get_errors(), 400
 
 
 class RemoveSSHKey(Resource):
     @staticmethod
-    def post():
+    def get():
         data = request.json
         validator = schemas.RemoveSSHSchema(data)
         if validator.is_valid():
 
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                shared.settings['etcd']['user_prefix'],
-                data['realm'],
-                data['name'],
-                'key',
-                data['key_name'],
+                settings["etcd"]["user_prefix"],
+                data["realm"],
+                data["name"],
+                "key",
+                data["key_name"],
             )
             etcd_entry = shared.etcd_client.get(
                 etcd_key, value_in_json=True
             )
             if etcd_entry:
                 shared.etcd_client.client.delete(etcd_key)
-                return {'message': 'Key successfully removed.'}
+                return {"message": "Key successfully removed."}
             else:
                 return {
-                    'message': 'No Key with name "{}" Exists at all.'.format(
-                        data['key_name']
+                    "message": "No Key with name '{}' Exists at all.".format(
+                        data["key_name"]
                     )
                 }
         else:
@@ -469,107 +468,104 @@ class CreateNetwork(Resource):
         if validator.is_valid():
 
             network_entry = {
-                'id': counters.increment_etcd_counter(
-                    shared.etcd_client, shared.settings['etcd']['vxlan_counter']
+                "id": counters.increment_etcd_counter(
+                    shared.etcd_client, "/v1/counter/vxlan"
                 ),
-                'type': data['type'],
+                "type": data["type"],
             }
             if validator.user.value:
                 try:
                     nb = pynetbox.api(
-                        url=shared.settings['netbox']['url'],
-                        token=shared.settings['netbox']['token'],
+                        url=settings["netbox"]["url"],
+                        token=settings["netbox"]["token"],
                     )
                     nb_prefix = nb.ipam.prefixes.get(
-                        prefix=shared.settings['network']['prefix']
+                        prefix=settings["network"]["prefix"]
                     )
                     prefix = nb_prefix.available_prefixes.create(
                         data={
-                            'prefix_length': int(
-                                shared.settings['network']['prefix_length']
+                            "prefix_length": int(
+                                settings["network"]["prefix_length"]
                             ),
-                            'description': '{}\'s network "{}"'.format(
-                                data['name'], data['network_name']
+                            "description": '{}\'s network "{}"'.format(
+                                data["name"], data["network_name"]
                             ),
-                            'is_pool': True,
+                            "is_pool": True,
                         }
                     )
                 except Exception as err:
                     app.logger.error(err)
                     return {
-                        'message': 'Error occured while creating network.'
+                        "message": "Error occured while creating network."
                     }
                 else:
-                    network_entry['ipv6'] = prefix['prefix']
+                    network_entry["ipv6"] = prefix["prefix"]
             else:
-                network_entry['ipv6'] = 'fd00::/64'
+                network_entry["ipv6"] = "fd00::/64"
 
             network_key = join_path(
-                shared.settings['etcd']['network_prefix'],
-                data['name'],
-                data['network_name'],
+                settings["etcd"]["network_prefix"],
+                data["name"],
+                data["network_name"],
             )
             shared.etcd_client.put(
                 network_key, network_entry, value_in_json=True
             )
-            return {'message': 'Network successfully added.'}
+            return {"message": "Network successfully added."}
         else:
             return validator.get_errors(), 400
 
 
 class ListUserNetwork(Resource):
     @staticmethod
-    def post():
+    def get():
         data = request.json
         validator = schemas.OTPSchema(data)
 
         if validator.is_valid():
             prefix = join_path(
-                shared.settings['etcd']['network_prefix'], data['name']
+                settings["etcd"]["network_prefix"], data["name"]
             )
             networks = shared.etcd_client.get_prefix(
                 prefix, value_in_json=True
             )
             user_networks = []
             for net in networks:
-                net.value['name'] = net.key.split('/')[-1]
+                net.value["name"] = net.key.split("/")[-1]
                 user_networks.append(net.value)
-            return {'networks': user_networks}, 200
+            return {"networks": user_networks}, 200
         else:
             return validator.get_errors(), 400
 
 
-api.add_resource(CreateVM, '/vm/create')
-api.add_resource(VmStatus, '/vm/status')
+api.add_resource(CreateVM, "/vm/create")
+api.add_resource(VmStatus, "/vm/status")
 
-api.add_resource(VMAction, '/vm/action')
-api.add_resource(VMMigration, '/vm/migrate')
+api.add_resource(VMAction, "/vm/action")
+api.add_resource(VMMigration, "/vm/migrate")
 
-api.add_resource(CreateImage, '/image/create')
-api.add_resource(ListPublicImages, '/image/list-public')
+api.add_resource(CreateImage, "/image/create")
+api.add_resource(ListPublicImages, "/image/list-public")
 
-api.add_resource(ListUserVM, '/user/vms')
-api.add_resource(ListUserFiles, '/user/files')
-api.add_resource(ListUserNetwork, '/user/networks')
+api.add_resource(ListUserVM, "/user/vms")
+api.add_resource(ListUserFiles, "/user/files")
+api.add_resource(ListUserNetwork, "/user/networks")
 
-api.add_resource(AddSSHKey, '/user/add-ssh')
-api.add_resource(RemoveSSHKey, '/user/remove-ssh')
-api.add_resource(GetSSHKeys, '/user/get-ssh')
+api.add_resource(AddSSHKey, "/user/add-ssh")
+api.add_resource(RemoveSSHKey, "/user/remove-ssh")
+api.add_resource(GetSSHKeys, "/user/get-ssh")
 
-api.add_resource(CreateHost, '/host/create')
-api.add_resource(ListHost, '/host/list')
+api.add_resource(CreateHost, "/host/create")
+api.add_resource(ListHost, "/host/list")
 
-api.add_resource(CreateNetwork, '/network/create')
+api.add_resource(CreateNetwork, "/network/create")
 
 
-def main(arguments):
-    debug = arguments['debug']
-    port = arguments['port']
-
+def main(debug=False, port=None):
     try:
         image_stores = list(
             shared.etcd_client.get_prefix(
-                shared.settings['etcd']['image_store_prefix'], value_in_json=True
+                settings["etcd"]["image_store_prefix"], value_in_json=True
             )
         )
     except KeyError:
@@ -580,21 +576,27 @@ def main(arguments):
     #
     # if not image_stores:
     #     data = {
-    #         'is_public': True,
-    #         'type': 'ceph',
-    #         'name': 'images',
-    #         'description': 'first ever public image-store',
-    #         'attributes': {'list': [], 'key': [], 'pool': 'images'},
+    #         "is_public": True,
+    #         "type": "ceph",
+    #         "name": "images",
+    #         "description": "first ever public image-store",
+    #         "attributes": {"list": [], "key": [], "pool": "images"},
     #     }
 
     #     shared.etcd_client.put(
     #         join_path(
-    #             shared.settings['etcd']['image_store_prefix'], uuid4().hex
+    #             settings["etcd"]["image_store_prefix"], uuid4().hex
     #         ),
     #         json.dumps(data),
     #     )
 
     try:
-        app.run(host='::', port=port, debug=debug)
+        app.run(host="::",
+                port=port,
+                debug=debug)
     except OSError as e:
-        raise UncloudException('Failed to start Flask: {}'.format(e))
+        raise UncloudException("Failed to start Flask: {}".format(e))
+
+
+if __name__ == "__main__":
+    main()

uncloud/api/schemas.py

@@ -21,7 +21,8 @@ import bitmath
 
 from uncloud.common.host import HostStatus
 from uncloud.common.vm import VMStatus
-from uncloud.common.shared import shared
+from uncloud.shared import shared
+from uncloud.settings import settings
 from . import helper, logger
 from .common_fields import Field, VmUUIDField
 from .helper import check_otp, resolve_vm_name
@@ -111,7 +112,7 @@ class CreateImageSchema(BaseSchema):
     def file_uuid_validation(self):
         file_entry = shared.etcd_client.get(
             os.path.join(
-                shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
+                settings["etcd"]["file_prefix"], self.uuid.value
             )
         )
         if file_entry is None:
@@ -124,7 +125,7 @@ class CreateImageSchema(BaseSchema):
     def image_store_name_validation(self):
         image_stores = list(
             shared.etcd_client.get_prefix(
-                shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
+                settings["etcd"]["image_store_prefix"]
             )
         )
 
@@ -282,7 +283,7 @@ class CreateVMSchema(OTPSchema):
             for net in _network:
                 network = shared.etcd_client.get(
                     os.path.join(
-                        shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
+                        settings["etcd"]["network_prefix"],
                         self.name.value,
                         net,
                     ),
@@ -321,7 +322,7 @@ class CreateVMSchema(OTPSchema):
                     "Your specified OS-SSD is not in correct units"
                 )
 
-            if int(_cpu) < 1:
+            if _cpu < 1:
                 self.add_error("CPU must be atleast 1")
 
             if parsed_ram < bitmath.GB(1):
@@ -487,7 +488,7 @@ class VmMigrationSchema(OTPSchema):
             self.add_error("Can't migrate non-running VM")
 
         if vm.hostname == os.path.join(
-            shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
+            settings["etcd"]["host_prefix"], self.destination.value
         ):
             self.add_error(
                 "Destination host couldn't be same as Source Host"
@@ -527,7 +528,9 @@ class GetSSHSchema(OTPSchema):
 
 class CreateNetwork(OTPSchema):
     def __init__(self, data):
-        self.network_name = Field("network_name", str, data.get("network_name", KeyError))
+        self.network_name = Field(
+            "network_name", str, data.get("network_name", KeyError)
+        )
         self.type = Field("type", str, data.get("type", KeyError))
         self.user = Field("user", bool, bool(data.get("user", False)))
 
@@ -538,8 +541,14 @@ class CreateNetwork(OTPSchema):
         super().__init__(data, fields=fields)
 
     def network_name_validation(self):
-        key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
-        network = shared.etcd_client.get(key, value_in_json=True)
+        network = shared.etcd_client.get(
+            os.path.join(
+                settings["etcd"]["network_prefix"],
+                self.name.value,
+                self.network_name.value,
+            ),
+            value_in_json=True,
+        )
         if network:
             self.add_error(
                 "Network with name {} already exists".format(

uncloud/cli/helper.py

@@ -1,46 +0,0 @@
-import requests
-import json
-import argparse
-import binascii
-
-from pyotp import TOTP
-from os.path import join as join_path
-from uncloud.common.shared import shared
-
-
-def get_otp_parser():
-    otp_parser = argparse.ArgumentParser('otp')
-    otp_parser.add_argument('--name')
-    otp_parser.add_argument('--realm')
-    otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
-
-    return otp_parser
-
-
-def load_dump_pretty(content):
-    if isinstance(content, bytes):
-        content = content.decode('utf-8')
-    parsed = json.loads(content)
-    return json.dumps(parsed, indent=4, sort_keys=True)
-
-
-def make_request(*args, data=None, request_method=requests.post):
-    try:
-        r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
-    except requests.exceptions.RequestException:
-        print('Error occurred while connecting to API server.')
-    else:
-        try:
-            print(load_dump_pretty(r.content))
-        except Exception:
-            print('Error occurred while getting output from api server.')
-
-
-def get_token(seed):
-    if seed is not None:
-        try:
-            token = TOTP(seed).now()
-        except binascii.Error:
-            raise argparse.ArgumentTypeError('Invalid seed')
-        else:
-            return token

uncloud/cli/host.py

@@ -1,45 +0,0 @@
-import requests
-
-from uncloud.cli.helper import make_request, get_otp_parser
-from uncloud.common.parser import BaseParser
-
-
-class HostParser(BaseParser):
-    def __init__(self):
-        super().__init__('host')
-
-    def create(self, **kwargs):
-        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs)
-        p.add_argument('--hostname', required=True)
-        p.add_argument('--cpu', required=True, type=int)
-        p.add_argument('--ram', required=True)
-        p.add_argument('--os-ssd', required=True)
-        p.add_argument('--hdd', default=list())
-
-    def list(self, **kwargs):
-        self.subparser.add_parser('list', **kwargs)
-
-
-parser = HostParser()
-arg_parser = parser.arg_parser
-
-
-def main(**kwargs):
-    subcommand = kwargs.pop('host_subcommand')
-    if not subcommand:
-        arg_parser.print_help()
-    else:
-        request_method = requests.post
-        data = None
-        if subcommand == 'create':
-            kwargs['specs'] = {
-                'cpu': kwargs.pop('cpu'),
-                'ram': kwargs.pop('ram'),
-                'os-ssd': kwargs.pop('os_ssd'),
-                'hdd': kwargs.pop('hdd')
-            }
-            data = kwargs
-        elif subcommand == 'list':
-            request_method = requests.get
-
-        make_request('host', subcommand, data=data, request_method=request_method)

uncloud/cli/image.py

@@ -1,38 +0,0 @@
-import requests
-
-from uncloud.cli.helper import make_request
-from uncloud.common.parser import BaseParser
-
-
-class ImageParser(BaseParser):
-    def __init__(self):
-        super().__init__('image')
-
-    def create(self, **kwargs):
-        p = self.subparser.add_parser('create', **kwargs)
-        p.add_argument('--name', required=True)
-        p.add_argument('--uuid', required=True)
-        p.add_argument('--image-store', required=True, dest='image_store')
-
-    def list(self, **kwargs):
-        self.subparser.add_parser('list', **kwargs)
-
-
-parser = ImageParser()
-arg_parser = parser.arg_parser
-
-
-def main(**kwargs):
-    subcommand = kwargs.pop('image_subcommand')
-    if not subcommand:
-        arg_parser.print_help()
-    else:
-        data = None
-        request_method = requests.post
-        if subcommand == 'list':
-            subcommand = 'list-public'
-            request_method = requests.get
-        elif subcommand == 'create':
-            data = kwargs
-
-        make_request('image', subcommand, data=data, request_method=request_method)

uncloud/cli/main.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import importlib
-
-arg_parser = argparse.ArgumentParser('cli', add_help=False)
-subparser = arg_parser.add_subparsers(dest='subcommand')
-
-for component in ['user', 'host', 'image', 'network', 'vm']:
-    module = importlib.import_module('uncloud.cli.{}'.format(component))
-    parser = getattr(module, 'arg_parser')
-    subparser.add_parser(name=parser.prog, parents=[parser])
-
-
-def main(arguments):
-    if not arguments['subcommand']:
-        arg_parser.print_help()
-    else:
-        name = arguments.pop('subcommand')
-        arguments.pop('debug')
-        mod = importlib.import_module('uncloud.cli.{}'.format(name))
-        _main = getattr(mod, 'main')
-        _main(**arguments)

uncloud/cli/network.py

@@ -1,32 +0,0 @@
-import requests
-
-from uncloud.cli.helper import make_request, get_otp_parser
-from uncloud.common.parser import BaseParser
-
-
-class NetworkParser(BaseParser):
-    def __init__(self):
-        super().__init__('network')
-
-    def create(self, **kwargs):
-        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs)
-        p.add_argument('--network-name', required=True)
-        p.add_argument('--network-type', required=True, dest='type')
-        p.add_argument('--user', action='store_true')
-
-
-parser = NetworkParser()
-arg_parser = parser.arg_parser
-
-
-def main(**kwargs):
-    subcommand = kwargs.pop('network_subcommand')
-    if not subcommand:
-        arg_parser.print_help()
-    else:
-        data = None
-        request_method = requests.post
-        if subcommand == 'create':
-            data = kwargs
-
-        make_request('network', subcommand, data=data, request_method=request_method)

uncloud/cli/user.py

@@ -1,41 +0,0 @@
-from uncloud.cli.helper import make_request, get_otp_parser
-from uncloud.common.parser import BaseParser
-
-
-class UserParser(BaseParser):
-    def __init__(self):
-        super().__init__('user')
-
-    def files(self, **kwargs):
-        self.subparser.add_parser('files', parents=[get_otp_parser()], **kwargs)
-
-    def vms(self, **kwargs):
-        self.subparser.add_parser('vms', parents=[get_otp_parser()], **kwargs)
-
-    def networks(self, **kwargs):
-        self.subparser.add_parser('networks', parents=[get_otp_parser()], **kwargs)
-
-    def add_ssh(self, **kwargs):
-        p = self.subparser.add_parser('add-ssh', parents=[get_otp_parser()], **kwargs)
-        p.add_argument('--key-name', required=True)
-        p.add_argument('--key', required=True)
-
-    def get_ssh(self, **kwargs):
-        p = self.subparser.add_parser('get-ssh', parents=[get_otp_parser()], **kwargs)
-        p.add_argument('--key-name', default='')
-
-    def remove_ssh(self, **kwargs):
-        p = self.subparser.add_parser('remove-ssh', parents=[get_otp_parser()], **kwargs)
-        p.add_argument('--key-name', required=True)
-
-
-parser = UserParser()
-arg_parser = parser.arg_parser
-
-
-def main(**kwargs):
-    subcommand = kwargs.pop('user_subcommand')
-    if not subcommand:
-        arg_parser.print_help()
-    else:
-        make_request('user', subcommand, data=kwargs)

uncloud/cli/vm.py

@@ -1,62 +0,0 @@
-from uncloud.common.parser import BaseParser
-from uncloud.cli.helper import make_request, get_otp_parser
-
-
-class VMParser(BaseParser):
-    def __init__(self):
-        super().__init__('vm')
-
-    def start(self, **args):
-        p = self.subparser.add_parser('start', parents=[get_otp_parser()], **args)
-        p.add_argument('--vm-name', required=True)
-
-    def stop(self, **args):
-        p = self.subparser.add_parser('stop', parents=[get_otp_parser()], **args)
-        p.add_argument('--vm-name', required=True)
-
-    def status(self, **args):
-        p = self.subparser.add_parser('status', parents=[get_otp_parser()], **args)
-        p.add_argument('--vm-name', required=True)
-
-    def delete(self, **args):
-        p = self.subparser.add_parser('delete', parents=[get_otp_parser()], **args)
-        p.add_argument('--vm-name', required=True)
-
-    def migrate(self, **args):
-        p = self.subparser.add_parser('migrate', parents=[get_otp_parser()], **args)
-        p.add_argument('--vm-name', required=True)
-        p.add_argument('--destination', required=True)
-
-    def create(self, **args):
-        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **args)
-        p.add_argument('--cpu', required=True)
-        p.add_argument('--ram', required=True)
-        p.add_argument('--os-ssd', required=True)
-        p.add_argument('--hdd', action='append', default=list())
-        p.add_argument('--image', required=True)
-        p.add_argument('--network', action='append', default=[])
-        p.add_argument('--vm-name', required=True)
-
-
-parser = VMParser()
-arg_parser = parser.arg_parser
-
-
-def main(**kwargs):
-    subcommand = kwargs.pop('vm_subcommand')
-    if not subcommand:
-        arg_parser.print_help()
-    else:
-        data = kwargs
-        endpoint = subcommand
-        if subcommand in ['start', 'stop', 'delete']:
-            endpoint = 'action'
-            data['action'] = subcommand
-        elif subcommand == 'create':
-            kwargs['specs'] = {
-                'cpu': kwargs.pop('cpu'),
-                'ram': kwargs.pop('ram'),
-                'os-ssd': kwargs.pop('os_ssd'),
-                'hdd': kwargs.pop('hdd')
-            }
-        make_request('vm', endpoint, data=data)

uncloud/common/cli.py

@@ -1,26 +0,0 @@
-from uncloud.common.shared import shared
-from pyotp import TOTP
-
-
-def get_token(seed):
-    if seed is not None:
-        try:
-            token = TOTP(seed).now()
-        except Exception:
-            raise Exception('Invalid seed')
-        else:
-            return token
-
-
-def resolve_otp_credentials(kwargs):
-    d = {
-        'name': shared.settings['client']['name'],
-        'realm': shared.settings['client']['realm'],
-        'token': get_token(shared.settings['client']['seed'])
-    }
-
-    for k, v in d.items():
-        if k in kwargs and kwargs[k] is None:
-            kwargs.update({k: v})
-
-    return d

uncloud/common/etcd_wrapper.py

@@ -1,21 +1,24 @@
 import etcd3
 import json
+import queue
+import copy
+from uncloud import UncloudException
 
+from collections import namedtuple
 from functools import wraps
 
-from uncloud import UncloudException
-from uncloud.common import logger
+from . import logger
+
+PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"])
 
 
 class EtcdEntry:
-    def __init__(self, meta_or_key, value, value_in_json=False):
-        if hasattr(meta_or_key, 'key'):
-            # if meta has attr 'key' then get it
-            self.key = meta_or_key.key.decode('utf-8')
-        else:
-            # otherwise meta is the 'key'
-            self.key = meta_or_key
-        self.value = value.decode('utf-8')
+    # key: str
+    # value: str
+
+    def __init__(self, meta, value, value_in_json=False):
+        self.key = meta.key.decode("utf-8")
+        self.value = value.decode("utf-8")
 
         if value_in_json:
             self.value = json.loads(self.value)
@@ -26,12 +29,18 @@ def readable_errors(func):
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
-        except etcd3.exceptions.ConnectionFailedError:
-            raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?')
+        except etcd3.exceptions.ConnectionFailedError as err:
+            raise UncloudException(
+                "Cannot connect to etcd: is etcd running as configured in uncloud.conf?"
+            )
         except etcd3.exceptions.ConnectionTimeoutError as err:
-            raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err
+            raise etcd3.exceptions.ConnectionTimeoutError(
+                "etcd connection timeout."
+            ) from err
         except Exception:
-            logger.exception('Some etcd error occured. See syslog for details.')
+            logger.exception(
+                "Some etcd error occured. See syslog for details."
+            )
 
     return wrapper
 
@@ -55,21 +64,55 @@ class Etcd3Wrapper:
             _value = json.dumps(_value)
 
         if not isinstance(_key, str):
-            _key = _key.decode('utf-8')
+            _key = _key.decode("utf-8")
 
         return self.client.put(_key, _value, **kwargs)
 
     @readable_errors
-    def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs):
-        event_iterator = self.client.get_prefix(*args, **kwargs)
-        for e in event_iterator:
-            yield EtcdEntry(*e[::-1], value_in_json=value_in_json)
+    def get_prefix(self, *args, value_in_json=False, **kwargs):
+        r = self.client.get_prefix(*args, **kwargs)
+        for entry in r:
+            e = EtcdEntry(*entry[::-1], value_in_json=value_in_json)
+            if e.value:
+                yield e
 
     @readable_errors
-    def watch_prefix(self, key, raise_exception=True, value_in_json=False):
-        event_iterator, cancel = self.client.watch_prefix(key)
-        for e in event_iterator:
-            if hasattr(e, '_event'):
-                e = e._event
-                if e.type == e.PUT:
-                    yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)
+    def watch_prefix(self, key, timeout=0, value_in_json=False):
+        timeout_event = EtcdEntry(
+            PseudoEtcdMeta(key=b"TIMEOUT"),
+            value=str.encode(
+                json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"})
+            ),
+            value_in_json=value_in_json,
+        )
+
+        event_queue = queue.Queue()
+
+        def add_event_to_queue(event):
+            if hasattr(event, "events"):
+                for e in event.events:
+                    if e.value:
+                        event_queue.put(
+                            EtcdEntry(
+                                e, e.value, value_in_json=value_in_json
+                            )
+                        )
+
+        self.client.add_watch_prefix_callback(key, add_event_to_queue)
+
+        while True:
+            try:
+                while True:
+                    v = event_queue.get(timeout=timeout)
+                    yield v
+            except queue.Empty:
+                event_queue.put(copy.deepcopy(timeout_event))
+
+
+class PsuedoEtcdEntry(EtcdEntry):
+    def __init__(self, key, value, value_in_json=False):
+        super().__init__(
+            PseudoEtcdMeta(key=key.encode("utf-8")),
+            value,
+            value_in_json=value_in_json,
+        )

uncloud/common/network.py

@@ -1,6 +1,8 @@
 import subprocess as sp
 import random
 import logging
+import socket
+from contextlib import closing
 
 logger = logging.getLogger(__name__)
 

uncloud/common/parser.py

@@ -1,13 +0,0 @@
-import argparse
-
-
-class BaseParser:
-    def __init__(self, command):
-        self.arg_parser = argparse.ArgumentParser(command, add_help=False)
-        self.subparser = self.arg_parser.add_subparsers(dest='{}_subcommand'.format(command))
-        self.common_args = {'add_help': False}
-
-        methods = [attr for attr in dir(self) if not attr.startswith('__')
-                   and type(getattr(self, attr)).__name__ == 'method']
-        for method in methods:
-            getattr(self, method)(**self.common_args)

uncloud/common/request.py

@@ -2,8 +2,8 @@ import json
 from os.path import join
 from uuid import uuid4
 
-from uncloud.common.etcd_wrapper import EtcdEntry
-from uncloud.common.classes import SpecificEtcdEntryBase
+from .etcd_wrapper import PsuedoEtcdEntry
+from .classes import SpecificEtcdEntryBase
 
 
 class RequestType:
@@ -29,8 +29,11 @@ class RequestEntry(SpecificEtcdEntryBase):
 
     @classmethod
     def from_scratch(cls, request_prefix, **kwargs):
-        e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex),
-                      value=json.dumps(kwargs).encode('utf-8'), value_in_json=True)
+        e = PsuedoEtcdEntry(
+            join(request_prefix, uuid4().hex),
+            value=json.dumps(kwargs).encode("utf-8"),
+            value_in_json=True,
+        )
         return cls(e)
 
 

uncloud/common/settings.py

@@ -1,136 +0,0 @@
-import configparser
-import logging
-import sys
-import os
-
-from datetime import datetime
-from uncloud.common.etcd_wrapper import Etcd3Wrapper
-from os.path import join as join_path
-
-logger = logging.getLogger(__name__)
-settings = None
-
-
-class CustomConfigParser(configparser.RawConfigParser):
-    def __getitem__(self, key):
-        try:
-            result = super().__getitem__(key)
-        except KeyError as err:
-            raise KeyError(
-                'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format(
-                    key
-                )
-            ) from err
-        else:
-            return result
-
-
-class Settings(object):
-    def __init__(self, conf_dir, seed_value=None):
-        conf_name = 'uncloud.conf'
-        self.config_file = join_path(conf_dir, conf_name)
-
-        # this is used to cache config from etcd for 1 minutes. Without this we
-        # would make a lot of requests to etcd which slows down everything.
-        self.last_config_update = datetime.fromtimestamp(0)
-
-        self.config_parser = CustomConfigParser(allow_no_value=True)
-        self.config_parser.add_section('etcd')
-        self.config_parser.set('etcd', 'base_prefix', '/')
-
-        if os.access(self.config_file, os.R_OK):
-            self.config_parser.read(self.config_file)
-        else:
-            raise FileNotFoundError('Config file %s not found!', self.config_file)
-        self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
-
-        self.read_internal_values()
-
-        if seed_value is None:
-            seed_value = dict()
-
-        self.config_parser.read_dict(seed_value)
-
-    def get_etcd_client(self):
-        args = tuple()
-        try:
-            kwargs = {
-                'host': self.config_parser.get('etcd', 'url'),
-                'port': self.config_parser.get('etcd', 'port'),
-                'ca_cert': self.config_parser.get('etcd', 'ca_cert'),
-                'cert_cert': self.config_parser.get('etcd', 'cert_cert'),
-                'cert_key': self.config_parser.get('etcd', 'cert_key'),
-            }
-        except configparser.Error as err:
-            raise configparser.Error(
-                '{} in config file {}'.format(
-                    err.message, self.config_file
-                )
-            ) from err
-        else:
-            try:
-                wrapper = Etcd3Wrapper(*args, **kwargs)
-            except Exception as err:
-                logger.error(
-                    'etcd connection not successfull. Please check your config file.'
-                    '\nDetails: %s\netcd connection parameters: %s',
-                    err,
-                    kwargs,
-                )
-                sys.exit(1)
-            else:
-                return wrapper
-
-    def read_internal_values(self):
-        base_prefix = self['etcd']['base_prefix']
-        self.config_parser.read_dict(
-            {
-                'etcd': {
-                    'file_prefix': join_path(base_prefix, 'files/'),
-                    'host_prefix': join_path(base_prefix, 'hosts/'),
-                    'image_prefix': join_path(base_prefix, 'images/'),
-                    'image_store_prefix': join_path(base_prefix, 'imagestore/'),
-                    'network_prefix': join_path(base_prefix, 'networks/'),
-                    'request_prefix': join_path(base_prefix, 'requests/'),
-                    'user_prefix': join_path(base_prefix, 'users/'),
-                    'vm_prefix': join_path(base_prefix, 'vms/'),
-                    'vxlan_counter': join_path(base_prefix, 'counters/vxlan'),
-                    'tap_counter': join_path(base_prefix, 'counters/tap')
-                }
-            }
-        )
-
-    def read_config_file_values(self, config_file):
-        try:
-            # Trying to read configuration file
-            with open(config_file) as config_file_handle:
-                self.config_parser.read_file(config_file_handle)
-        except FileNotFoundError:
-            sys.exit('Configuration file {} not found!'.format(config_file))
-        except Exception as err:
-            logger.exception(err)
-            sys.exit('Error occurred while reading configuration file')
-
-    def read_values_from_etcd(self):
-        etcd_client = self.get_etcd_client()
-        if (datetime.utcnow() - self.last_config_update).total_seconds() > 60:
-            config_from_etcd = etcd_client.get(self.config_key, value_in_json=True)
-            if config_from_etcd:
-                self.config_parser.read_dict(config_from_etcd.value)
-                self.last_config_update = datetime.utcnow()
-            else:
-                raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key))
-
-    def __getitem__(self, key):
-        # Allow failing to read from etcd if we have
-        # it locally
-        if key not in self.config_parser.sections():
-            try:
-                self.read_values_from_etcd()
-            except KeyError:
-                pass
-        return self.config_parser[key]
-
-
-def get_settings():
-    return settings

uncloud/common/shared.py

@@ -1,34 +0,0 @@
-from uncloud.common.settings import get_settings
-from uncloud.common.vm import VmPool
-from uncloud.common.host import HostPool
-from uncloud.common.request import RequestPool
-import uncloud.common.storage_handlers as storage_handlers
-
-
-class Shared:
-    @property
-    def settings(self):
-        return get_settings()
-
-    @property
-    def etcd_client(self):
-        return self.settings.get_etcd_client()
-
-    @property
-    def host_pool(self):
-        return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
-
-    @property
-    def vm_pool(self):
-        return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
-
-    @property
-    def request_pool(self):
-        return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
-
-    @property
-    def storage_handler(self):
-        return storage_handlers.get_storage_handler()
-
-
-shared = Shared()

uncloud/common/storage_handlers.py

@@ -6,7 +6,8 @@ import stat
 from abc import ABC
 from . import logger
 from os.path import join as join_path
-import uncloud.common.shared as shared
+
+from uncloud.settings import settings as config
 
 
 class ImageStorageHandler(ABC):
@@ -192,16 +193,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
 
 
 def get_storage_handler():
-    __storage_backend = shared.shared.settings["storage"]["storage_backend"]
+    __storage_backend = config["storage"]["storage_backend"]
     if __storage_backend == "filesystem":
         return FileSystemBasedImageStorageHandler(
-            vm_base=shared.shared.settings["storage"]["vm_dir"],
-            image_base=shared.shared.settings["storage"]["image_dir"],
+            vm_base=config["storage"]["vm_dir"],
+            image_base=config["storage"]["image_dir"],
         )
     elif __storage_backend == "ceph":
         return CEPHBasedImageStorageHandler(
-            vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
-            image_base=shared.shared.settings["storage"]["ceph_image_pool"],
+            vm_base=config["storage"]["ceph_vm_pool"],
+            image_base=config["storage"]["ceph_image_pool"],
         )
     else:
-        raise Exception("Unknown Image Storage Handler")
+        raise Exception("Unknown Image Storage Handler")

uncloud/configure/main.py

@@ -1,57 +1,79 @@
 import os
-import argparse
 
-from uncloud.common.shared import shared
-
-arg_parser = argparse.ArgumentParser('configure', add_help=False)
-configure_subparsers = arg_parser.add_subparsers(dest='subcommand')
-
-otp_parser = configure_subparsers.add_parser('otp')
-otp_parser.add_argument('--verification-controller-url', required=True, metavar='URL')
-otp_parser.add_argument('--auth-name', required=True, metavar='OTP-NAME')
-otp_parser.add_argument('--auth-realm', required=True, metavar='OTP-REALM')
-otp_parser.add_argument('--auth-seed', required=True, metavar='OTP-SEED')
-
-network_parser = configure_subparsers.add_parser('network')
-network_parser.add_argument('--prefix-length', required=True, type=int)
-network_parser.add_argument('--prefix', required=True)
-network_parser.add_argument('--vxlan-phy-dev', required=True)
-
-netbox_parser = configure_subparsers.add_parser('netbox')
-netbox_parser.add_argument('--url', required=True)
-netbox_parser.add_argument('--token', required=True)
-
-ssh_parser = configure_subparsers.add_parser('ssh')
-ssh_parser.add_argument('--username', default='root')
-ssh_parser.add_argument('--private-key-path', default=os.path.expanduser('~/.ssh/id_rsa'),)
-
-storage_parser = configure_subparsers.add_parser('storage')
-storage_parser.add_argument('--file-dir', required=True)
-storage_parser_subparsers = storage_parser.add_subparsers(dest='storage_backend')
-
-filesystem_storage_parser = storage_parser_subparsers.add_parser('filesystem')
-filesystem_storage_parser.add_argument('--vm-dir', required=True)
-filesystem_storage_parser.add_argument('--image-dir', required=True)
-
-ceph_storage_parser = storage_parser_subparsers.add_parser('ceph')
-ceph_storage_parser.add_argument('--ceph-vm-pool', required=True)
-ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
+from uncloud.settings import settings
+from uncloud.shared import shared
 
 
 def update_config(section, kwargs):
-    uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
+    uncloud_config = shared.etcd_client.get(
+        settings.config_key, value_in_json=True
+    )
     if not uncloud_config:
         uncloud_config = {}
     else:
         uncloud_config = uncloud_config.value
 
     uncloud_config[section] = kwargs
-    shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
+    shared.etcd_client.put(
+        settings.config_key, uncloud_config, value_in_json=True
+    )
 
 
-def main(arguments):
-    subcommand = arguments['subcommand']
+def configure_parser(parser):
+    configure_subparsers = parser.add_subparsers(dest="subcommand")
+
+    otp_parser = configure_subparsers.add_parser("otp")
+    otp_parser.add_argument(
+        "--verification-controller-url", required=True, metavar="URL"
+    )
+    otp_parser.add_argument(
+        "--auth-name", required=True, metavar="OTP-NAME"
+    )
+    otp_parser.add_argument(
+        "--auth-realm", required=True, metavar="OTP-REALM"
+    )
+    otp_parser.add_argument(
+        "--auth-seed", required=True, metavar="OTP-SEED"
+    )
+
+    network_parser = configure_subparsers.add_parser("network")
+    network_parser.add_argument(
+        "--prefix-length", required=True, type=int
+    )
+    network_parser.add_argument("--prefix", required=True)
+    network_parser.add_argument("--vxlan-phy-dev", required=True)
+
+    netbox_parser = configure_subparsers.add_parser("netbox")
+    netbox_parser.add_argument("--url", required=True)
+    netbox_parser.add_argument("--token", required=True)
+
+    ssh_parser = configure_subparsers.add_parser("ssh")
+    ssh_parser.add_argument("--username", default="root")
+    ssh_parser.add_argument(
+        "--private-key-path",
+        default=os.path.expanduser("~/.ssh/id_rsa"),
+    )
+
+    storage_parser = configure_subparsers.add_parser("storage")
+    storage_parser.add_argument("--file-dir", required=True)
+    storage_parser_subparsers = storage_parser.add_subparsers(
+        dest="storage_backend"
+    )
+
+    filesystem_storage_parser = storage_parser_subparsers.add_parser(
+        "filesystem"
+    )
+    filesystem_storage_parser.add_argument("--vm-dir", required=True)
+    filesystem_storage_parser.add_argument("--image-dir", required=True)
+
+    ceph_storage_parser = storage_parser_subparsers.add_parser("ceph")
+    ceph_storage_parser.add_argument("--ceph-vm-pool", required=True)
+    ceph_storage_parser.add_argument("--ceph-image-pool", required=True)
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop("subcommand")
     if not subcommand:
-        arg_parser.print_help()
+        pass
     else:
-        update_config(subcommand, arguments)
+        update_config(subcommand, kwargs)

uncloud/docs/Makefile

@@ -7,7 +7,7 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source/
 BUILDDIR      = build/
-DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/
+DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/
 
 .PHONY: all build clean
 

uncloud/docs/source/admin-guide

@@ -56,13 +56,40 @@ To start host we created earlier, execute the following command
 
     ucloud host ungleich.ch
 
-File & image scanners
---------------------------
+Create OS Image
+---------------
 
-Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our
-uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by
-ucloud. We can only make images from tracked files. So, we need to track the
-file by running File Scanner
+Create ucloud-init ready OS image (Optional)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+This step is optional if you just want to test ucloud. However, sooner or later
+you want to create OS images with ucloud-init to properly
+contexualize VMs.
+
+1. Start a VM with OS image on which you want to install ucloud-init
+2. Execute the following command on the started VM
+
+   .. code-block:: sh
+
+    apk add git
+    git clone https://code.ungleich.ch/ucloud/ucloud-init.git
+    cd ucloud-init
+    sh ./install.sh
+3. Congratulations. Your image is now ucloud-init ready.
+
+
+Upload Sample OS Image
+~~~~~~~~~~~~~~~~~~~~~~
+Execute the following to get the sample OS image file.
+
+.. code-block:: sh
+
+    mkdir /var/www/admin
+    (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download)
+
+Run File Scanner and Image Scanner
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make
+images from tracked files. So, we need to track the file by running File Scanner
 
 .. code-block:: sh
 

uncloud/docs/source/hacking.rst

@@ -0,0 +1,17 @@
+Hacking
+=======
+How to hack on the code.
+
+[ to be done by Balazs:
+
+* make nice
+* indent with shell script mode
+
+]
+
+* git clone the repo
+* cd to the repo
+* Setup your venv: python -m venv venv
+* . ./venv/bin/activate # you need the leading dot for sourcing!
+* Run ./bin/ucloud-run-reinstall - it should print you an error
+  message on how to use ucloud

uncloud/docs/source/index.rst

@@ -11,13 +11,14 @@ Welcome to ucloud's documentation!
    :caption: Contents:
 
    introduction
-   setup-install
-   vm-images
    user-guide
+   setup-install
    admin-guide
+   user-guide/how-to-create-an-os-image-for-ucloud
    troubleshooting
    hacking
 
+
 Indices and tables
 ==================
 

uncloud/filescanner/main.py

@@ -3,16 +3,12 @@ import os
 import pathlib
 import subprocess as sp
 import time
-import argparse
-import bitmath
 
 from uuid import uuid4
 
 from . import logger
-from uncloud.common.shared import shared
-
-arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
-arg_parser.add_argument('--hostname', required=True)
+from uncloud.settings import settings
+from uncloud.shared import shared
 
 
 def sha512sum(file: str):
@@ -28,58 +24,66 @@ def sha512sum(file: str):
     if not isinstance(file, str):
         raise TypeError
     try:
-        output = sp.check_output(['sha512sum', file], stderr=sp.PIPE)
+        output = sp.check_output(["sha512sum", file], stderr=sp.PIPE)
     except sp.CalledProcessError as e:
-        error = e.stderr.decode('utf-8')
-        if 'No such file or directory' in error:
+        error = e.stderr.decode("utf-8")
+        if "No such file or directory" in error:
             raise FileNotFoundError from None
     else:
-        output = output.decode('utf-8').strip()
-        output = output.split(' ')
+        output = output.decode("utf-8").strip()
+        output = output.split(" ")
         return output[0]
     return None
 
 
-def track_file(file, base_dir, host):
-    file_path = file.relative_to(base_dir)
-    file_str = str(file)
+def track_file(file, base_dir):
+    file_id = uuid4()
+
     # Get Username
-    try:
-        owner = file_path.parts[0]
-    except IndexError:
-        pass
-    else:
-        file_path = file_path.relative_to(owner)
-        creation_date = time.ctime(os.stat(file_str).st_ctime)
+    owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)]
 
-        entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
-        entry_value = {
-            'filename': str(file_path),
-            'owner': owner,
-            'sha512sum': sha512sum(file_str),
-            'creation_date': creation_date,
-            'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()),
-            'host': host
-        }
+    # Get Creation Date of File
+    # Here, we are assuming that ctime is creation time
+    # which is mostly not true.
+    creation_date = time.ctime(os.stat(file).st_ctime)
 
-        logger.info('Tracking %s', file_str)
+    file_path = pathlib.Path(file).parts[-1]
 
-        shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
+    # Create Entry
+    entry_key = os.path.join(
+        settings["etcd"]["file_prefix"], str(file_id)
+    )
+    entry_value = {
+        "filename": file_path,
+        "owner": owner,
+        "sha512sum": sha512sum(file),
+        "creation_date": creation_date,
+        "size": os.path.getsize(file),
+    }
+
+    logger.info("Tracking %s", file)
+
+    shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
+    os.setxattr(file, "user.utracked", b"True")
 
 
-def main(arguments):
-    hostname = arguments['hostname']
-    base_dir = shared.settings['storage']['file_dir']
+def main():
+    base_dir = settings["storage"]["file_dir"]
+
     # Recursively Get All Files and Folder below BASE_DIR
-    files = glob.glob('{}/**'.format(base_dir), recursive=True)
-    files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
+    files = glob.glob("{}/**".format(base_dir), recursive=True)
 
-    # Files that are already tracked
-    tracked_files = [
-        pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
-        for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
-        if f.value['host'] == hostname
-    ]
-    untracked_files = set(files) - set(tracked_files)
-    for file in untracked_files:
-        track_file(file, base_dir, hostname)
+    # Retain only Files
+    files = [file for file in files if os.path.isfile(file)]
+
+    untracked_files = []
+    for file in files:
+        try:
+            os.getxattr(file, "user.utracked")
+        except OSError:
+            track_file(file, base_dir)
+            untracked_files.append(file)
+
+
+if __name__ == "__main__":
+    main()

uncloud/hack/config.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
-#
-# This file is part of uncloud.
-#
-# uncloud is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# uncloud is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-class Config(object):
-    def __init__(self, arguments):
-        """ read arguments dicts as a base """
-
-        self.arguments = arguments
-
-        # Split them so *etcd_args can be used and we can
-        # iterate over etcd_hosts
-        self.etcd_hosts = [ arguments['etcd_host'] ]
-        self.etcd_args = {
-            'ca_cert': arguments['etcd_ca_cert'],
-            'cert_cert': arguments['etcd_cert_cert'],
-            'cert_key': arguments['etcd_cert_key'],
-#            'user': None,
-#            'password': None
-        }
-        self.etcd_prefix = '/nicohack/'

uncloud/hack/db.py

@@ -1,113 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
-#
-# This file is part of uncloud.
-#
-# uncloud is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# uncloud is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import etcd3
-import json
-import logging
-
-from functools import wraps
-from uncloud import UncloudException
-
-log = logging.getLogger(__name__)
-
-
-def readable_errors(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except etcd3.exceptions.ConnectionFailedError as e:
-            raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
-        except etcd3.exceptions.ConnectionTimeoutError as e:
-            raise UncloudException('etcd connection timeout. {}'.format(e))
-
-    return wrapper
-
-
-class DB(object):
-    def __init__(self, config, prefix="/"):
-        self.config = config
-
-        # Root for everything
-        self.base_prefix= '/nicohack'
-
-        # Can be set from outside
-        self.prefix = prefix
-
-        self.connect()
-
-    @readable_errors
-    def connect(self):
-        self._db_clients = []
-        for endpoint in self.config.etcd_hosts:
-            client = etcd3.client(host=endpoint, **self.config.etcd_args)
-            self._db_clients.append(client)
-
-    def realkey(self, key):
-        return "{}{}/{}".format(self.base_prefix,
-                                self.prefix,
-                                key)
-
-    @readable_errors
-    def get(self, key, as_json=False, **kwargs):
-        value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
-
-        if as_json:
-            value = json.loads(value)
-
-        return value
-
-
-    @readable_errors
-    def set(self, key, value, as_json=False, **kwargs):
-        if as_json:
-            value = json.dumps(value)
-
-        # FIXME: iterate over clients in case of failure ?
-        return self._db_clients[0].put(self.realkey(key), value, **kwargs)
-
-    @readable_errors
-    def increment(self, key, **kwargs):
-        print(self.realkey(key))
-
-
-        print("prelock")
-        lock = self._db_clients[0].lock('/nicohack/foo')
-        print("prelockacq")
-        lock.acquire()
-        print("prelockrelease")
-        lock.release()
-
-        with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock:
-            print("in lock")
-            pass
-
-#        with self._db_clients[0].lock(self.realkey(key)) as lock:#            value = int(self.get(self.realkey(key), **kwargs))
-#            self.set(self.realkey(key), str(value + 1), **kwargs)
-
-
-if __name__ == '__main__':
-    endpoints = [ "https://etcd1.ungleich.ch:2379",
-                  "https://etcd2.ungleich.ch:2379",
-                  "https://etcd3.ungleich.ch:2379" ]
-
-    db = DB(url=endpoints)

uncloud/hack/hackcloud/.gitignore

@@ -1,3 +0,0 @@
-*.iso
-radvdpid
-foo

uncloud/hack/hackcloud/__init__.py

@@ -1 +0,0 @@
-

uncloud/hack/hackcloud/etcd-client.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-etcdctl  --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \
-         --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \
-         --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \
-         --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@"

uncloud/hack/hackcloud/ifdown.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-echo $@

uncloud/hack/hackcloud/ifup.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-dev=$1; shift
-
-# bridge is setup from outside
-ip link set dev "$dev" master ${bridge}
-ip link set dev "$dev" up

uncloud/hack/hackcloud/mac-last

@@ -1 +0,0 @@
-000000000252

uncloud/hack/hackcloud/mac-prefix

@@ -1 +0,0 @@
-02:00

uncloud/hack/hackcloud/net.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-set -x
-
-netid=100
-dev=wlp2s0
-dev=wlp0s20f3
-#dev=wlan0
-
-ip=2a0a:e5c1:111:888::48/64
-vxlandev=vxlan${netid}
-bridgedev=br${netid}
-
-ip -6 link add ${vxlandev} type vxlan \
-    id ${netid} \
-    dstport 4789 \
-    group ff05::${netid} \
-    dev ${dev} \
-    ttl 5
-
-ip link set ${vxlandev} up
-
-
-ip link add ${bridgedev} type bridge
-ip link set ${bridgedev} up
-
-ip link set ${vxlandev} master ${bridgedev} up
-
-ip addr add ${ip} dev ${bridgedev}

uncloud/hack/hackcloud/nftrules

@@ -1,31 +0,0 @@
-flush ruleset
-
-table bridge filter {
-    chain prerouting {
-        type filter hook prerouting priority 0;
-        policy accept;
-
-        ibrname br100 jump br100
-    }
-
-    chain br100 {
-        # Allow all incoming traffic from outside
-        iifname vxlan100 accept
-
-        # Default blocks: router advertisements, dhcpv6, dhcpv4
-        icmpv6 type nd-router-advert drop
-        ip6 version 6 udp sport 547 drop
-        ip  version 4 udp sport 67 drop
-
-        jump br100_vmlist
-        drop
-    }
-    chain br100_vmlist {
-        # VM1
-        iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept
-
-        # VM2
-        iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept
-        iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept
-    }
-}

uncloud/hack/hackcloud/radvd.conf

@@ -1,13 +0,0 @@
-interface br100
-{
-  AdvSendAdvert on;
-  MinRtrAdvInterval 3;
-  MaxRtrAdvInterval 5;
-  AdvDefaultLifetime 3600;
-
-  prefix 2a0a:e5c1:111:888::/64 {
-  };
-
-  RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; };
-  DNSSL place7.ungleich.ch {  AdvDNSSLLifetime 6000; } ;
-};

uncloud/hack/hackcloud/radvd.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-radvd -C ./radvd.conf  -n -p ./radvdpid

uncloud/hack/hackcloud/vm.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-# if [ $# -ne 1 ]; then
-#     echo "$0: owner"
-#     exit 1
-# fi
-
-qemu=/usr/bin/qemu-system-x86_64
-
-accel=kvm
-#accel=tcg
-
-memory=1024
-cores=2
-uuid=$(uuidgen)
-mac=$(./mac-gen.py)
-owner=nico
-
-export bridge=br100
-
-set -x
-$qemu -name "uncloud-${uuid}" \
-      -machine pc,accel=${accel} \
-      -m ${memory} \
-      -smp ${cores} \
-      -uuid ${uuid} \
-      -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \
-      -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \
-      -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac}

uncloud/hack/mac.py

@@ -1,102 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2012 Nico Schottelius (nico-cinv at schottelius.org)
-#
-# This file is part of cinv.
-#
-# cinv is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cinv is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cinv. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import argparse
-import logging
-import os.path
-import os
-import re
-import json
-
-from uncloud import UncloudException
-from uncloud.hack.db import DB
-
-log = logging.getLogger(__name__)
-
-
-class MAC(object):
-    def __init__(self, config):
-        self.config = config
-        self.no_db = self.config.arguments['no_db']
-        if not self.no_db:
-            self.db = DB(config, prefix="/mac")
-
-        self.prefix = 0x420000000000
-        self._number = 0 # Not set by default
-
-    @staticmethod
-    def validate_mac(mac):
-        if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I):
-            raise Error("Not a valid mac address: %s" % mac)
-
-    def last_used_index(self):
-        if not self.no_db:
-            value = self.db.get("last_used_index")
-            if not value:
-                self.db.set("last_used_index", "0")
-                value = self.db.get("last_used_index")
-
-        else:
-            value = "0"
-
-        return int(value)
-
-    def last_used_mac(self):
-        return self.int_to_mac(self.prefix + self.last_used_index())
-
-    def to_colon_format(self):
-        b = self._number.to_bytes(6, byteorder="big")
-        return ':'.join(format(s, '02x') for s in b)
-
-    def to_str_format(self):
-        b = self._number.to_bytes(6, byteorder="big")
-        return ''.join(format(s, '02x') for s in b)
-
-    def create(self):
-        last_number = self.last_used_index()
-
-        if last_number == int('0xffffffff', 16):
-            raise UncloudException("Exhausted all possible mac addresses - try to free some")
-
-        next_number = last_number + 1
-        self._number = self.prefix + next_number
-
-        #next_number_string = "{:012x}".format(next_number)
-        #next_mac = self.int_to_mac(next_mac_number)
-        # db_entry = {}
-        # db_entry['vm_uuid'] = vmuuid
-        # db_entry['index'] = next_number
-        # db_entry['mac_address'] = next_mac
-
-        # should be one transaction
-        # self.db.increment("last_used_index")
-        # self.db.set("used/{}".format(next_mac),
-        #             db_entry, as_json=True)
-
-    def __int__(self):
-        return self._number
-
-    def __repr__(self):
-        return self.to_str_format()
-
-    def __str__(self):
-        return self.to_colon_format()

uncloud/hack/main.py

@@ -1,92 +0,0 @@
-import argparse
-import logging
-
-from uncloud.hack.vm import VM
-from uncloud.hack.config import Config
-from uncloud.hack.mac import MAC
-from uncloud.hack.net import VXLANBridge, DNSRA
-
-from uncloud import UncloudException
-
-arg_parser = argparse.ArgumentParser('hack', add_help=False)
-                                     #description="Commands that are unfinished - use at own risk")
-arg_parser.add_argument('--last-used-mac', action='store_true')
-arg_parser.add_argument('--get-new-mac', action='store_true')
-
-arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true')
-arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true')
-arg_parser.add_argument('--network', help="/64 IPv6 network")
-arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0")
-arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int)
-arg_parser.add_argument('--run-dns-ra', action='store_true',
-                        help="Provide router advertisements and DNS resolution via dnsmasq")
-arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true')
-
-arg_parser.add_argument('--create-vm', action='store_true')
-arg_parser.add_argument('--destroy-vm', action='store_true')
-arg_parser.add_argument('--get-vm-status', action='store_true')
-arg_parser.add_argument('--get-vm-vnc', action='store_true')
-arg_parser.add_argument('--list-vms', action='store_true')
-arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int)
-arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int)
-arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image")
-arg_parser.add_argument('--uuid', help="VM UUID")
-
-arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true')
-arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh")
-
-
-log = logging.getLogger(__name__)
-
-
-def main(arguments):
-    config = Config(arguments)
-
-    if arguments['create_vm']:
-        vm = VM(config)
-        vm.create()
-
-    if arguments['destroy_vm']:
-        vm = VM(config)
-        vm.stop()
-
-    if arguments['get_vm_status']:
-        vm = VM(config)
-        vm.status()
-
-    if arguments['get_vm_vnc']:
-        vm = VM(config)
-        vm.vnc_addr()
-
-    if arguments['list_vms']:
-        vm = VM(config)
-        vm.list()
-
-    if arguments['last_used_mac']:
-        m = MAC(config)
-        print(m.last_used_mac())
-
-    if arguments['get_new_mac']:
-        print(MAC(config).get_next())
-
-    #if arguments['init_network']:
-    if arguments['create_vxlan']:
-        if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']:
-            raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)")
-        vb = VXLANBridge(vni=arguments['vni'],
-                         route=arguments['network'],
-                         uplinkdev=arguments['vxlan_uplink_device'],
-                         use_sudo=arguments['use_sudo'])
-        vb._setup_vxlan()
-        vb._setup_bridge()
-        vb._add_vxlan_to_bridge()
-        vb._route_network()
-
-    if arguments['run_dns_ra']:
-        if not arguments['network'] or not arguments['vni']:
-            raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)")
-
-        dnsra = DNSRA(route=arguments['network'],
-                      vni=arguments['vni'],
-                      use_sudo=arguments['use_sudo'])
-        dnsra._setup_dnsmasq()

uncloud/hack/net.py

@@ -1,116 +0,0 @@
-import subprocess
-import ipaddress
-import logging
-
-
-from uncloud import UncloudException
-
-log = logging.getLogger(__name__)
-
-
-class VXLANBridge(object):
-    cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5"
-    cmd_up_dev = "{sudo}ip link set {dev} up"
-    cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge"
-    cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up"
-    cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}"
-    cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}"
-
-    # VXLAN ids are at maximum 24 bit - use a /104
-    multicast_network = ipaddress.IPv6Network("ff05::/104")
-    max_vni = (2**24)-1
-
-    def __init__(self,
-                 vni,
-                 uplinkdev,
-                 route=None,
-                 use_sudo=False):
-        self.config = {}
-
-        if vni > self.max_vni:
-            raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni))
-
-        if use_sudo:
-            self.config['sudo'] = 'sudo '
-        else:
-            self.config['sudo'] = ''
-
-        self.config['vni_dec'] = vni
-        self.config['vni_hex'] = "{:x}".format(vni)
-        self.config['multicast_address'] = self.multicast_network[vni]
-
-        self.config['route_network'] = ipaddress.IPv6Network(route)
-        self.config['route'] = route
-
-        self.config['uplinkdev'] = uplinkdev
-        self.config['vxlandev'] = "vx{}".format(self.config['vni_hex'])
-        self.config['bridgedev'] = "br{}".format(self.config['vni_hex'])
-
-
-    def setup_networking(self):
-        pass
-
-    def _setup_vxlan(self):
-        self._execute_cmd(self.cmd_create_vxlan)
-        self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev'])
-
-    def _setup_bridge(self):
-        self._execute_cmd(self.cmd_create_bridge)
-        self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev'])
-
-    def _route_network(self):
-        self._execute_cmd(self.cmd_add_route_dev)
-
-    def _add_vxlan_to_bridge(self):
-        self._execute_cmd(self.cmd_add_to_bridge)
-
-    def _execute_cmd(self, cmd_string, **kwargs):
-        cmd = cmd_string.format(**self.config, **kwargs)
-        log.info("Executing: {}".format(cmd))
-        subprocess.run(cmd.split())
-
-class ManagementBridge(VXLANBridge):
-    pass
-
-
-class DNSRA(object):
-    # VXLAN ids are at maximum 24 bit
-    max_vni = (2**24)-1
-
-
-    # Command to start dnsmasq
-    cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra"
-
-    def __init__(self,
-                 vni,
-                 route=None,
-                 use_sudo=False):
-        self.config = {}
-
-        if vni > self.max_vni:
-            raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni))
-
-        if use_sudo:
-            self.config['sudo'] = 'sudo '
-        else:
-            self.config['sudo'] = ''
-
-        #TODO: remove if not needed
-        #self.config['vni_dec'] = vni
-        self.config['vni_hex'] = "{:x}".format(vni)
-
-        # dnsmasq only wants the network without the prefix, therefore, cut it off
-        self.config['route'] = ipaddress.IPv6Network(route).network_address
-        self.config['bridgedev'] = "br{}".format(self.config['vni_hex'])
-
-    def _setup_dnsmasq(self):
-         self._execute_cmd(self.cmd_start_dnsmasq)
-
-    def _execute_cmd(self, cmd_string, **kwargs):
-        cmd = cmd_string.format(**self.config, **kwargs)
-        log.info("Executing: {}".format(cmd))
-        print("Executing: {}".format(cmd))
-        subprocess.run(cmd.split())
-
-class Firewall(object):
-    pass

uncloud/hack/uncloud-hack-init-host

@@ -1,26 +0,0 @@
-id=100
-rawdev=eth0
-
-# create vxlan
-ip -6 link add vxlan${id} type vxlan \
-    id ${id} \
-    dstport 4789 \
-    group ff05::${id} \
-    dev ${rawdev} \
-    ttl 5
-
-ip link set vxlan${id} up
-
-# create bridge
-ip link set vxlan${id} up
-ip link set br${id} up
-
-# Add vxlan into bridge
-ip link set vxlan${id} master br${id}
-
-
-#  useradd -m uncloud
-# [18:05] tablett.place10:~# id uncloud
-# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu)
-# apk add qemu-system-x86_64
-# also needs group netdev

uncloud/hack/uncloud-run-vm

@@ -1,25 +0,0 @@
-#!/bin/sh
-
-if [ $# -ne 1 ]; then
-    echo $0 vmid
-    exit 1
-fi
-
-id=$1; shift
-
-memory=512
-macaddress=02:00:b9:cb:70:${id}
-netname=net${id}-1
-
-qemu-system-x86_64 \
-    -name uncloud-${id} \
-    -accel kvm \
-    -m ${memory} \
-    -smp 2,sockets=2,cores=1,threads=1 \
-    -device virtio-net-pci,netdev=net0,mac=$macaddress \
-    -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \
-    -vnc [::]:0
-
-# To be changed:
-# -vnc to unix path
-# or -spice

uncloud/hack/vm.py

@@ -1,136 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
-#
-# This file is part of uncloud.
-#
-# uncloud is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# uncloud is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
-
-# This module is directly called from the hack module, and can be used as follow:
-#
-#   Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix.
-#   `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2`
-#
-#   List running VMs (returns a list of UUIDs).
-#   `uncloud hack --hackprefix /tmp/hackcloud --list-vms
-#
-#   Get VM status:
-#   `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid`
-#
-#   Stop a VM:
-#   `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid`
-#    ``
-
-import subprocess
-import uuid
-import os
-import logging
-
-from uncloud.hack.db import DB
-from uncloud.hack.mac import MAC
-from uncloud.vmm import VMM
-
-log = logging.getLogger(__name__)
-log.setLevel(logging.DEBUG)
-
-class VM(object):
-    def __init__(self, config):
-        self.config = config
-
-        #TODO: Enable etcd lookup
-        self.no_db = self.config.arguments['no_db']
-        if not self.no_db:
-             self.db = DB(self.config, prefix="/vm")
-
-        # General CLI arguments.
-        self.hackprefix = self.config.arguments['hackprefix']
-        self.uuid = self.config.arguments['uuid']
-        self.memory = self.config.arguments['memory'] or '1024M'
-        self.cores = self.config.arguments['cores'] or 1
-        if self.config.arguments['image']:
-            self.image = os.path.join(self.hackprefix, self.config.arguments['image'])
-        else:
-            self.image = None
-
-        # External components.
-        self.vmm = VMM(vmm_backend=self.hackprefix)
-        self.mac = MAC(self.config)
-
-        # Harcoded & generated values.
-        self.owner = 'uncoud'
-        self.image_format='qcow2'
-        self.accel = 'kvm'
-        self.threads = 1
-        self.ifup = os.path.join(self.hackprefix, "ifup.sh")
-        self.ifdown = os.path.join(self.hackprefix, "ifdown.sh")
-        self.ifname = "uc{}".format(self.mac.to_str_format())
-
-    def get_qemu_args(self):
-        command = (
-            "-name {owner}-{name}"
-            " -machine pc,accel={accel}"
-            " -drive file={image},format={image_format},if=virtio"
-            " -device virtio-rng-pci"
-            " -m {memory} -smp cores={cores},threads={threads}"
-            " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}"
-            " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}"
-        ).format(
-            owner=self.owner, name=self.uuid,
-            accel=self.accel,
-            image=self.image, image_format=self.image_format,
-            memory=self.memory, cores=self.cores, threads=self.threads,
-            ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname,
-            mac=self.mac
-        )
-
-        return command.split(" ")
-
-    def create(self):
-        # New VM: new UUID, new MAC.
-        self.uuid = str(uuid.uuid4())
-        self.mac.create()
-
-        qemu_args = self.get_qemu_args()
-        log.debug("QEMU args passed to VMM: {}".format(qemu_args))
-        self.vmm.start(
-                uuid=self.uuid,
-                migration=False,
-                *qemu_args
-                )
-
-    def stop(self):
-        if not self.uuid:
-            print("Please specific an UUID with the --uuid flag.")
-            exit(1)
-
-        self.vmm.stop(self.uuid)
-
-    def status(self):
-        if not self.uuid:
-            print("Please specific an UUID with the --uuid flag.")
-            exit(1)
-
-        print(self.vmm.get_status(self.uuid))
-
-    def vnc_addr(self):
-        if not self.uuid:
-            print("Please specific an UUID with the --uuid flag.")
-            exit(1)
-
-        print(self.vmm.get_vnc(self.uuid))
-
-    def list(self):
-        print(self.vmm.discover())
-

uncloud/host/main.py

@@ -1,20 +1,17 @@
 import argparse
 import multiprocessing as mp
 import time
-
 from uuid import uuid4
 
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.common.shared import shared
+from uncloud.shared import shared
+from uncloud.settings import settings
 from uncloud.common.vm import VMStatus
 from uncloud.vmm import VMM
 from os.path import join as join_path
 
 from . import virtualmachine, logger
 
-arg_parser = argparse.ArgumentParser('host', add_help=False)
-arg_parser.add_argument('--hostname', required=True)
-
 
 def update_heartbeat(hostname):
     """Update Last HeartBeat Time for :param hostname: in etcd"""
@@ -32,10 +29,10 @@ def maintenance(host):
     vmm = VMM()
     running_vms = vmm.discover()
     for vm_uuid in running_vms:
-        if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
+        if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running":
             logger.debug('VM {} is running on {}'.format(vm_uuid, host))
             vm = shared.vm_pool.get(
-                join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
+                join_path(settings["etcd"]["vm_prefix"], vm_uuid)
             )
             vm.status = VMStatus.running
             vm.vnc_socket = vmm.get_vnc(vm_uuid)
@@ -43,81 +40,77 @@ def maintenance(host):
             shared.vm_pool.put(vm)
 
 
-def main(arguments):
-    hostname = arguments['hostname']
+def main(hostname):
     host_pool = shared.host_pool
     host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
 
     # Does not yet exist, create it
     if not host:
         host_key = join_path(
-            shared.settings['etcd']['host_prefix'], uuid4().hex
+            settings["etcd"]["host_prefix"], uuid4().hex
         )
         host_entry = {
-            'specs': '',
-            'hostname': hostname,
-            'status': 'DEAD',
-            'last_heartbeat': '',
+            "specs": "",
+            "hostname": hostname,
+            "status": "DEAD",
+            "last_heartbeat": "",
         }
         shared.etcd_client.put(
             host_key, host_entry, value_in_json=True
         )
 
-    # update, get ourselves now for sure
-    host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
-
     try:
         heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,))
         heartbeat_updating_process.start()
     except Exception as e:
-        raise Exception('uncloud-host heartbeat updating mechanism is not working') from e
+        raise Exception("uncloud-host heartbeat updating mechanism is not working") from e
 
-    # The below while True is neccessary for gracefully handling leadership transfer and temporary
-    # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
-    # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
-    # which ends the loop immediately. So, having it inside infinite loop we try again and again to
-    # get prefix until either success or deamon death comes.
-    while True:
-        for events_iterator in [
-            shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
-                                          raise_exception=False),
-            shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
-                                            raise_exception=False)
-        ]:
-            for request_event in events_iterator:
-                request_event = RequestEntry(request_event)
+    for events_iterator in [
+        shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True),
+        shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True)
+    ]:
+        for request_event in events_iterator:
+            request_event = RequestEntry(request_event)
 
+            if request_event.type == "TIMEOUT":
                 maintenance(host.key)
 
-                if request_event.hostname == host.key:
-                    logger.debug('VM Request: %s on Host %s', request_event, host.hostname)
+            elif request_event.hostname == host.key:
+                logger.debug("VM Request: %s on Host %s", request_event, host.hostname)
+                shared.request_pool.client.client.delete(request_event.key)
+                vm_entry = shared.etcd_client.get(
+                    join_path(settings["etcd"]["vm_prefix"], request_event.uuid)
+                )
+                logger.debug("VM hostname: {}".format(vm_entry.value))
+                vm = virtualmachine.VM(vm_entry)
+                if request_event.type == RequestType.StartVM:
+                    vm.start()
 
-                    shared.request_pool.client.client.delete(request_event.key)
-                    vm_entry = shared.etcd_client.get(
-                        join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid)
-                    )
+                elif request_event.type == RequestType.StopVM:
+                    vm.stop()
 
-                    logger.debug('VM hostname: {}'.format(vm_entry.value))
+                elif request_event.type == RequestType.DeleteVM:
+                    vm.delete()
 
-                    vm = virtualmachine.VM(vm_entry)
-                    if request_event.type == RequestType.StartVM:
-                        vm.start()
+                elif request_event.type == RequestType.InitVMMigration:
+                    vm.start(destination_host_key=host.key)
 
-                    elif request_event.type == RequestType.StopVM:
-                        vm.stop()
+                elif request_event.type == RequestType.TransferVM:
+                    destination_host = host_pool.get(request_event.destination_host_key)
+                    if destination_host:
+                        vm.migrate(
+                            destination_host=destination_host.hostname,
+                            destination_sock_path=request_event.destination_sock_path,
+                        )
+                    else:
+                        logger.error("Host %s not found!", request_event.destination_host_key)
 
-                    elif request_event.type == RequestType.DeleteVM:
-                        vm.delete()
 
-                    elif request_event.type == RequestType.InitVMMigration:
-                        vm.start(destination_host_key=host.key)
-
-                    elif request_event.type == RequestType.TransferVM:
-                        destination_host = host_pool.get(request_event.destination_host_key)
-                        if destination_host:
-                            vm.migrate(
-                                destination_host=destination_host.hostname,
-                                destination_sock_path=request_event.destination_sock_path,
-                            )
-                        else:
-                            logger.error('Host %s not found!', request_event.destination_host_key)
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser()
+    argparser.add_argument(
+        "hostname", help="Name of this host. e.g uncloud1.ungleich.ch"
+    )
+    args = argparser.parse_args()
+    mp.set_start_method("spawn")
+    main(args.hostname)

uncloud/host/virtualmachine.py

@@ -16,7 +16,8 @@ from uncloud.common.vm import VMStatus, declare_stopped
 from uncloud.common.network import create_dev, delete_network_interface
 from uncloud.common.schemas import VMSchema, NetworkSchema
 from uncloud.host import logger
-from uncloud.common.shared import shared
+from uncloud.shared import shared
+from uncloud.settings import settings
 from uncloud.vmm import VMM
 
 from marshmallow import ValidationError
@@ -41,7 +42,7 @@ class VM:
 
     def get_qemu_args(self):
         command = (
-            "-drive file={file},format=raw,if=virtio"
+            "-drive file={file},format=raw,if=virtio,cache=none"
             " -device virtio-rng-pci"
             " -m {memory} -smp cores={cores},threads={threads}"
             " -name {owner}_{name}"
@@ -90,7 +91,7 @@ class VM:
                         self.vmm.socket_dir, self.uuid
                     ),
                     destination_host_key=destination_host_key,  # Where source host transfer VM
-                    request_prefix=shared.settings["etcd"]["request_prefix"],
+                    request_prefix=settings["etcd"]["request_prefix"],
                 )
                 shared.request_pool.put(r)
             else:
@@ -118,7 +119,7 @@ class VM:
             network_name, mac, tap = network_mac_and_tap
 
             _key = os.path.join(
-                shared.settings["etcd"]["network_prefix"],
+                settings["etcd"]["network_prefix"],
                 self.vm["owner"],
                 network_name,
             )
@@ -132,13 +133,13 @@ class VM:
             if network["type"] == "vxlan":
                 tap = create_vxlan_br_tap(
                     _id=network["id"],
-                    _dev=shared.settings["network"]["vxlan_phy_dev"],
+                    _dev=settings["network"]["vxlan_phy_dev"],
                     tap_id=tap,
                     ip=network["ipv6"],
                 )
 
                 all_networks = shared.etcd_client.get_prefix(
-                    shared.settings["etcd"]["network_prefix"],
+                    settings["etcd"]["network_prefix"],
                     value_in_json=True,
                 )
 
@@ -152,10 +153,7 @@ class VM:
                 )
             )
 
-        if command:
-            command = command.split(' ')
-
-        return command
+        return command.split(" ")
 
     def delete_network_dev(self):
         try:
@@ -228,7 +226,7 @@ class VM:
 def resolve_network(network_name, network_owner):
     network = shared.etcd_client.get(
         join_path(
-            shared.settings["etcd"]["network_prefix"],
+            settings["etcd"]["network_prefix"],
             network_owner,
             network_name,
         ),

uncloud/imagescanner/main.py

@@ -1,16 +1,13 @@
 import json
 import os
-import argparse
 import subprocess as sp
 
 from os.path import join as join_path
-from uncloud.common.shared import shared
+from uncloud.settings import settings
+from uncloud.shared import shared
 from uncloud.imagescanner import logger
 
 
-arg_parser = argparse.ArgumentParser('imagescanner', add_help=False)
-
-
 def qemu_img_type(path):
     qemu_img_info_command = [
         "qemu-img",
@@ -29,10 +26,10 @@ def qemu_img_type(path):
         return qemu_img_info["format"]
 
 
-def main(arguments):
+def main():
     # We want to get images entries that requests images to be created
     images = shared.etcd_client.get_prefix(
-        shared.settings["etcd"]["image_prefix"], value_in_json=True
+        settings["etcd"]["image_prefix"], value_in_json=True
     )
     images_to_be_created = list(
         filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
@@ -45,13 +42,13 @@ def main(arguments):
             image_filename = image.value["filename"]
             image_store_name = image.value["store_name"]
             image_full_path = join_path(
-                shared.settings["storage"]["file_dir"],
+                settings["storage"]["file_dir"],
                 image_owner,
                 image_filename,
             )
 
             image_stores = shared.etcd_client.get_prefix(
-                shared.settings["etcd"]["image_store_prefix"],
+                settings["etcd"]["image_store_prefix"],
                 value_in_json=True,
             )
             user_image_store = next(

uncloud/metadata/main.py

@@ -1,22 +1,17 @@
 import os
-import argparse
 
 from flask import Flask, request
 from flask_restful import Resource, Api
 from werkzeug.exceptions import HTTPException
 
-from uncloud.common.shared import shared
+from uncloud.settings import settings
+from uncloud.shared import shared
 
 app = Flask(__name__)
 api = Api(app)
 
 app.logger.handlers.clear()
 
-DEFAULT_PORT=1234
-
-arg_parser = argparse.ArgumentParser('metadata', add_help=False)
-arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT))
-
 
 @app.errorhandler(Exception)
 def handle_exception(e):
@@ -73,7 +68,7 @@ class Root(Resource):
             )
         else:
             etcd_key = os.path.join(
-                shared.settings["etcd"]["user_prefix"],
+                settings["etcd"]["user_prefix"],
                 data.value["owner_realm"],
                 data.value["owner"],
                 "key",
@@ -85,11 +80,40 @@ class Root(Resource):
             data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys
             return data.value["metadata"], 200
 
+    @staticmethod
+    def post():
+        return {"message": "Previous Implementation is deprecated."}
+        # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True)
+        # print(data)
+        # if data:
+        #     for k in request.json:
+        #         if k not in data.value:
+        #             data.value[k] = request.json[k]
+        #             if k.endswith("-list"):
+        #                 data.value[k] = [request.json[k]]
+        #         else:
+        #             if k.endswith("-list"):
+        #                 data.value[k].append(request.json[k])
+        #             else:
+        #                 data.value[k] = request.json[k]
+        #     etcd_client.put("/v1/metadata/{}".format(request.remote_addr),
+        #                     data.value, value_in_json=True)
+        # else:
+        #     data = {}
+        #     for k in request.json:
+        #         data[k] = request.json[k]
+        #         if k.endswith("-list"):
+        #             data[k] = [request.json[k]]
+        #     etcd_client.put("/v1/metadata/{}".format(request.remote_addr),
+        #                     data, value_in_json=True)
+
 
 api.add_resource(Root, "/")
 
 
-def main(arguments):
-    port = arguments['port']
-    debug = arguments['debug']
-    app.run(debug=debug, host="::", port=port)
+def main():
+    app.run(debug=True, host="::", port="80")
+
+
+if __name__ == "__main__":
+    main()

uncloud/oneshot/__init__.py

@@ -1,3 +0,0 @@
-import logging
-
-logger = logging.getLogger(__name__)

uncloud/oneshot/main.py

@@ -1,123 +0,0 @@
-import argparse
-import os
-
-
-from pathlib import Path
-from uncloud.vmm import VMM
-from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap
-
-from . import virtualmachine, logger
-
-###
-# Argument parser loaded by scripts/uncloud.
-arg_parser = argparse.ArgumentParser('oneshot', add_help=False)
-
-# Actions.
-arg_parser.add_argument('--list', action='store_true',
-        help='list UUID and name of running VMs')
-arg_parser.add_argument('--start', nargs=3,
-        metavar=('IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'),
-        help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix')
-arg_parser.add_argument('--stop', metavar='UUID',
-        help='stop a VM')
-arg_parser.add_argument('--get-status', metavar='UUID',
-        help='return the status of the VM')
-arg_parser.add_argument('--get-vnc', metavar='UUID',
-        help='return the path of the VNC socket of the VM')
-arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK',
-        help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix')
-
-# Arguments.
-arg_parser.add_argument('--workdir', default=Path.home(),
-        help='Working directory, defaulting to $HOME')
-arg_parser.add_argument('--mac',
-        help='MAC address of the VM to create (--start)')
-arg_parser.add_argument('--memory', type=int,
-        help='Memory (MB) to allocate (--start)')
-arg_parser.add_argument('--cores', type=int,
-        help='Number of cores to allocate (--start)')
-arg_parser.add_argument('--threads', type=int,
-        help='Number of threads to allocate (--start)')
-arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'],
-        help='Format of OS image (--start)')
-arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm',
-        help='QEMU acceleration to use (--start)')
-arg_parser.add_argument('--upstream-interface', default='eth0',
-        help='Name of upstream interface (--start)')
-
-###
-# Helpers.
-
-# XXX: check if it is possible to use the type returned by ETCD queries.
-class UncloudEntryWrapper:
-    def __init__(self, value):
-        self.value = value
-
-    def value(self):
-        return self.value
-
-def status_line(vm):
-     return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status())
-
-###
-# Entrypoint.
-
-def main(arguments):
-    # Initialize VMM.
-    workdir = arguments['workdir']
-    vmm = VMM(vmm_backend=workdir)
-
-    # Harcoded debug values.
-    net_id = 0
-
-    # Build VM configuration.
-    vm_config = {}
-    vm_options = [
-            'mac', 'memory', 'cores', 'threads', 'image', 'image_format',
-            '--upstream_interface', 'upstream_interface', 'network', 'accel'
-            ]
-    for option in vm_options:
-        if arguments.get(option):
-            vm_config[option] = arguments[option]
-
-    vm_config['net_id'] = net_id
-
-    # Execute requested VM action.
-    if arguments['reconfigure_radvd']:
-        # TODO: check that RADVD is available.
-        prefix = arguments['reconfigure_radvd']
-        network = UncloudEntryWrapper({
-                'id': net_id,
-                'ipv6': prefix
-                })
-
-        # Make use of uncloud.host.virtualmachine for network configuration.
-        update_radvd_conf([network])
-    elif arguments['start']:
-        # Extract from --start positional arguments. Quite fragile.
-        vm_config['image'] = arguments['start'][0]
-        vm_config['network'] = arguments['start'][1]
-        vm_config['upstream_interface'] = arguments['start'][2]
-
-        vm_config['tap_interface'] = "uc{}".format(len(vmm.discover()))
-        vm = virtualmachine.VM(vmm, vm_config)
-        vm.start()
-    elif arguments['stop']:
-        vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']})
-        vm = virtualmachine.VM(vmm, vm_config)
-        vm.stop()
-    elif arguments['get_status']:
-        vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']})
-        print(status_line(vm))
-    elif arguments['get_vnc']:
-        vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']})
-        print(vm.get_vnc_addr())
-    elif arguments['list']:
-        vms = vmm.discover()
-        print("Found {} VMs.".format(len(vms)))
-        for uuid in vms:
-            vm = virtualmachine.VM(vmm, {'uuid': uuid})
-            print(status_line(vm))
-    else:
-        print('Please specify an action: --start, --stop, --list,\
---get-status, --get-vnc, --reconfigure-radvd')

uncloud/oneshot/virtualmachine.py

@@ -1,81 +0,0 @@
-import uuid
-import os
-
-from uncloud.host.virtualmachine import create_vxlan_br_tap
-from uncloud.oneshot import logger
-
-class VM(object):
-    def __init__(self, vmm, config):
-        self.config = config
-        self.vmm = vmm
-
-        # Extract VM specs/metadata from configuration.
-        self.name         = config.get('name', 'no-name')
-        self.memory       = config.get('memory', 1024)
-        self.cores        = config.get('cores', 1)
-        self.threads      = config.get('threads', 1)
-        self.image_format = config.get('image_format', 'qcow2')
-        self.image        = config.get('image')
-        self.uuid         = config.get('uuid', str(uuid.uuid4()))
-        self.mac          = config.get('mac')
-        self.accel        = config.get('accel', 'kvm')
-
-        self.net_id             = config.get('net_id', 0)
-        self.upstream_interface = config.get('upstream_interface', 'eth0')
-        self.tap_interface      = config.get('tap_interface', 'uc0')
-        self.network            = config.get('network')
-
-    def get_qemu_args(self):
-        command = (
-            "-uuid {uuid} -name {name} -machine pc,accel={accel}"
-            " -drive file={image},format={image_format},if=virtio"
-            " -device virtio-rng-pci"
-            " -m {memory} -smp cores={cores},threads={threads}"
-            " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no"
-            " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}"
-        ).format(
-            uuid=self.uuid, name=self.name, accel=self.accel,
-            image=self.image, image_format=self.image_format,
-            memory=self.memory, cores=self.cores, threads=self.threads,
-            net_id=self.net_id, tap=self.tap_interface, mac=self.mac
-        )
-
-        return command.split(" ")
-
-    def start(self):
-        # Check that VM image is available.
-        if not os.path.isfile(self.image):
-            logger.error("Image {} does not exist. Aborting.".format(self.image))
-
-        # Create Bridge, VXLAN and tap interface for VM.
-        create_vxlan_br_tap(
-                self.net_id, self.upstream_interface, self.tap_interface, self.network
-                )
-
-        # Generate config for and run QEMU.
-        qemu_args = self.get_qemu_args()
-        logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args))
-        self.vmm.start(
-                uuid=self.uuid,
-                migration=False,
-                *qemu_args
-                )
-
-    def stop(self):
-        self.vmm.stop(self.uuid)
-
-    def get_status(self):
-        return self.vmm.get_status(self.uuid)
-
-    def get_vnc_addr(self):
-        return self.vmm.get_vnc(self.uuid)
-
-    def get_uuid(self):
-        return self.uuid
-
-    def get_name(self):
-        success, json = self.vmm.execute_command(uuid, 'query-name')
-        if success:
-            return json['return']['name']
-
-        return None

uncloud/scheduler/helper.py

@@ -6,7 +6,8 @@ import bitmath
 from uncloud.common.host import HostStatus
 from uncloud.common.request import RequestEntry, RequestType
 from uncloud.common.vm import VMStatus
-from uncloud.common.shared import shared
+from uncloud.shared import shared
+from uncloud.settings import settings
 
 
 def accumulated_specs(vms_specs):
@@ -129,7 +130,7 @@ def assign_host(vm):
         type=RequestType.StartVM,
         uuid=vm.uuid,
         hostname=vm.hostname,
-        request_prefix=shared.settings["etcd"]["request_prefix"],
+        request_prefix=settings["etcd"]["request_prefix"],
     )
     shared.request_pool.put(r)
 

uncloud/scheduler/main.py

@@ -4,48 +4,72 @@
 #  2. Introduce a status endpoint of the scheduler -
 #     maybe expose a prometheus compatible output
 
-import argparse
-
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.common.shared import shared
-from uncloud.scheduler import logger
-from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection,
-                                      assign_host, NoSuitableHostFound)
-
-arg_parser = argparse.ArgumentParser('scheduler', add_help=False)
+from uncloud.shared import shared
+from uncloud.settings import settings
+from .helper import (
+    dead_host_mitigation,
+    dead_host_detection,
+    assign_host,
+    NoSuitableHostFound,
+)
+from . import logger
 
 
-def main(arguments):
-    # The below while True is neccessary for gracefully handling leadership transfer and temporary
-    # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
-    # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
-    # which ends the loop immediately. So, having it inside infinite loop we try again and again to
-    # get prefix until either success or deamon death comes.
-    while True:
-        for request_iterator in [
-            shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
-                                          raise_exception=False),
-            shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
-                                            raise_exception=False),
-        ]:
-            for request_event in request_iterator:
-                dead_host_mitigation(dead_host_detection())
-                request_entry = RequestEntry(request_event)
+def main():
+    for request_iterator in [
+        shared.etcd_client.get_prefix(
+            settings["etcd"]["request_prefix"], value_in_json=True
+        ),
+        shared.etcd_client.watch_prefix(
+            settings["etcd"]["request_prefix"],
+            timeout=5,
+            value_in_json=True,
+        ),
+    ]:
+        for request_event in request_iterator:
+            request_entry = RequestEntry(request_event)
+            # Never Run time critical mechanism inside timeout
+            # mechanism because timeout mechanism only comes
+            # when no other event is happening. It means under
+            # heavy load there would not be a timeout event.
+            if request_entry.type == "TIMEOUT":
 
-                if request_entry.type == RequestType.ScheduleVM:
-                    logger.debug('%s, %s', request_entry.key, request_entry.value)
+                # Detect hosts that are dead and set their status
+                # to "DEAD", and their VMs' status to "KILLED"
+                dead_hosts = dead_host_detection()
+                if dead_hosts:
+                    logger.debug("Dead hosts: %s", dead_hosts)
+                    dead_host_mitigation(dead_hosts)
 
-                    vm_entry = shared.vm_pool.get(request_entry.uuid)
-                    if vm_entry is None:
-                        logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid))
-                        continue
+            elif request_entry.type == RequestType.ScheduleVM:
+                print(request_event.value)
+                logger.debug(
+                    "%s, %s", request_entry.key, request_entry.value
+                )
 
-                    shared.etcd_client.client.delete(request_entry.key)  # consume Request
+                vm_entry = shared.vm_pool.get(request_entry.uuid)
+                if vm_entry is None:
+                    logger.info(
+                        "Trying to act on {} but it is deleted".format(
+                            request_entry.uuid
+                        )
+                    )
+                    continue
+                shared.etcd_client.client.delete(
+                    request_entry.key
+                )  # consume Request
 
-                    try:
-                        assign_host(vm_entry)
-                    except NoSuitableHostFound:
-                        vm_entry.add_log('Can\'t schedule VM. No Resource Left.')
-                        shared.vm_pool.put(vm_entry)
+                try:
+                    assign_host(vm_entry)
+                except NoSuitableHostFound:
+                    vm_entry.add_log(
+                        "Can't schedule VM. No Resource Left."
+                    )
+                    shared.vm_pool.put(vm_entry)
 
-                        logger.info('No Resource Left. Emailing admin....')
+                    logger.info("No Resource Left. Emailing admin....")
+
+
+if __name__ == "__main__":
+    main()

uncloud/settings/__init__.py

@@ -0,0 +1,128 @@
+import configparser
+import logging
+import sys
+import os
+
+from uncloud.common.etcd_wrapper import Etcd3Wrapper
+
+logger = logging.getLogger(__name__)
+
+
+class CustomConfigParser(configparser.RawConfigParser):
+    def __getitem__(self, key):
+        try:
+            result = super().__getitem__(key)
+        except KeyError as err:
+            raise KeyError(
+                "Key '{}' not found in configuration. Make sure you configure uncloud.".format(
+                    key
+                )
+            ) from err
+        else:
+            return result
+
+
+class Settings(object):
+    def __init__(self, config_key="/uncloud/config/"):
+        conf_name = "uncloud.conf"
+        conf_dir = os.environ.get(
+            "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/")
+        )
+        self.config_file = os.path.join(conf_dir, conf_name)
+
+        self.config_parser = CustomConfigParser(allow_no_value=True)
+        self.config_key = config_key
+
+        self.read_internal_values()
+        try:
+            self.config_parser.read(self.config_file)
+        except Exception as err:
+            logger.error("%s", err)
+
+    def get_etcd_client(self):
+        args = tuple()
+        try:
+            kwargs = {
+                "host": self.config_parser.get("etcd", "url"),
+                "port": self.config_parser.get("etcd", "port"),
+                "ca_cert": self.config_parser.get("etcd", "ca_cert"),
+                "cert_cert": self.config_parser.get(
+                    "etcd", "cert_cert"
+                ),
+                "cert_key": self.config_parser.get("etcd", "cert_key"),
+            }
+        except configparser.Error as err:
+            raise configparser.Error(
+                "{} in config file {}".format(
+                    err.message, self.config_file
+                )
+            ) from err
+        else:
+            try:
+                wrapper = Etcd3Wrapper(*args, **kwargs)
+            except Exception as err:
+                logger.error(
+                    "etcd connection not successfull. Please check your config file."
+                    "\nDetails: %s\netcd connection parameters: %s",
+                    err,
+                    kwargs,
+                )
+                sys.exit(1)
+            else:
+                return wrapper
+
+    def read_internal_values(self):
+        self.config_parser.read_dict(
+            {
+                "etcd": {
+                    "file_prefix": "/files/",
+                    "host_prefix": "/hosts/",
+                    "image_prefix": "/images/",
+                    "image_store_prefix": "/imagestore/",
+                    "network_prefix": "/networks/",
+                    "request_prefix": "/requests/",
+                    "user_prefix": "/users/",
+                    "vm_prefix": "/vms/",
+                }
+            }
+        )
+
+    def read_config_file_values(self, config_file):
+        try:
+            # Trying to read configuration file
+            with open(config_file, "r") as config_file_handle:
+                self.config_parser.read_file(config_file_handle)
+        except FileNotFoundError:
+            sys.exit(
+                "Configuration file {} not found!".format(config_file)
+            )
+        except Exception as err:
+            logger.exception(err)
+            sys.exit("Error occurred while reading configuration file")
+
+    def read_values_from_etcd(self):
+        etcd_client = self.get_etcd_client()
+        config_from_etcd = etcd_client.get(
+            self.config_key, value_in_json=True
+        )
+        if config_from_etcd:
+            self.config_parser.read_dict(config_from_etcd.value)
+        else:
+            raise KeyError(
+                "Key '{}' not found in etcd. Please configure uncloud.".format(
+                    self.config_key
+                )
+            )
+
+    def __getitem__(self, key):
+        # Allow failing to read from etcd if we have
+        # it locally
+        try:
+            self.read_values_from_etcd()
+        except KeyError as e:
+            pass
+
+        return self.config_parser[key]
+
+
+settings = Settings()

uncloud/shared/__init__.py

@@ -0,0 +1,34 @@
+from uncloud.settings import settings
+from uncloud.common.vm import VmPool
+from uncloud.common.host import HostPool
+from uncloud.common.request import RequestPool
+from uncloud.common.storage_handlers import get_storage_handler
+
+
+class Shared:
+    @property
+    def etcd_client(self):
+        return settings.get_etcd_client()
+
+    @property
+    def host_pool(self):
+        return HostPool(
+            self.etcd_client, settings["etcd"]["host_prefix"]
+        )
+
+    @property
+    def vm_pool(self):
+        return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"])
+
+    @property
+    def request_pool(self):
+        return RequestPool(
+            self.etcd_client, settings["etcd"]["request_prefix"]
+        )
+
+    @property
+    def storage_handler(self):
+        return get_storage_handler()
+
+
+shared = Shared()

uncloud/vmm/__init__.py

@@ -100,9 +100,9 @@ class TransferVM(Process):
 class VMM:
     # Virtual Machine Manager
     def __init__(
-            self,
-            qemu_path="/usr/bin/qemu-system-x86_64",
-            vmm_backend=os.path.expanduser("~/uncloud/vmm/"),
+        self,
+        qemu_path="/usr/bin/qemu-system-x86_64",
+        vmm_backend=os.path.expanduser("~/uncloud/vmm/"),
     ):
         self.qemu_path = qemu_path
         self.vmm_backend = vmm_backend
@@ -125,7 +125,7 @@ class VMM:
             os.makedirs(self.socket_dir, exist_ok=True)
 
     def is_running(self, uuid):
-        sock_path = os.path.join(self.socket_dir, uuid)
+        sock_path = os.path.join(self.vmm_backend, uuid)
         try:
             sock = socket.socket(socket.AF_UNIX)
             sock.connect(sock_path)
@@ -163,7 +163,7 @@ class VMM:
             qmp_arg = (
                 "-qmp",
                 "unix:{},server,nowait".format(
-                    join_path(self.socket_dir, uuid)
+                    join_path(self.vmm_backend, uuid)
                 ),
             )
             vnc_arg = (
@@ -190,10 +190,18 @@ class VMM:
                     err.stderr.decode("utf-8"),
                 )
             else:
-                sp.check_output(
-                    ["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory",
-                     "chmod", "-R", "o=rwx,g=rwx", self.vmm_backend]
-                )
+                with suppress(sp.CalledProcessError):
+                    sp.check_output(
+                        [
+                            "sudo",
+                            "-p",
+                            "Enter password to correct permission for uncloud-vmm's directory",
+                            "chmod",
+                            "-R",
+                            "o=rwx,g=rwx",
+                            self.vmm_backend,
+                        ]
+                    )
 
                 # TODO: Find some good way to check whether the virtual machine is up and
                 #       running without relying on non-guarenteed ways.
@@ -212,7 +220,7 @@ class VMM:
     def execute_command(self, uuid, command, **kwargs):
         # execute_command -> sucess?, output
         try:
-            with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as (
+            with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as (
                 sock_handle,
                 file_handle,
             ):
@@ -255,8 +263,8 @@ class VMM:
     def discover(self):
         vms = [
             uuid
-            for uuid in os.listdir(self.socket_dir)
-            if not isdir(join_path(self.socket_dir, uuid))
+            for uuid in os.listdir(self.vmm_backend)
+            if not isdir(join_path(self.vmm_backend, uuid))
         ]
         return vms