Fix --accel parameter for oneshot

Totally not related to my previous commit

.gitignore

@@ -16,3 +16,5 @@ uncloud/version.py
 build/
 venv/
 dist/
+
+*.iso

bin/gen-version

@@ -1,22 +1,22 @@
 #!/bin/sh
 # -*- coding: utf-8 -*-
 #
-# 2019 Nico Schottelius (nico-ucloud at schottelius.org)
+# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org)
 #
-# This file is part of ucloud.
+# This file is part of uncloud.
 #
-# ucloud is free software: you can redistribute it and/or modify
+# uncloud is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# ucloud is distributed in the hope that it will be useful,
+# uncloud is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with ucloud. If not, see <http://www.gnu.org/licenses/>.
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
 #
 #
 
@@ -26,4 +26,4 @@ dir=${0%/*}
 
 # Ensure version is present - the bundled/shipped version contains a static version,
 # the git version contains a dynamic version
-printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py
+printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py

bin/uncloud-run-reinstall

@@ -24,6 +24,6 @@
 dir=${0%/*}
 
 ${dir}/gen-version;
-pip uninstall -y uncloud
+pip uninstall -y uncloud >/dev/null
-python setup.py install
+python setup.py install  >/dev/null
 ${dir}/uncloud "$@"

conf/uncloud.conf

@@ -1,7 +1,13 @@
 [etcd]
 url = localhost
 port = 2379
-
+base_prefix = /
 ca_cert
 cert_cert
 cert_key
+
+[client]
+name = replace_me
+realm = replace_me
+seed = replace_me
+api_server = http://localhost:5000

docs/Makefile

@@ -7,7 +7,7 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source/
 BUILDDIR      = build/
-DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/
+DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/
 
 .PHONY: all build clean
 

docs/README.md

@@ -0,0 +1,12 @@
+# uncloud docs
+
+## Requirements
+1. Python3
+2. Sphinx
+
+## Usage
+Run `make build` to build docs.
+
+Run `make clean` to remove build directory.
+
+Run `make publish` to push build dir to https://ungleich.ch/ucloud/

docs/source/admin-guide.rst

@@ -56,40 +56,13 @@ To start host we created earlier, execute the following command
 
     ucloud host ungleich.ch
 
-Create OS Image
+File & image scanners
----------------
+--------------------------
 
-Create ucloud-init ready OS image (Optional)
+Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by
-This step is optional if you just want to test ucloud. However, sooner or later
+ucloud. We can only make images from tracked files. So, we need to track the
-you want to create OS images with ucloud-init to properly
+file by running File Scanner
-contexualize VMs.
-
-1. Start a VM with OS image on which you want to install ucloud-init
-2. Execute the following command on the started VM
-
-   .. code-block:: sh
-
-    apk add git
-    git clone https://code.ungleich.ch/ucloud/ucloud-init.git
-    cd ucloud-init
-    sh ./install.sh
-3. Congratulations. Your image is now ucloud-init ready.
-
-
-Upload Sample OS Image
-~~~~~~~~~~~~~~~~~~~~~~
-Execute the following to get the sample OS image file.
-
-.. code-block:: sh
-
-    mkdir /var/www/admin
-    (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download)
-
-Run File Scanner and Image Scanner
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make
-images from tracked files. So, we need to track the file by running File Scanner
 
 .. code-block:: sh
 

docs/source/hacking.rst

@@ -0,0 +1,36 @@
+Hacking
+=======
+Using uncloud in hacking (aka development) mode.
+
+
+Get the code
+------------
+.. code-block:: sh
+    :linenos:
+
+     git clone https://code.ungleich.ch/uncloud/uncloud.git
+
+
+
+Install python requirements
+---------------------------
+You need to have python3 installed.
+
+.. code-block:: sh
+    :linenos:
+
+    cd uncloud!
+    python -m venv venv
+    . ./venv/bin/activate
+    ./bin/uncloud-run-reinstall
+
+
+
+Install os requirements
+-----------------------
+Install the following software packages: **dnsmasq**.
+
+If you already have a working IPv6 SLAAC and DNS setup,
+this step can be skipped.
+
+Note that you need at least one /64 IPv6 network to run uncloud.

docs/source/index.rst

@@ -11,14 +11,13 @@ Welcome to ucloud's documentation!
    :caption: Contents:
 
    introduction
-   user-guide
    setup-install
+   vm-images
+   user-guide
    admin-guide
-   user-guide/how-to-create-an-os-image-for-ucloud
    troubleshooting
    hacking
 
-
 Indices and tables
 ==================
 

docs/source/vm-images.rst

@@ -0,0 +1,66 @@
+VM images
+==================================
+
+Overview
+---------
+
+ucloud tries to be least invasise towards VMs and only require
+strictly necessary changes for running in a virtualised
+environment. This includes configurations for:
+
+* Configuring the network
+* Managing access via ssh keys
+* Resizing the attached disk(s)
+
+Upstream images
+---------------
+
+The 'official' uncloud images are defined in the `uncloud/images
+<https://code.ungleich.ch/uncloud/images>`_ repository.
+
+How to make you own Uncloud images
+----------------------------------
+
+.. note::
+  It is fairly easy to create your own images for uncloud, as the common
+  operations (which are detailed below) can be automatically handled by the
+  `uncloud/uncloud-init <https://code.ungleich.ch/uncloud/uncloud-init>`_ tool.
+
+Network configuration
+~~~~~~~~~~~~~~~~~~~~~
+All VMs in ucloud are required to support IPv6. The primary network
+configuration is always done using SLAAC. A VM thus needs only to be
+configured to
+
+* accept router advertisements on all network interfaces
+* use the router advertisements to configure the network interfaces
+* accept the DNS entries from the router advertisements
+
+
+Configuring SSH keys
+~~~~~~~~~~~~~~~~~~~~
+
+To be able to access the VM, ucloud support provisioning SSH keys.
+
+To accept ssh keys in your VM, request the URL
+*http://metadata/ssh_keys*. Add the content to the appropriate user's
+**authorized_keys** file. Below you find sample code to accomplish
+this task:
+
+.. code-block:: sh
+
+    tmp=$(mktemp)
+    curl -s http://metadata/ssk_keys > "$tmp"
+    touch ~/.ssh/authorized_keys # ensure it exists
+    cat ~/.ssh/authorized_keys >> "$tmp"
+    sort "$tmp" | uniq > ~/.ssh/authorized_keys
+
+
+Disk resize
+~~~~~~~~~~~
+In virtualised environments, the disk sizes might grow. The operating
+system should detect disks that are bigger than the existing partition
+table and resize accordingly. This task is os specific.
+
+ucloud does not support shrinking disks due to the complexity and
+intra OS dependencies.

scripts/uncloud

@@ -1,70 +1,88 @@
 #!/usr/bin/env python3
-import argparse
 import logging
-import importlib
-import multiprocessing as mp
 import sys
+import importlib
+import argparse
+import os
 
-from logging.handlers import SysLogHandler
+from etcd3.exceptions import ConnectionFailedError
-from uncloud.configure.main import configure_parser
 
+from uncloud.common import settings
 from uncloud import UncloudException
+from uncloud.common.cli import resolve_otp_credentials
 
-def exception_hook(exc_type, exc_value, exc_traceback):
+# Components that use etcd
-    logging.getLogger(__name__).error(
+ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner',
-        'Uncaught exception',
+                   'imagescanner', 'metadata', 'configure', 'hack']
-        exc_info=(exc_type, exc_value, exc_traceback)
-    )
 
-
+ALL_COMPONENTS = ETCD_COMPONENTS.copy()
-sys.excepthook = exception_hook
+ALL_COMPONENTS.append('oneshot')
+#ALL_COMPONENTS.append('cli')
 
 
 if __name__ == '__main__':
-    # Setting up root logger
+    arg_parser = argparse.ArgumentParser()
-    logger = logging.getLogger()
+    subparsers = arg_parser.add_subparsers(dest='command')
-
-    logger.setLevel(logging.DEBUG)
 
     parent_parser = argparse.ArgumentParser(add_help=False)
-    parent_parser.add_argument("--debug", "-d", action='store_true')
+    parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
+                               help='More verbose logging')
+    parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
+                               default=os.path.expanduser('~/uncloud'))
 
-    arg_parser = argparse.ArgumentParser()
+    etcd_parser = argparse.ArgumentParser(add_help=False)
+    etcd_parser.add_argument('--etcd-host')
+    etcd_parser.add_argument('--etcd-port')
+    etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
+    etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
+    etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
 
-    subparsers = arg_parser.add_subparsers(dest="command")
+    for component in ALL_COMPONENTS:
+        mod = importlib.import_module('uncloud.{}.main'.format(component))
+        parser = getattr(mod, 'arg_parser')
 
-    api_parser = subparsers.add_parser("api", parents=[parent_parser])
+        if component in ETCD_COMPONENTS:
-    api_parser.add_argument("--port", "-p")
+            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser])
+        else:
+            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
 
-    host_parser = subparsers.add_parser("host")
+    arguments = vars(arg_parser.parse_args())
-    host_parser.add_argument("--hostname", required=True)
+    etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
-
+    etcd_arguments = {
-    scheduler_parser = subparsers.add_parser("scheduler")
+        'etcd': {
-    filescanner_parser = subparsers.add_parser("filescanner")
+            key.replace('etcd_', ''): arguments[key]
-    imagescanner_parser = subparsers.add_parser("imagescanner")
+            for key in etcd_arguments
-    metadata_parser = subparsers.add_parser("metadata")
+        }
-    config_parser = subparsers.add_parser("configure")
+    }
-
+    if not arguments['command']:
-    configure_parser(config_parser)
-    args = arg_parser.parse_args()
-
-    if not args.command:
         arg_parser.print_help()
     else:
+        # Initializing Settings and resolving otp_credentials
+        # It is neccessary to resolve_otp_credentials after argument parsing is done because
+        # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
+        # providing the default values for --name, --realm and --seed arguments from the values
+        # we read from file. But, now we are asking user about where the config file lives. So,
+        # to providing default value is not possible before parsing arguments. So, we are doing
+        # it after..
+#        settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
+#        resolve_otp_credentials(arguments)
 
-        # if we start etcd in seperate process with default settings
+        name = arguments.pop('command')
-        # i.e inheriting few things from parent process etcd3 module
+        mod = importlib.import_module('uncloud.{}.main'.format(name))
-        # errors out, so the following command configure multiprocessing
+        main = getattr(mod, 'main')
-        # module to not inherit anything from parent.
+
-        mp.set_start_method('spawn')
+        if arguments['debug']:
+            logging.basicConfig(level=logging.DEBUG)
+        else:
+            logging.basicConfig(level=logging.INFO)
+
+        log = logging.getLogger()
 
-        arguments = vars(args)
         try:
-            name = arguments.pop('command')
+            main(arguments)
-            mod = importlib.import_module("uncloud.{}.main".format(name))
-            main = getattr(mod, "main")
-            main(**arguments)
         except UncloudException as err:
-            logger.error(err)
+            log.error(err)
+#        except ConnectionFailedError as err:
+#            log.error('Cannot connect to etcd: {}'.format(err))
         except Exception as err:
-            logger.exception(err)
+            log.exception(err)

setup.py

@@ -40,7 +40,7 @@ setup(
         "pynetbox",
         "colorama",
         "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3",
-        "marshmallow",
+        "marshmallow"
     ],
     scripts=["scripts/uncloud"],
     data_files=[

uncloud/api/common_fields.py

@@ -1,7 +1,6 @@
 import os
 
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
 
 
 class Optional:
@@ -54,9 +53,7 @@ class VmUUIDField(Field):
 
     def vm_uuid_validation(self):
         r = shared.etcd_client.get(
-            os.path.join(settings["etcd"]["vm_prefix"], self.uuid)
+            os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
         )
         if not r:
-            self.add_error(
+            self.add_error("VM with uuid {} does not exists".format(self.uuid))
-                "VM with uuid {} does not exists".format(self.uuid)
-            )

uncloud/api/create_image_store.py

@@ -3,18 +3,17 @@ import os
 
 from uuid import uuid4
 
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
 
 data = {
-    "is_public": True,
+    'is_public': True,
-    "type": "ceph",
+    'type': 'ceph',
-    "name": "images",
+    'name': 'images',
-    "description": "first ever public image-store",
+    'description': 'first ever public image-store',
-    "attributes": {"list": [], "key": [], "pool": "images"},
+    'attributes': {'list': [], 'key': [], 'pool': 'images'},
 }
 
 shared.etcd_client.put(
-    os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex),
+    os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
     json.dumps(data),
 )

uncloud/api/helper.py

@@ -1,14 +1,12 @@
 import binascii
 import ipaddress
 import random
-import subprocess as sp
 import logging
 import requests
 
 from pyotp import TOTP
 
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
 
 logger = logging.getLogger(__name__)
 
@@ -16,9 +14,9 @@ logger = logging.getLogger(__name__)
 def check_otp(name, realm, token):
     try:
         data = {
-            "auth_name": settings["otp"]["auth_name"],
+            "auth_name": shared.settings["otp"]["auth_name"],
-            "auth_token": TOTP(settings["otp"]["auth_seed"]).now(),
+            "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
-            "auth_realm": settings["otp"]["auth_realm"],
+            "auth_realm": shared.settings["otp"]["auth_realm"],
             "name": name,
             "realm": realm,
             "token": token,
@@ -26,13 +24,13 @@ def check_otp(name, realm, token):
     except binascii.Error as err:
         logger.error(
             "Cannot compute OTP for seed: {}".format(
-                settings["otp"]["auth_seed"]
+                shared.settings["otp"]["auth_seed"]
             )
         )
         return 400
 
     response = requests.post(
-        settings["otp"]["verification_controller_url"], json=data
+        shared.settings["otp"]["verification_controller_url"], json=data
     )
     return response.status_code
 
@@ -88,7 +86,7 @@ def resolve_image_name(name, etcd_client):
         )
 
     images = etcd_client.get_prefix(
-        settings["etcd"]["image_prefix"], value_in_json=True
+        shared.settings["etcd"]["image_prefix"], value_in_json=True
     )
 
     # Try to find image with name == image_name and store_name == store_name
@@ -112,9 +110,7 @@ def random_bytes(num=6):
     return [random.randrange(256) for _ in range(num)]
 
 
-def generate_mac(
+def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
-    uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
-):
     mac = random_bytes()
     if oui:
         if type(oui) == str:
@@ -149,3 +145,4 @@ def mac2ipv6(mac, prefix):
     lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
     prefix = ipaddress.IPv6Address(prefix)
     return str(prefix + int(lower_part))
+

uncloud/api/main.py

@@ -1,6 +1,7 @@
 import json
 import pynetbox
 import logging
+import argparse
 
 from uuid import uuid4
 from os.path import join as join_path
@@ -9,14 +10,13 @@ from flask import Flask, request
 from flask_restful import Resource, Api
 from werkzeug.exceptions import HTTPException
 
+from uncloud.common.shared import shared
+
 from uncloud.common import counters
 from uncloud.common.vm import VMStatus
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.settings import settings
+from uncloud.api import schemas
-from uncloud.shared import shared
+from uncloud.api.helper import generate_mac, mac2ipv6
-
-from . import schemas
-from .helper import generate_mac, mac2ipv6
 from uncloud import UncloudException
 
 logger = logging.getLogger(__name__)
@@ -25,6 +25,9 @@ app = Flask(__name__)
 api = Api(app)
 app.logger.handlers.clear()
 
+arg_parser = argparse.ArgumentParser('api', add_help=False)
+arg_parser.add_argument('--port', '-p')
+
 
 @app.errorhandler(Exception)
 def handle_exception(e):
@@ -34,7 +37,7 @@ def handle_exception(e):
         return e
 
     # now you're handling non-HTTP exceptions only
-    return {"message": "Server Error"}, 500
+    return {'message': 'Server Error'}, 500
 
 
 class CreateVM(Resource):
@@ -46,33 +49,33 @@ class CreateVM(Resource):
         validator = schemas.CreateVMSchema(data)
         if validator.is_valid():
             vm_uuid = uuid4().hex
-            vm_key = join_path(settings["etcd"]["vm_prefix"], vm_uuid)
+            vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
             specs = {
-                "cpu": validator.specs["cpu"],
+                'cpu': validator.specs['cpu'],
-                "ram": validator.specs["ram"],
+                'ram': validator.specs['ram'],
-                "os-ssd": validator.specs["os-ssd"],
+                'os-ssd': validator.specs['os-ssd'],
-                "hdd": validator.specs["hdd"],
+                'hdd': validator.specs['hdd'],
             }
-            macs = [generate_mac() for _ in range(len(data["network"]))]
+            macs = [generate_mac() for _ in range(len(data['network']))]
             tap_ids = [
                 counters.increment_etcd_counter(
-                    shared.etcd_client, "/v1/counter/tap"
+                    shared.etcd_client, shared.settings['etcd']['tap_counter']
                 )
-                for _ in range(len(data["network"]))
+                for _ in range(len(data['network']))
             ]
             vm_entry = {
-                "name": data["vm_name"],
+                'name': data['vm_name'],
-                "owner": data["name"],
+                'owner': data['name'],
-                "owner_realm": data["realm"],
+                'owner_realm': data['realm'],
-                "specs": specs,
+                'specs': specs,
-                "hostname": "",
+                'hostname': '',
-                "status": VMStatus.stopped,
+                'status': VMStatus.stopped,
-                "image_uuid": validator.image_uuid,
+                'image_uuid': validator.image_uuid,
-                "log": [],
+                'log': [],
-                "vnc_socket": "",
+                'vnc_socket': '',
-                "network": list(zip(data["network"], macs, tap_ids)),
+                'network': list(zip(data['network'], macs, tap_ids)),
-                "metadata": {"ssh-keys": []},
+                'metadata': {'ssh-keys': []},
-                "in_migration": False,
+                'in_migration': False,
             }
             shared.etcd_client.put(vm_key, vm_entry, value_in_json=True)
 
@@ -80,39 +83,39 @@ class CreateVM(Resource):
             r = RequestEntry.from_scratch(
                 type=RequestType.ScheduleVM,
                 uuid=vm_uuid,
-                request_prefix=settings["etcd"]["request_prefix"],
+                request_prefix=shared.settings['etcd']['request_prefix'],
             )
             shared.request_pool.put(r)
 
-            return {"message": "VM Creation Queued"}, 200
+            return {'message': 'VM Creation Queued'}, 200
         return validator.get_errors(), 400
 
 
 class VmStatus(Resource):
     @staticmethod
-    def get():
+    def post():
         data = request.json
         validator = schemas.VMStatusSchema(data)
         if validator.is_valid():
             vm = shared.vm_pool.get(
-                join_path(settings["etcd"]["vm_prefix"], data["uuid"])
+                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
             )
             vm_value = vm.value.copy()
-            vm_value["ip"] = []
+            vm_value['ip'] = []
             for network_mac_and_tap in vm.network:
                 network_name, mac, tap = network_mac_and_tap
                 network = shared.etcd_client.get(
                     join_path(
-                        settings["etcd"]["network_prefix"],
+                        shared.settings['etcd']['network_prefix'],
-                        data["name"],
+                        data['name'],
                         network_name,
                     ),
                     value_in_json=True,
                 )
                 ipv6_addr = (
-                    network.value.get("ipv6").split("::")[0] + "::"
+                    network.value.get('ipv6').split('::')[0] + '::'
                 )
-                vm_value["ip"].append(mac2ipv6(mac, ipv6_addr))
+                vm_value['ip'].append(mac2ipv6(mac, ipv6_addr))
             vm.value = vm_value
             return vm.value
         else:
@@ -126,26 +129,26 @@ class CreateImage(Resource):
         validator = schemas.CreateImageSchema(data)
         if validator.is_valid():
             file_entry = shared.etcd_client.get(
-                join_path(settings["etcd"]["file_prefix"], data["uuid"])
+                join_path(shared.settings['etcd']['file_prefix'], data['uuid'])
             )
             file_entry_value = json.loads(file_entry.value)
 
             image_entry_json = {
-                "status": "TO_BE_CREATED",
+                'status': 'TO_BE_CREATED',
-                "owner": file_entry_value["owner"],
+                'owner': file_entry_value['owner'],
-                "filename": file_entry_value["filename"],
+                'filename': file_entry_value['filename'],
-                "name": data["name"],
+                'name': data['name'],
-                "store_name": data["image_store"],
+                'store_name': data['image_store'],
-                "visibility": "public",
+                'visibility': 'public',
             }
             shared.etcd_client.put(
                 join_path(
-                    settings["etcd"]["image_prefix"], data["uuid"]
+                    shared.settings['etcd']['image_prefix'], data['uuid']
                 ),
                 json.dumps(image_entry_json),
             )
 
-            return {"message": "Image queued for creation."}
+            return {'message': 'Image queued for creation.'}
         return validator.get_errors(), 400
 
 
@@ -153,15 +156,15 @@ class ListPublicImages(Resource):
     @staticmethod
     def get():
         images = shared.etcd_client.get_prefix(
-            settings["etcd"]["image_prefix"], value_in_json=True
+            shared.settings['etcd']['image_prefix'], value_in_json=True
         )
-        r = {"images": []}
+        r = {'images': []}
         for image in images:
-            image_key = "{}:{}".format(
+            image_key = '{}:{}'.format(
-                image.value["store_name"], image.value["name"]
+                image.value['store_name'], image.value['name']
             )
-            r["images"].append(
+            r['images'].append(
-                {"name": image_key, "status": image.value["status"]}
+                {'name': image_key, 'status': image.value['status']}
             )
         return r, 200
 
@@ -174,14 +177,14 @@ class VMAction(Resource):
 
         if validator.is_valid():
             vm_entry = shared.vm_pool.get(
-                join_path(settings["etcd"]["vm_prefix"], data["uuid"])
+                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
             )
-            action = data["action"]
+            action = data['action']
 
-            if action == "start":
+            if action == 'start':
-                action = "schedule"
+                action = 'schedule'
 
-            if action == "delete" and vm_entry.hostname == "":
+            if action == 'delete' and vm_entry.hostname == '':
                 if shared.storage_handler.is_vm_image_exists(
                     vm_entry.uuid
                 ):
@@ -190,25 +193,25 @@ class VMAction(Resource):
                     )
                     if r_status:
                         shared.etcd_client.client.delete(vm_entry.key)
-                        return {"message": "VM successfully deleted"}
+                        return {'message': 'VM successfully deleted'}
                     else:
                         logger.error(
-                            "Some Error Occurred while deleting VM"
+                            'Some Error Occurred while deleting VM'
                         )
-                        return {"message": "VM deletion unsuccessfull"}
+                        return {'message': 'VM deletion unsuccessfull'}
                 else:
                     shared.etcd_client.client.delete(vm_entry.key)
-                    return {"message": "VM successfully deleted"}
+                    return {'message': 'VM successfully deleted'}
 
             r = RequestEntry.from_scratch(
-                type="{}VM".format(action.title()),
+                type='{}VM'.format(action.title()),
-                uuid=data["uuid"],
+                uuid=data['uuid'],
                 hostname=vm_entry.hostname,
-                request_prefix=settings["etcd"]["request_prefix"],
+                request_prefix=shared.settings['etcd']['request_prefix'],
             )
             shared.request_pool.put(r)
             return (
-                {"message": "VM {} Queued".format(action.title())},
+                {'message': 'VM {} Queued'.format(action.title())},
                 200,
             )
         else:
@@ -222,20 +225,20 @@ class VMMigration(Resource):
         validator = schemas.VmMigrationSchema(data)
 
         if validator.is_valid():
-            vm = shared.vm_pool.get(data["uuid"])
+            vm = shared.vm_pool.get(data['uuid'])
             r = RequestEntry.from_scratch(
                 type=RequestType.InitVMMigration,
                 uuid=vm.uuid,
                 hostname=join_path(
-                    settings["etcd"]["host_prefix"],
+                    shared.settings['etcd']['host_prefix'],
                     validator.destination.value,
                 ),
-                request_prefix=settings["etcd"]["request_prefix"],
+                request_prefix=shared.settings['etcd']['request_prefix'],
             )
 
             shared.request_pool.put(r)
             return (
-                {"message": "VM Migration Initialization Queued"},
+                {'message': 'VM Migration Initialization Queued'},
                 200,
             )
         else:
@@ -244,32 +247,32 @@ class VMMigration(Resource):
 
 class ListUserVM(Resource):
     @staticmethod
-    def get():
+    def post():
         data = request.json
         validator = schemas.OTPSchema(data)
 
         if validator.is_valid():
             vms = shared.etcd_client.get_prefix(
-                settings["etcd"]["vm_prefix"], value_in_json=True
+                shared.settings['etcd']['vm_prefix'], value_in_json=True
             )
             return_vms = []
             user_vms = filter(
-                lambda v: v.value["owner"] == data["name"], vms
+                lambda v: v.value['owner'] == data['name'], vms
             )
             for vm in user_vms:
                 return_vms.append(
                     {
-                        "name": vm.value["name"],
+                        'name': vm.value['name'],
-                        "vm_uuid": vm.key.split("/")[-1],
+                        'vm_uuid': vm.key.split('/')[-1],
-                        "specs": vm.value["specs"],
+                        'specs': vm.value['specs'],
-                        "status": vm.value["status"],
+                        'status': vm.value['status'],
-                        "hostname": vm.value["hostname"],
+                        'hostname': vm.value['hostname'],
-                        "vnc_socket": vm.value.get("vnc_socket", None),
+                        'vnc_socket': vm.value.get('vnc_socket', None),
                     }
                 )
             if return_vms:
-                return {"message": return_vms}, 200
+                return {'message': return_vms}, 200
-            return {"message": "No VM found"}, 404
+            return {'message': 'No VM found'}, 404
 
         else:
             return validator.get_errors(), 400
@@ -277,28 +280,26 @@ class ListUserVM(Resource):
 
 class ListUserFiles(Resource):
     @staticmethod
-    def get():
+    def post():
         data = request.json
         validator = schemas.OTPSchema(data)
 
         if validator.is_valid():
             files = shared.etcd_client.get_prefix(
-                settings["etcd"]["file_prefix"], value_in_json=True
+                shared.settings['etcd']['file_prefix'], value_in_json=True
             )
             return_files = []
-            user_files = list(
+            user_files = [f for f in files if f.value['owner'] == data['name']]
-                filter(
-                    lambda f: f.value["owner"] == data["name"], files
-                )
-            )
             for file in user_files:
-                return_files.append(
+                file_uuid = file.key.split('/')[-1]
-                    {
+                file = file.value
-                        "filename": file.value["filename"],
+                file['uuid'] = file_uuid
-                        "uuid": file.key.split("/")[-1],
+
-                    }
+                file.pop('sha512sum', None)
-                )
+                file.pop('owner', None)
-            return {"message": return_files}, 200
+
+                return_files.append(file)
+            return {'message': return_files}, 200
         else:
             return validator.get_errors(), 400
 
@@ -310,19 +311,19 @@ class CreateHost(Resource):
         validator = schemas.CreateHostSchema(data)
         if validator.is_valid():
             host_key = join_path(
-                settings["etcd"]["host_prefix"], uuid4().hex
+                shared.settings['etcd']['host_prefix'], uuid4().hex
             )
             host_entry = {
-                "specs": data["specs"],
+                'specs': data['specs'],
-                "hostname": data["hostname"],
+                'hostname': data['hostname'],
-                "status": "DEAD",
+                'status': 'DEAD',
-                "last_heartbeat": "",
+                'last_heartbeat': '',
             }
             shared.etcd_client.put(
                 host_key, host_entry, value_in_json=True
             )
 
-            return {"message": "Host Created"}, 200
+            return {'message': 'Host Created'}, 200
 
         return validator.get_errors(), 400
 
@@ -333,9 +334,9 @@ class ListHost(Resource):
         hosts = shared.host_pool.hosts
         r = {
             host.key: {
-                "status": host.status,
+                'status': host.status,
-                "specs": host.specs,
+                'specs': host.specs,
-                "hostname": host.hostname,
+                'hostname': host.hostname,
             }
             for host in hosts
         }
@@ -344,7 +345,7 @@ class ListHost(Resource):
 
 class GetSSHKeys(Resource):
     @staticmethod
-    def get():
+    def post():
         data = request.json
         validator = schemas.GetSSHSchema(data)
         if validator.is_valid():
@@ -352,29 +353,29 @@ class GetSSHKeys(Resource):
 
                 # {user_prefix}/{realm}/{name}/key/
                 etcd_key = join_path(
-                    settings["etcd"]["user_prefix"],
+                    shared.settings['etcd']['user_prefix'],
-                    data["realm"],
+                    data['realm'],
-                    data["name"],
+                    data['name'],
-                    "key",
+                    'key',
                 )
                 etcd_entry = shared.etcd_client.get_prefix(
                     etcd_key, value_in_json=True
                 )
 
                 keys = {
-                    key.key.split("/")[-1]: key.value
+                    key.key.split('/')[-1]: key.value
                     for key in etcd_entry
                 }
-                return {"keys": keys}
+                return {'keys': keys}
             else:
 
                 # {user_prefix}/{realm}/{name}/key/{key_name}
                 etcd_key = join_path(
-                    settings["etcd"]["user_prefix"],
+                    shared.settings['etcd']['user_prefix'],
-                    data["realm"],
+                    data['realm'],
-                    data["name"],
+                    data['name'],
-                    "key",
+                    'key',
-                    data["key_name"],
+                    data['key_name'],
                 )
                 etcd_entry = shared.etcd_client.get(
                     etcd_key, value_in_json=True
@@ -382,14 +383,14 @@ class GetSSHKeys(Resource):
 
                 if etcd_entry:
                     return {
-                        "keys": {
+                        'keys': {
-                            etcd_entry.key.split("/")[
+                            etcd_entry.key.split('/')[
                                 -1
                             ]: etcd_entry.value
                         }
                     }
                 else:
-                    return {"keys": {}}
+                    return {'keys': {}}
         else:
             return validator.get_errors(), 400
 
@@ -403,56 +404,56 @@ class AddSSHKey(Resource):
 
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                settings["etcd"]["user_prefix"],
+                shared.settings['etcd']['user_prefix'],
-                data["realm"],
+                data['realm'],
-                data["name"],
+                data['name'],
-                "key",
+                'key',
-                data["key_name"],
+                data['key_name'],
             )
             etcd_entry = shared.etcd_client.get(
                 etcd_key, value_in_json=True
             )
             if etcd_entry:
                 return {
-                    "message": "Key with name '{}' already exists".format(
+                    'message': 'Key with name "{}" already exists'.format(
-                        data["key_name"]
+                        data['key_name']
                     )
                 }
             else:
                 # Key Not Found. It implies user' haven't added any key yet.
                 shared.etcd_client.put(
-                    etcd_key, data["key"], value_in_json=True
+                    etcd_key, data['key'], value_in_json=True
                 )
-                return {"message": "Key added successfully"}
+                return {'message': 'Key added successfully'}
         else:
             return validator.get_errors(), 400
 
 
 class RemoveSSHKey(Resource):
     @staticmethod
-    def get():
+    def post():
         data = request.json
         validator = schemas.RemoveSSHSchema(data)
         if validator.is_valid():
 
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                settings["etcd"]["user_prefix"],
+                shared.settings['etcd']['user_prefix'],
-                data["realm"],
+                data['realm'],
-                data["name"],
+                data['name'],
-                "key",
+                'key',
-                data["key_name"],
+                data['key_name'],
             )
             etcd_entry = shared.etcd_client.get(
                 etcd_key, value_in_json=True
             )
             if etcd_entry:
                 shared.etcd_client.client.delete(etcd_key)
-                return {"message": "Key successfully removed."}
+                return {'message': 'Key successfully removed.'}
             else:
                 return {
-                    "message": "No Key with name '{}' Exists at all.".format(
+                    'message': 'No Key with name "{}" Exists at all.'.format(
-                        data["key_name"]
+                        data['key_name']
                     )
                 }
         else:
@@ -468,104 +469,107 @@ class CreateNetwork(Resource):
         if validator.is_valid():
 
             network_entry = {
-                "id": counters.increment_etcd_counter(
+                'id': counters.increment_etcd_counter(
-                    shared.etcd_client, "/v1/counter/vxlan"
+                    shared.etcd_client, shared.settings['etcd']['vxlan_counter']
                 ),
-                "type": data["type"],
+                'type': data['type'],
             }
             if validator.user.value:
                 try:
                     nb = pynetbox.api(
-                        url=settings["netbox"]["url"],
+                        url=shared.settings['netbox']['url'],
-                        token=settings["netbox"]["token"],
+                        token=shared.settings['netbox']['token'],
                     )
                     nb_prefix = nb.ipam.prefixes.get(
-                        prefix=settings["network"]["prefix"]
+                        prefix=shared.settings['network']['prefix']
                     )
                     prefix = nb_prefix.available_prefixes.create(
                         data={
-                            "prefix_length": int(
+                            'prefix_length': int(
-                                settings["network"]["prefix_length"]
+                                shared.settings['network']['prefix_length']
                             ),
-                            "description": '{}\'s network "{}"'.format(
+                            'description': '{}\'s network "{}"'.format(
-                                data["name"], data["network_name"]
+                                data['name'], data['network_name']
                             ),
-                            "is_pool": True,
+                            'is_pool': True,
                         }
                     )
                 except Exception as err:
                     app.logger.error(err)
                     return {
-                        "message": "Error occured while creating network."
+                        'message': 'Error occured while creating network.'
                     }
                 else:
-                    network_entry["ipv6"] = prefix["prefix"]
+                    network_entry['ipv6'] = prefix['prefix']
             else:
-                network_entry["ipv6"] = "fd00::/64"
+                network_entry['ipv6'] = 'fd00::/64'
 
             network_key = join_path(
-                settings["etcd"]["network_prefix"],
+                shared.settings['etcd']['network_prefix'],
-                data["name"],
+                data['name'],
-                data["network_name"],
+                data['network_name'],
             )
             shared.etcd_client.put(
                 network_key, network_entry, value_in_json=True
             )
-            return {"message": "Network successfully added."}
+            return {'message': 'Network successfully added.'}
         else:
             return validator.get_errors(), 400
 
 
 class ListUserNetwork(Resource):
     @staticmethod
-    def get():
+    def post():
         data = request.json
         validator = schemas.OTPSchema(data)
 
         if validator.is_valid():
             prefix = join_path(
-                settings["etcd"]["network_prefix"], data["name"]
+                shared.settings['etcd']['network_prefix'], data['name']
             )
             networks = shared.etcd_client.get_prefix(
                 prefix, value_in_json=True
             )
             user_networks = []
             for net in networks:
-                net.value["name"] = net.key.split("/")[-1]
+                net.value['name'] = net.key.split('/')[-1]
                 user_networks.append(net.value)
-            return {"networks": user_networks}, 200
+            return {'networks': user_networks}, 200
         else:
             return validator.get_errors(), 400
 
 
-api.add_resource(CreateVM, "/vm/create")
+api.add_resource(CreateVM, '/vm/create')
-api.add_resource(VmStatus, "/vm/status")
+api.add_resource(VmStatus, '/vm/status')
 
-api.add_resource(VMAction, "/vm/action")
+api.add_resource(VMAction, '/vm/action')
-api.add_resource(VMMigration, "/vm/migrate")
+api.add_resource(VMMigration, '/vm/migrate')
 
-api.add_resource(CreateImage, "/image/create")
+api.add_resource(CreateImage, '/image/create')
-api.add_resource(ListPublicImages, "/image/list-public")
+api.add_resource(ListPublicImages, '/image/list-public')
 
-api.add_resource(ListUserVM, "/user/vms")
+api.add_resource(ListUserVM, '/user/vms')
-api.add_resource(ListUserFiles, "/user/files")
+api.add_resource(ListUserFiles, '/user/files')
-api.add_resource(ListUserNetwork, "/user/networks")
+api.add_resource(ListUserNetwork, '/user/networks')
 
-api.add_resource(AddSSHKey, "/user/add-ssh")
+api.add_resource(AddSSHKey, '/user/add-ssh')
-api.add_resource(RemoveSSHKey, "/user/remove-ssh")
+api.add_resource(RemoveSSHKey, '/user/remove-ssh')
-api.add_resource(GetSSHKeys, "/user/get-ssh")
+api.add_resource(GetSSHKeys, '/user/get-ssh')
 
-api.add_resource(CreateHost, "/host/create")
+api.add_resource(CreateHost, '/host/create')
-api.add_resource(ListHost, "/host/list")
+api.add_resource(ListHost, '/host/list')
 
-api.add_resource(CreateNetwork, "/network/create")
+api.add_resource(CreateNetwork, '/network/create')
 
 
-def main(debug=False, port=None):
+def main(arguments):
+    debug = arguments['debug']
+    port = arguments['port']
+
     try:
         image_stores = list(
             shared.etcd_client.get_prefix(
-                settings["etcd"]["image_store_prefix"], value_in_json=True
+                shared.settings['etcd']['image_store_prefix'], value_in_json=True
             )
         )
     except KeyError:
@@ -576,27 +580,21 @@ def main(debug=False, port=None):
     #
     # if not image_stores:
     #     data = {
-    #         "is_public": True,
+    #         'is_public': True,
-    #         "type": "ceph",
+    #         'type': 'ceph',
-    #         "name": "images",
+    #         'name': 'images',
-    #         "description": "first ever public image-store",
+    #         'description': 'first ever public image-store',
-    #         "attributes": {"list": [], "key": [], "pool": "images"},
+    #         'attributes': {'list': [], 'key': [], 'pool': 'images'},
     #     }
 
     #     shared.etcd_client.put(
     #         join_path(
-    #             settings["etcd"]["image_store_prefix"], uuid4().hex
+    #             shared.settings['etcd']['image_store_prefix'], uuid4().hex
     #         ),
     #         json.dumps(data),
     #     )
 
     try:
-        app.run(host="::",
+        app.run(host='::', port=port, debug=debug)
-                port=port,
-                debug=debug)
     except OSError as e:
-        raise UncloudException("Failed to start Flask: {}".format(e))
+        raise UncloudException('Failed to start Flask: {}'.format(e))
-
-
-if __name__ == "__main__":
-    main()

uncloud/api/schemas.py

@@ -21,8 +21,7 @@ import bitmath
 
 from uncloud.common.host import HostStatus
 from uncloud.common.vm import VMStatus
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
 from . import helper, logger
 from .common_fields import Field, VmUUIDField
 from .helper import check_otp, resolve_vm_name
@@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema):
     def file_uuid_validation(self):
         file_entry = shared.etcd_client.get(
             os.path.join(
-                settings["etcd"]["file_prefix"], self.uuid.value
+                shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
             )
         )
         if file_entry is None:
@@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema):
     def image_store_name_validation(self):
         image_stores = list(
             shared.etcd_client.get_prefix(
-                settings["etcd"]["image_store_prefix"]
+                shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
             )
         )
 
@@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema):
             for net in _network:
                 network = shared.etcd_client.get(
                     os.path.join(
-                        settings["etcd"]["network_prefix"],
+                        shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
                         self.name.value,
                         net,
                     ),
@@ -322,7 +321,7 @@ class CreateVMSchema(OTPSchema):
                     "Your specified OS-SSD is not in correct units"
                 )
 
-            if _cpu < 1:
+            if int(_cpu) < 1:
                 self.add_error("CPU must be atleast 1")
 
             if parsed_ram < bitmath.GB(1):
@@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema):
             self.add_error("Can't migrate non-running VM")
 
         if vm.hostname == os.path.join(
-            settings["etcd"]["host_prefix"], self.destination.value
+            shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
         ):
             self.add_error(
                 "Destination host couldn't be same as Source Host"
@@ -528,9 +527,7 @@ class GetSSHSchema(OTPSchema):
 
 class CreateNetwork(OTPSchema):
     def __init__(self, data):
-        self.network_name = Field(
+        self.network_name = Field("network_name", str, data.get("network_name", KeyError))
-            "network_name", str, data.get("network_name", KeyError)
-        )
         self.type = Field("type", str, data.get("type", KeyError))
         self.user = Field("user", bool, bool(data.get("user", False)))
 
@@ -541,14 +538,8 @@ class CreateNetwork(OTPSchema):
         super().__init__(data, fields=fields)
 
     def network_name_validation(self):
-        network = shared.etcd_client.get(
+        key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
-            os.path.join(
+        network = shared.etcd_client.get(key, value_in_json=True)
-                settings["etcd"]["network_prefix"],
-                self.name.value,
-                self.network_name.value,
-            ),
-            value_in_json=True,
-        )
         if network:
             self.add_error(
                 "Network with name {} already exists".format(

uncloud/cli/helper.py

@@ -0,0 +1,46 @@
+import requests
+import json
+import argparse
+import binascii
+
+from pyotp import TOTP
+from os.path import join as join_path
+from uncloud.common.shared import shared
+
+
+def get_otp_parser():
+    otp_parser = argparse.ArgumentParser('otp')
+    otp_parser.add_argument('--name')
+    otp_parser.add_argument('--realm')
+    otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
+
+    return otp_parser
+
+
+def load_dump_pretty(content):
+    if isinstance(content, bytes):
+        content = content.decode('utf-8')
+    parsed = json.loads(content)
+    return json.dumps(parsed, indent=4, sort_keys=True)
+
+
+def make_request(*args, data=None, request_method=requests.post):
+    try:
+        r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
+    except requests.exceptions.RequestException:
+        print('Error occurred while connecting to API server.')
+    else:
+        try:
+            print(load_dump_pretty(r.content))
+        except Exception:
+            print('Error occurred while getting output from api server.')
+
+
+def get_token(seed):
+    if seed is not None:
+        try:
+            token = TOTP(seed).now()
+        except binascii.Error:
+            raise argparse.ArgumentTypeError('Invalid seed')
+        else:
+            return token

uncloud/cli/host.py

@@ -0,0 +1,45 @@
+import requests
+
+from uncloud.cli.helper import make_request, get_otp_parser
+from uncloud.common.parser import BaseParser
+
+
+class HostParser(BaseParser):
+    def __init__(self):
+        super().__init__('host')
+
+    def create(self, **kwargs):
+        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--hostname', required=True)
+        p.add_argument('--cpu', required=True, type=int)
+        p.add_argument('--ram', required=True)
+        p.add_argument('--os-ssd', required=True)
+        p.add_argument('--hdd', default=list())
+
+    def list(self, **kwargs):
+        self.subparser.add_parser('list', **kwargs)
+
+
+parser = HostParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('host_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        request_method = requests.post
+        data = None
+        if subcommand == 'create':
+            kwargs['specs'] = {
+                'cpu': kwargs.pop('cpu'),
+                'ram': kwargs.pop('ram'),
+                'os-ssd': kwargs.pop('os_ssd'),
+                'hdd': kwargs.pop('hdd')
+            }
+            data = kwargs
+        elif subcommand == 'list':
+            request_method = requests.get
+
+        make_request('host', subcommand, data=data, request_method=request_method)

uncloud/cli/image.py

@@ -0,0 +1,38 @@
+import requests
+
+from uncloud.cli.helper import make_request
+from uncloud.common.parser import BaseParser
+
+
+class ImageParser(BaseParser):
+    def __init__(self):
+        super().__init__('image')
+
+    def create(self, **kwargs):
+        p = self.subparser.add_parser('create', **kwargs)
+        p.add_argument('--name', required=True)
+        p.add_argument('--uuid', required=True)
+        p.add_argument('--image-store', required=True, dest='image_store')
+
+    def list(self, **kwargs):
+        self.subparser.add_parser('list', **kwargs)
+
+
+parser = ImageParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('image_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        data = None
+        request_method = requests.post
+        if subcommand == 'list':
+            subcommand = 'list-public'
+            request_method = requests.get
+        elif subcommand == 'create':
+            data = kwargs
+
+        make_request('image', subcommand, data=data, request_method=request_method)

uncloud/cli/main.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import argparse
+import importlib
+
+arg_parser = argparse.ArgumentParser('cli', add_help=False)
+subparser = arg_parser.add_subparsers(dest='subcommand')
+
+for component in ['user', 'host', 'image', 'network', 'vm']:
+    module = importlib.import_module('uncloud.cli.{}'.format(component))
+    parser = getattr(module, 'arg_parser')
+    subparser.add_parser(name=parser.prog, parents=[parser])
+
+
+def main(arguments):
+    if not arguments['subcommand']:
+        arg_parser.print_help()
+    else:
+        name = arguments.pop('subcommand')
+        arguments.pop('debug')
+        mod = importlib.import_module('uncloud.cli.{}'.format(name))
+        _main = getattr(mod, 'main')
+        _main(**arguments)

uncloud/cli/network.py

@@ -0,0 +1,32 @@
+import requests
+
+from uncloud.cli.helper import make_request, get_otp_parser
+from uncloud.common.parser import BaseParser
+
+
+class NetworkParser(BaseParser):
+    def __init__(self):
+        super().__init__('network')
+
+    def create(self, **kwargs):
+        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--network-name', required=True)
+        p.add_argument('--network-type', required=True, dest='type')
+        p.add_argument('--user', action='store_true')
+
+
+parser = NetworkParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('network_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        data = None
+        request_method = requests.post
+        if subcommand == 'create':
+            data = kwargs
+
+        make_request('network', subcommand, data=data, request_method=request_method)

uncloud/cli/user.py

@@ -0,0 +1,41 @@
+from uncloud.cli.helper import make_request, get_otp_parser
+from uncloud.common.parser import BaseParser
+
+
+class UserParser(BaseParser):
+    def __init__(self):
+        super().__init__('user')
+
+    def files(self, **kwargs):
+        self.subparser.add_parser('files', parents=[get_otp_parser()], **kwargs)
+
+    def vms(self, **kwargs):
+        self.subparser.add_parser('vms', parents=[get_otp_parser()], **kwargs)
+
+    def networks(self, **kwargs):
+        self.subparser.add_parser('networks', parents=[get_otp_parser()], **kwargs)
+
+    def add_ssh(self, **kwargs):
+        p = self.subparser.add_parser('add-ssh', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--key-name', required=True)
+        p.add_argument('--key', required=True)
+
+    def get_ssh(self, **kwargs):
+        p = self.subparser.add_parser('get-ssh', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--key-name', default='')
+
+    def remove_ssh(self, **kwargs):
+        p = self.subparser.add_parser('remove-ssh', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--key-name', required=True)
+
+
+parser = UserParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('user_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        make_request('user', subcommand, data=kwargs)

uncloud/cli/vm.py

@@ -0,0 +1,62 @@
+from uncloud.common.parser import BaseParser
+from uncloud.cli.helper import make_request, get_otp_parser
+
+
+class VMParser(BaseParser):
+    def __init__(self):
+        super().__init__('vm')
+
+    def start(self, **args):
+        p = self.subparser.add_parser('start', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+
+    def stop(self, **args):
+        p = self.subparser.add_parser('stop', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+
+    def status(self, **args):
+        p = self.subparser.add_parser('status', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+
+    def delete(self, **args):
+        p = self.subparser.add_parser('delete', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+
+    def migrate(self, **args):
+        p = self.subparser.add_parser('migrate', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+        p.add_argument('--destination', required=True)
+
+    def create(self, **args):
+        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **args)
+        p.add_argument('--cpu', required=True)
+        p.add_argument('--ram', required=True)
+        p.add_argument('--os-ssd', required=True)
+        p.add_argument('--hdd', action='append', default=list())
+        p.add_argument('--image', required=True)
+        p.add_argument('--network', action='append', default=[])
+        p.add_argument('--vm-name', required=True)
+
+
+parser = VMParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('vm_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        data = kwargs
+        endpoint = subcommand
+        if subcommand in ['start', 'stop', 'delete']:
+            endpoint = 'action'
+            data['action'] = subcommand
+        elif subcommand == 'create':
+            kwargs['specs'] = {
+                'cpu': kwargs.pop('cpu'),
+                'ram': kwargs.pop('ram'),
+                'os-ssd': kwargs.pop('os_ssd'),
+                'hdd': kwargs.pop('hdd')
+            }
+        make_request('vm', endpoint, data=data)

uncloud/common/cli.py

@@ -0,0 +1,26 @@
+from uncloud.common.shared import shared
+from pyotp import TOTP
+
+
+def get_token(seed):
+    if seed is not None:
+        try:
+            token = TOTP(seed).now()
+        except Exception:
+            raise Exception('Invalid seed')
+        else:
+            return token
+
+
+def resolve_otp_credentials(kwargs):
+    d = {
+        'name': shared.settings['client']['name'],
+        'realm': shared.settings['client']['realm'],
+        'token': get_token(shared.settings['client']['seed'])
+    }
+
+    for k, v in d.items():
+        if k in kwargs and kwargs[k] is None:
+            kwargs.update({k: v})
+
+    return d

uncloud/common/etcd_wrapper.py

@@ -1,24 +1,21 @@
 import etcd3
 import json
-import queue
-import copy
-from uncloud import UncloudException
 
-from collections import namedtuple
 from functools import wraps
 
-from . import logger
+from uncloud import UncloudException
-
+from uncloud.common import logger
-PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"])
 
 
 class EtcdEntry:
-    # key: str
+    def __init__(self, meta_or_key, value, value_in_json=False):
-    # value: str
+        if hasattr(meta_or_key, 'key'):
-
+            # if meta has attr 'key' then get it
-    def __init__(self, meta, value, value_in_json=False):
+            self.key = meta_or_key.key.decode('utf-8')
-        self.key = meta.key.decode("utf-8")
+        else:
-        self.value = value.decode("utf-8")
+            # otherwise meta is the 'key'
+            self.key = meta_or_key
+        self.value = value.decode('utf-8')
 
         if value_in_json:
             self.value = json.loads(self.value)
@@ -29,18 +26,12 @@ def readable_errors(func):
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
-        except etcd3.exceptions.ConnectionFailedError as err:
+        except etcd3.exceptions.ConnectionFailedError:
-            raise UncloudException(
+            raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?')
-                "Cannot connect to etcd: is etcd running as configured in uncloud.conf?"
-            )
         except etcd3.exceptions.ConnectionTimeoutError as err:
-            raise etcd3.exceptions.ConnectionTimeoutError(
+            raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err
-                "etcd connection timeout."
-            ) from err
         except Exception:
-            logger.exception(
+            logger.exception('Some etcd error occured. See syslog for details.')
-                "Some etcd error occured. See syslog for details."
-            )
 
     return wrapper
 
@@ -64,55 +55,21 @@ class Etcd3Wrapper:
             _value = json.dumps(_value)
 
         if not isinstance(_key, str):
-            _key = _key.decode("utf-8")
+            _key = _key.decode('utf-8')
 
         return self.client.put(_key, _value, **kwargs)
 
     @readable_errors
-    def get_prefix(self, *args, value_in_json=False, **kwargs):
+    def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs):
-        r = self.client.get_prefix(*args, **kwargs)
+        event_iterator = self.client.get_prefix(*args, **kwargs)
-        for entry in r:
+        for e in event_iterator:
-            e = EtcdEntry(*entry[::-1], value_in_json=value_in_json)
+            yield EtcdEntry(*e[::-1], value_in_json=value_in_json)
-            if e.value:
-                yield e
 
     @readable_errors
-    def watch_prefix(self, key, timeout=0, value_in_json=False):
+    def watch_prefix(self, key, raise_exception=True, value_in_json=False):
-        timeout_event = EtcdEntry(
+        event_iterator, cancel = self.client.watch_prefix(key)
-            PseudoEtcdMeta(key=b"TIMEOUT"),
+        for e in event_iterator:
-            value=str.encode(
+            if hasattr(e, '_event'):
-                json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"})
+                e = e._event
-            ),
+                if e.type == e.PUT:
-            value_in_json=value_in_json,
+                    yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)
-        )
-
-        event_queue = queue.Queue()
-
-        def add_event_to_queue(event):
-            if hasattr(event, "events"):
-                for e in event.events:
-                    if e.value:
-                        event_queue.put(
-                            EtcdEntry(
-                                e, e.value, value_in_json=value_in_json
-                            )
-                        )
-
-        self.client.add_watch_prefix_callback(key, add_event_to_queue)
-
-        while True:
-            try:
-                while True:
-                    v = event_queue.get(timeout=timeout)
-                    yield v
-            except queue.Empty:
-                event_queue.put(copy.deepcopy(timeout_event))
-
-
-class PsuedoEtcdEntry(EtcdEntry):
-    def __init__(self, key, value, value_in_json=False):
-        super().__init__(
-            PseudoEtcdMeta(key=key.encode("utf-8")),
-            value,
-            value_in_json=value_in_json,
-        )

uncloud/common/network.py

@@ -1,8 +1,6 @@
 import subprocess as sp
 import random
 import logging
-import socket
-from contextlib import closing
 
 logger = logging.getLogger(__name__)
 

uncloud/common/parser.py

@@ -0,0 +1,13 @@
+import argparse
+
+
+class BaseParser:
+    def __init__(self, command):
+        self.arg_parser = argparse.ArgumentParser(command, add_help=False)
+        self.subparser = self.arg_parser.add_subparsers(dest='{}_subcommand'.format(command))
+        self.common_args = {'add_help': False}
+
+        methods = [attr for attr in dir(self) if not attr.startswith('__')
+                   and type(getattr(self, attr)).__name__ == 'method']
+        for method in methods:
+            getattr(self, method)(**self.common_args)

uncloud/common/request.py

@@ -2,8 +2,8 @@ import json
 from os.path import join
 from uuid import uuid4
 
-from .etcd_wrapper import PsuedoEtcdEntry
+from uncloud.common.etcd_wrapper import EtcdEntry
-from .classes import SpecificEtcdEntryBase
+from uncloud.common.classes import SpecificEtcdEntryBase
 
 
 class RequestType:
@@ -29,11 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase):
 
     @classmethod
     def from_scratch(cls, request_prefix, **kwargs):
-        e = PsuedoEtcdEntry(
+        e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex),
-            join(request_prefix, uuid4().hex),
+                      value=json.dumps(kwargs).encode('utf-8'), value_in_json=True)
-            value=json.dumps(kwargs).encode("utf-8"),
-            value_in_json=True,
-        )
         return cls(e)
 
 

uncloud/common/settings.py

@@ -0,0 +1,136 @@
+import configparser
+import logging
+import sys
+import os
+
+from datetime import datetime
+from uncloud.common.etcd_wrapper import Etcd3Wrapper
+from os.path import join as join_path
+
+logger = logging.getLogger(__name__)
+settings = None
+
+
+class CustomConfigParser(configparser.RawConfigParser):
+    def __getitem__(self, key):
+        try:
+            result = super().__getitem__(key)
+        except KeyError as err:
+            raise KeyError(
+                'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format(
+                    key
+                )
+            ) from err
+        else:
+            return result
+
+
+class Settings(object):
+    def __init__(self, conf_dir, seed_value=None):
+        conf_name = 'uncloud.conf'
+        self.config_file = join_path(conf_dir, conf_name)
+
+        # this is used to cache config from etcd for 1 minutes. Without this we
+        # would make a lot of requests to etcd which slows down everything.
+        self.last_config_update = datetime.fromtimestamp(0)
+
+        self.config_parser = CustomConfigParser(allow_no_value=True)
+        self.config_parser.add_section('etcd')
+        self.config_parser.set('etcd', 'base_prefix', '/')
+
+        if os.access(self.config_file, os.R_OK):
+            self.config_parser.read(self.config_file)
+        else:
+            raise FileNotFoundError('Config file %s not found!', self.config_file)
+        self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
+
+        self.read_internal_values()
+
+        if seed_value is None:
+            seed_value = dict()
+
+        self.config_parser.read_dict(seed_value)
+
+    def get_etcd_client(self):
+        args = tuple()
+        try:
+            kwargs = {
+                'host': self.config_parser.get('etcd', 'url'),
+                'port': self.config_parser.get('etcd', 'port'),
+                'ca_cert': self.config_parser.get('etcd', 'ca_cert'),
+                'cert_cert': self.config_parser.get('etcd', 'cert_cert'),
+                'cert_key': self.config_parser.get('etcd', 'cert_key'),
+            }
+        except configparser.Error as err:
+            raise configparser.Error(
+                '{} in config file {}'.format(
+                    err.message, self.config_file
+                )
+            ) from err
+        else:
+            try:
+                wrapper = Etcd3Wrapper(*args, **kwargs)
+            except Exception as err:
+                logger.error(
+                    'etcd connection not successfull. Please check your config file.'
+                    '\nDetails: %s\netcd connection parameters: %s',
+                    err,
+                    kwargs,
+                )
+                sys.exit(1)
+            else:
+                return wrapper
+
+    def read_internal_values(self):
+        base_prefix = self['etcd']['base_prefix']
+        self.config_parser.read_dict(
+            {
+                'etcd': {
+                    'file_prefix': join_path(base_prefix, 'files/'),
+                    'host_prefix': join_path(base_prefix, 'hosts/'),
+                    'image_prefix': join_path(base_prefix, 'images/'),
+                    'image_store_prefix': join_path(base_prefix, 'imagestore/'),
+                    'network_prefix': join_path(base_prefix, 'networks/'),
+                    'request_prefix': join_path(base_prefix, 'requests/'),
+                    'user_prefix': join_path(base_prefix, 'users/'),
+                    'vm_prefix': join_path(base_prefix, 'vms/'),
+                    'vxlan_counter': join_path(base_prefix, 'counters/vxlan'),
+                    'tap_counter': join_path(base_prefix, 'counters/tap')
+                }
+            }
+        )
+
+    def read_config_file_values(self, config_file):
+        try:
+            # Trying to read configuration file
+            with open(config_file) as config_file_handle:
+                self.config_parser.read_file(config_file_handle)
+        except FileNotFoundError:
+            sys.exit('Configuration file {} not found!'.format(config_file))
+        except Exception as err:
+            logger.exception(err)
+            sys.exit('Error occurred while reading configuration file')
+
+    def read_values_from_etcd(self):
+        etcd_client = self.get_etcd_client()
+        if (datetime.utcnow() - self.last_config_update).total_seconds() > 60:
+            config_from_etcd = etcd_client.get(self.config_key, value_in_json=True)
+            if config_from_etcd:
+                self.config_parser.read_dict(config_from_etcd.value)
+                self.last_config_update = datetime.utcnow()
+            else:
+                raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key))
+
+    def __getitem__(self, key):
+        # Allow failing to read from etcd if we have
+        # it locally
+        if key not in self.config_parser.sections():
+            try:
+                self.read_values_from_etcd()
+            except KeyError:
+                pass
+        return self.config_parser[key]
+
+
+def get_settings():
+    return settings

uncloud/common/shared.py

@@ -0,0 +1,34 @@
+from uncloud.common.settings import get_settings
+from uncloud.common.vm import VmPool
+from uncloud.common.host import HostPool
+from uncloud.common.request import RequestPool
+import uncloud.common.storage_handlers as storage_handlers
+
+
+class Shared:
+    @property
+    def settings(self):
+        return get_settings()
+
+    @property
+    def etcd_client(self):
+        return self.settings.get_etcd_client()
+
+    @property
+    def host_pool(self):
+        return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
+
+    @property
+    def vm_pool(self):
+        return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
+
+    @property
+    def request_pool(self):
+        return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
+
+    @property
+    def storage_handler(self):
+        return storage_handlers.get_storage_handler()
+
+
+shared = Shared()

uncloud/common/storage_handlers.py

@@ -6,8 +6,7 @@ import stat
 from abc import ABC
 from . import logger
 from os.path import join as join_path
-
+import uncloud.common.shared as shared
-from uncloud.settings import settings as config
 
 
 class ImageStorageHandler(ABC):
@@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
 
 
 def get_storage_handler():
-    __storage_backend = config["storage"]["storage_backend"]
+    __storage_backend = shared.shared.settings["storage"]["storage_backend"]
     if __storage_backend == "filesystem":
         return FileSystemBasedImageStorageHandler(
-            vm_base=config["storage"]["vm_dir"],
+            vm_base=shared.shared.settings["storage"]["vm_dir"],
-            image_base=config["storage"]["image_dir"],
+            image_base=shared.shared.settings["storage"]["image_dir"],
         )
     elif __storage_backend == "ceph":
         return CEPHBasedImageStorageHandler(
-            vm_base=config["storage"]["ceph_vm_pool"],
+            vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
-            image_base=config["storage"]["ceph_image_pool"],
+            image_base=shared.shared.settings["storage"]["ceph_image_pool"],
         )
     else:
-        raise Exception("Unknown Image Storage Handler")
+        raise Exception("Unknown Image Storage Handler")

uncloud/configure/main.py

@@ -1,79 +1,57 @@
 import os
+import argparse
 
-from uncloud.settings import settings
+from uncloud.common.shared import shared
-from uncloud.shared import shared
+
+arg_parser = argparse.ArgumentParser('configure', add_help=False)
+configure_subparsers = arg_parser.add_subparsers(dest='subcommand')
+
+otp_parser = configure_subparsers.add_parser('otp')
+otp_parser.add_argument('--verification-controller-url', required=True, metavar='URL')
+otp_parser.add_argument('--auth-name', required=True, metavar='OTP-NAME')
+otp_parser.add_argument('--auth-realm', required=True, metavar='OTP-REALM')
+otp_parser.add_argument('--auth-seed', required=True, metavar='OTP-SEED')
+
+network_parser = configure_subparsers.add_parser('network')
+network_parser.add_argument('--prefix-length', required=True, type=int)
+network_parser.add_argument('--prefix', required=True)
+network_parser.add_argument('--vxlan-phy-dev', required=True)
+
+netbox_parser = configure_subparsers.add_parser('netbox')
+netbox_parser.add_argument('--url', required=True)
+netbox_parser.add_argument('--token', required=True)
+
+ssh_parser = configure_subparsers.add_parser('ssh')
+ssh_parser.add_argument('--username', default='root')
+ssh_parser.add_argument('--private-key-path', default=os.path.expanduser('~/.ssh/id_rsa'),)
+
+storage_parser = configure_subparsers.add_parser('storage')
+storage_parser.add_argument('--file-dir', required=True)
+storage_parser_subparsers = storage_parser.add_subparsers(dest='storage_backend')
+
+filesystem_storage_parser = storage_parser_subparsers.add_parser('filesystem')
+filesystem_storage_parser.add_argument('--vm-dir', required=True)
+filesystem_storage_parser.add_argument('--image-dir', required=True)
+
+ceph_storage_parser = storage_parser_subparsers.add_parser('ceph')
+ceph_storage_parser.add_argument('--ceph-vm-pool', required=True)
+ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
 
 
 def update_config(section, kwargs):
-    uncloud_config = shared.etcd_client.get(
+    uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
-        settings.config_key, value_in_json=True
-    )
     if not uncloud_config:
         uncloud_config = {}
     else:
         uncloud_config = uncloud_config.value
 
     uncloud_config[section] = kwargs
-    shared.etcd_client.put(
+    shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
-        settings.config_key, uncloud_config, value_in_json=True
-    )
 
 
-def configure_parser(parser):
+def main(arguments):
-    configure_subparsers = parser.add_subparsers(dest="subcommand")
+    subcommand = arguments['subcommand']
-
-    otp_parser = configure_subparsers.add_parser("otp")
-    otp_parser.add_argument(
-        "--verification-controller-url", required=True, metavar="URL"
-    )
-    otp_parser.add_argument(
-        "--auth-name", required=True, metavar="OTP-NAME"
-    )
-    otp_parser.add_argument(
-        "--auth-realm", required=True, metavar="OTP-REALM"
-    )
-    otp_parser.add_argument(
-        "--auth-seed", required=True, metavar="OTP-SEED"
-    )
-
-    network_parser = configure_subparsers.add_parser("network")
-    network_parser.add_argument(
-        "--prefix-length", required=True, type=int
-    )
-    network_parser.add_argument("--prefix", required=True)
-    network_parser.add_argument("--vxlan-phy-dev", required=True)
-
-    netbox_parser = configure_subparsers.add_parser("netbox")
-    netbox_parser.add_argument("--url", required=True)
-    netbox_parser.add_argument("--token", required=True)
-
-    ssh_parser = configure_subparsers.add_parser("ssh")
-    ssh_parser.add_argument("--username", default="root")
-    ssh_parser.add_argument(
-        "--private-key-path",
-        default=os.path.expanduser("~/.ssh/id_rsa"),
-    )
-
-    storage_parser = configure_subparsers.add_parser("storage")
-    storage_parser.add_argument("--file-dir", required=True)
-    storage_parser_subparsers = storage_parser.add_subparsers(
-        dest="storage_backend"
-    )
-
-    filesystem_storage_parser = storage_parser_subparsers.add_parser(
-        "filesystem"
-    )
-    filesystem_storage_parser.add_argument("--vm-dir", required=True)
-    filesystem_storage_parser.add_argument("--image-dir", required=True)
-
-    ceph_storage_parser = storage_parser_subparsers.add_parser("ceph")
-    ceph_storage_parser.add_argument("--ceph-vm-pool", required=True)
-    ceph_storage_parser.add_argument("--ceph-image-pool", required=True)
-
-
-def main(**kwargs):
-    subcommand = kwargs.pop("subcommand")
     if not subcommand:
-        pass
+        arg_parser.print_help()
     else:
-        update_config(subcommand, kwargs)
+        update_config(subcommand, arguments)

uncloud/docs/source/hacking.rst

@@ -1,17 +0,0 @@
-Hacking
-=======
-How to hack on the code.
-
-[ to be done by Balazs:
-
-* make nice
-* indent with shell script mode
-
-]
-
-* git clone the repo
-* cd to the repo
-* Setup your venv: python -m venv venv
-* . ./venv/bin/activate # you need the leading dot for sourcing!
-* Run ./bin/ucloud-run-reinstall - it should print you an error
-  message on how to use ucloud

uncloud/filescanner/main.py

@@ -3,12 +3,16 @@ import os
 import pathlib
 import subprocess as sp
 import time
+import argparse
+import bitmath
 
 from uuid import uuid4
 
 from . import logger
-from uncloud.settings import settings
+from uncloud.common.shared import shared
-from uncloud.shared import shared
+
+arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
+arg_parser.add_argument('--hostname', required=True)
 
 
 def sha512sum(file: str):
@@ -24,66 +28,58 @@ def sha512sum(file: str):
     if not isinstance(file, str):
         raise TypeError
     try:
-        output = sp.check_output(["sha512sum", file], stderr=sp.PIPE)
+        output = sp.check_output(['sha512sum', file], stderr=sp.PIPE)
     except sp.CalledProcessError as e:
-        error = e.stderr.decode("utf-8")
+        error = e.stderr.decode('utf-8')
-        if "No such file or directory" in error:
+        if 'No such file or directory' in error:
             raise FileNotFoundError from None
     else:
-        output = output.decode("utf-8").strip()
+        output = output.decode('utf-8').strip()
-        output = output.split(" ")
+        output = output.split(' ')
         return output[0]
     return None
 
 
-def track_file(file, base_dir):
+def track_file(file, base_dir, host):
-    file_id = uuid4()
+    file_path = file.relative_to(base_dir)
-
+    file_str = str(file)
     # Get Username
-    owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)]
+    try:
+        owner = file_path.parts[0]
+    except IndexError:
+        pass
+    else:
+        file_path = file_path.relative_to(owner)
+        creation_date = time.ctime(os.stat(file_str).st_ctime)
 
-    # Get Creation Date of File
+        entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
-    # Here, we are assuming that ctime is creation time
+        entry_value = {
-    # which is mostly not true.
+            'filename': str(file_path),
-    creation_date = time.ctime(os.stat(file).st_ctime)
+            'owner': owner,
+            'sha512sum': sha512sum(file_str),
+            'creation_date': creation_date,
+            'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()),
+            'host': host
+        }
 
-    file_path = pathlib.Path(file).parts[-1]
+        logger.info('Tracking %s', file_str)
 
-    # Create Entry
+        shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
-    entry_key = os.path.join(
-        settings["etcd"]["file_prefix"], str(file_id)
-    )
-    entry_value = {
-        "filename": file_path,
-        "owner": owner,
-        "sha512sum": sha512sum(file),
-        "creation_date": creation_date,
-        "size": os.path.getsize(file),
-    }
-
-    logger.info("Tracking %s", file)
-
-    shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
-    os.setxattr(file, "user.utracked", b"True")
 
 
-def main():
+def main(arguments):
-    base_dir = settings["storage"]["file_dir"]
+    hostname = arguments['hostname']
-
+    base_dir = shared.settings['storage']['file_dir']
     # Recursively Get All Files and Folder below BASE_DIR
-    files = glob.glob("{}/**".format(base_dir), recursive=True)
+    files = glob.glob('{}/**'.format(base_dir), recursive=True)
+    files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
 
-    # Retain only Files
+    # Files that are already tracked
-    files = [file for file in files if os.path.isfile(file)]
+    tracked_files = [
-
+        pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
-    untracked_files = []
+        for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
-    for file in files:
+        if f.value['host'] == hostname
-        try:
+    ]
-            os.getxattr(file, "user.utracked")
+    untracked_files = set(files) - set(tracked_files)
-        except OSError:
+    for file in untracked_files:
-            track_file(file, base_dir)
+        track_file(file, base_dir, hostname)
-            untracked_files.append(file)
-
-
-if __name__ == "__main__":
-    main()

uncloud/hack/config.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
+#
+# This file is part of uncloud.
+#
+# uncloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uncloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+class Config(object):
+    def __init__(self, arguments):
+        """ read arguments dicts as a base """
+
+        self.arguments = arguments
+
+        # Split them so *etcd_args can be used and we can
+        # iterate over etcd_hosts
+        self.etcd_hosts = [ arguments['etcd_host'] ]
+        self.etcd_args = {
+            'ca_cert': arguments['etcd_ca_cert'],
+            'cert_cert': arguments['etcd_cert_cert'],
+            'cert_key': arguments['etcd_cert_key'],
+#            'user': None,
+#            'password': None
+        }
+        self.etcd_prefix = '/nicohack/'

uncloud/hack/db.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
+#
+# This file is part of uncloud.
+#
+# uncloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uncloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import etcd3
+import json
+import logging
+
+from functools import wraps
+from uncloud import UncloudException
+
+log = logging.getLogger(__name__)
+
+
+def readable_errors(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except etcd3.exceptions.ConnectionFailedError as e:
+            raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
+        except etcd3.exceptions.ConnectionTimeoutError as e:
+            raise UncloudException('etcd connection timeout. {}'.format(e))
+
+    return wrapper
+
+
+class DB(object):
+    def __init__(self, config, prefix="/"):
+        self.config = config
+
+        # Root for everything
+        self.base_prefix= '/nicohack'
+
+        # Can be set from outside
+        self.prefix = prefix
+
+        self.connect()
+
+    @readable_errors
+    def connect(self):
+        self._db_clients = []
+        for endpoint in self.config.etcd_hosts:
+            client = etcd3.client(host=endpoint, **self.config.etcd_args)
+            self._db_clients.append(client)
+
+    def realkey(self, key):
+        return "{}{}/{}".format(self.base_prefix,
+                                self.prefix,
+                                key)
+
+    @readable_errors
+    def get(self, key, as_json=False, **kwargs):
+        value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
+
+        if as_json:
+            value = json.loads(value)
+
+        return value
+
+
+    @readable_errors
+    def set(self, key, value, as_json=False, **kwargs):
+        if as_json:
+            value = json.dumps(value)
+
+        # FIXME: iterate over clients in case of failure ?
+        return self._db_clients[0].put(self.realkey(key), value, **kwargs)
+
+    @readable_errors
+    def increment(self, key, **kwargs):
+        print(self.realkey(key))
+
+
+        print("prelock")
+        lock = self._db_clients[0].lock('/nicohack/foo')
+        print("prelockacq")
+        lock.acquire()
+        print("prelockrelease")
+        lock.release()
+
+        with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock:
+            print("in lock")
+            pass
+
+#        with self._db_clients[0].lock(self.realkey(key)) as lock:#            value = int(self.get(self.realkey(key), **kwargs))
+#            self.set(self.realkey(key), str(value + 1), **kwargs)
+
+
+if __name__ == '__main__':
+    endpoints = [ "https://etcd1.ungleich.ch:2379",
+                  "https://etcd2.ungleich.ch:2379",
+                  "https://etcd3.ungleich.ch:2379" ]
+
+    db = DB(url=endpoints)

uncloud/hack/hackcloud/.gitignore

@@ -0,0 +1,3 @@
+*.iso
+radvdpid
+foo

uncloud/hack/hackcloud/__init__.py

@@ -0,0 +1 @@
+

uncloud/hack/hackcloud/etcd-client.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+etcdctl  --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \
+         --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \
+         --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \
+         --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@"

uncloud/hack/hackcloud/ifdown.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo $@

uncloud/hack/hackcloud/ifup.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+dev=$1; shift
+
+# bridge is setup from outside
+ip link set dev "$dev" master ${bridge}
+ip link set dev "$dev" up

uncloud/hack/hackcloud/mac-last

@@ -0,0 +1 @@
+000000000252

uncloud/hack/hackcloud/mac-prefix

@@ -0,0 +1 @@
+02:00

uncloud/hack/hackcloud/net.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -x
+
+netid=100
+dev=wlp2s0
+dev=wlp0s20f3
+#dev=wlan0
+
+ip=2a0a:e5c1:111:888::48/64
+vxlandev=vxlan${netid}
+bridgedev=br${netid}
+
+ip -6 link add ${vxlandev} type vxlan \
+    id ${netid} \
+    dstport 4789 \
+    group ff05::${netid} \
+    dev ${dev} \
+    ttl 5
+
+ip link set ${vxlandev} up
+
+
+ip link add ${bridgedev} type bridge
+ip link set ${bridgedev} up
+
+ip link set ${vxlandev} master ${bridgedev} up
+
+ip addr add ${ip} dev ${bridgedev}

uncloud/hack/hackcloud/nftrules

@@ -0,0 +1,31 @@
+flush ruleset
+
+table bridge filter {
+    chain prerouting {
+        type filter hook prerouting priority 0;
+        policy accept;
+
+        ibrname br100 jump br100
+    }
+
+    chain br100 {
+        # Allow all incoming traffic from outside
+        iifname vxlan100 accept
+
+        # Default blocks: router advertisements, dhcpv6, dhcpv4
+        icmpv6 type nd-router-advert drop
+        ip6 version 6 udp sport 547 drop
+        ip  version 4 udp sport 67 drop
+
+        jump br100_vmlist
+        drop
+    }
+    chain br100_vmlist {
+        # VM1
+        iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept
+
+        # VM2
+        iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept
+        iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept
+    }
+}

uncloud/hack/hackcloud/radvd.conf

@@ -0,0 +1,13 @@
+interface br100
+{
+  AdvSendAdvert on;
+  MinRtrAdvInterval 3;
+  MaxRtrAdvInterval 5;
+  AdvDefaultLifetime 3600;
+
+  prefix 2a0a:e5c1:111:888::/64 {
+  };
+
+  RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; };
+  DNSSL place7.ungleich.ch {  AdvDNSSLLifetime 6000; } ;
+};

uncloud/hack/hackcloud/radvd.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+radvd -C ./radvd.conf  -n -p ./radvdpid

uncloud/hack/hackcloud/vm.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+# if [ $# -ne 1 ]; then
+#     echo "$0: owner"
+#     exit 1
+# fi
+
+qemu=/usr/bin/qemu-system-x86_64
+
+accel=kvm
+#accel=tcg
+
+memory=1024
+cores=2
+uuid=$(uuidgen)
+mac=$(./mac-gen.py)
+owner=nico
+
+export bridge=br100
+
+set -x
+$qemu -name "uncloud-${uuid}" \
+      -machine pc,accel=${accel} \
+      -m ${memory} \
+      -smp ${cores} \
+      -uuid ${uuid} \
+      -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \
+      -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \
+      -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac}

uncloud/hack/mac.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2012 Nico Schottelius (nico-cinv at schottelius.org)
+#
+# This file is part of cinv.
+#
+# cinv is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cinv is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cinv. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import argparse
+import logging
+import os.path
+import os
+import re
+import json
+
+from uncloud import UncloudException
+from uncloud.hack.db import DB
+
+log = logging.getLogger(__name__)
+
+
+class MAC(object):
+    def __init__(self, config):
+        self.config = config
+        self.no_db = self.config.arguments['no_db']
+        if not self.no_db:
+            self.db = DB(config, prefix="/mac")
+
+        self.prefix = 0x420000000000
+        self._number = 0 # Not set by default
+
+    @staticmethod
+    def validate_mac(mac):
+        if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I):
+            raise Error("Not a valid mac address: %s" % mac)
+
+    def last_used_index(self):
+        if not self.no_db:
+            value = self.db.get("last_used_index")
+            if not value:
+                self.db.set("last_used_index", "0")
+                value = self.db.get("last_used_index")
+
+        else:
+            value = "0"
+
+        return int(value)
+
+    def last_used_mac(self):
+        return self.int_to_mac(self.prefix + self.last_used_index())
+
+    def to_colon_format(self):
+        b = self._number.to_bytes(6, byteorder="big")
+        return ':'.join(format(s, '02x') for s in b)
+
+    def to_str_format(self):
+        b = self._number.to_bytes(6, byteorder="big")
+        return ''.join(format(s, '02x') for s in b)
+
+    def create(self):
+        last_number = self.last_used_index()
+
+        if last_number == int('0xffffffff', 16):
+            raise UncloudException("Exhausted all possible mac addresses - try to free some")
+
+        next_number = last_number + 1
+        self._number = self.prefix + next_number
+
+        #next_number_string = "{:012x}".format(next_number)
+        #next_mac = self.int_to_mac(next_mac_number)
+        # db_entry = {}
+        # db_entry['vm_uuid'] = vmuuid
+        # db_entry['index'] = next_number
+        # db_entry['mac_address'] = next_mac
+
+        # should be one transaction
+        # self.db.increment("last_used_index")
+        # self.db.set("used/{}".format(next_mac),
+        #             db_entry, as_json=True)
+
+    def __int__(self):
+        return self._number
+
+    def __repr__(self):
+        return self.to_str_format()
+
+    def __str__(self):
+        return self.to_colon_format()

uncloud/hack/main.py

@@ -0,0 +1,92 @@
+import argparse
+import logging
+
+from uncloud.hack.vm import VM
+from uncloud.hack.config import Config
+from uncloud.hack.mac import MAC
+from uncloud.hack.net import VXLANBridge, DNSRA
+
+from uncloud import UncloudException
+
+arg_parser = argparse.ArgumentParser('hack', add_help=False)
+                                     #description="Commands that are unfinished - use at own risk")
+arg_parser.add_argument('--last-used-mac', action='store_true')
+arg_parser.add_argument('--get-new-mac', action='store_true')
+
+arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true')
+arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true')
+arg_parser.add_argument('--network', help="/64 IPv6 network")
+arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0")
+arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int)
+arg_parser.add_argument('--run-dns-ra', action='store_true',
+                        help="Provide router advertisements and DNS resolution via dnsmasq")
+arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true')
+
+arg_parser.add_argument('--create-vm', action='store_true')
+arg_parser.add_argument('--destroy-vm', action='store_true')
+arg_parser.add_argument('--get-vm-status', action='store_true')
+arg_parser.add_argument('--get-vm-vnc', action='store_true')
+arg_parser.add_argument('--list-vms', action='store_true')
+arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int)
+arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int)
+arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image")
+arg_parser.add_argument('--uuid', help="VM UUID")
+
+arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true')
+arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh")
+
+
+log = logging.getLogger(__name__)
+
+
+def main(arguments):
+    config = Config(arguments)
+
+    if arguments['create_vm']:
+        vm = VM(config)
+        vm.create()
+
+    if arguments['destroy_vm']:
+        vm = VM(config)
+        vm.stop()
+
+    if arguments['get_vm_status']:
+        vm = VM(config)
+        vm.status()
+
+    if arguments['get_vm_vnc']:
+        vm = VM(config)
+        vm.vnc_addr()
+
+    if arguments['list_vms']:
+        vm = VM(config)
+        vm.list()
+
+    if arguments['last_used_mac']:
+        m = MAC(config)
+        print(m.last_used_mac())
+
+    if arguments['get_new_mac']:
+        print(MAC(config).get_next())
+
+    #if arguments['init_network']:
+    if arguments['create_vxlan']:
+        if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']:
+            raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)")
+        vb = VXLANBridge(vni=arguments['vni'],
+                         route=arguments['network'],
+                         uplinkdev=arguments['vxlan_uplink_device'],
+                         use_sudo=arguments['use_sudo'])
+        vb._setup_vxlan()
+        vb._setup_bridge()
+        vb._add_vxlan_to_bridge()
+        vb._route_network()
+
+    if arguments['run_dns_ra']:
+        if not arguments['network'] or not arguments['vni']:
+            raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)")
+
+        dnsra = DNSRA(route=arguments['network'],
+                      vni=arguments['vni'],
+                      use_sudo=arguments['use_sudo'])
+        dnsra._setup_dnsmasq()

uncloud/hack/net.py

@@ -0,0 +1,116 @@
+import subprocess
+import ipaddress
+import logging
+
+
+from uncloud import UncloudException
+
+log = logging.getLogger(__name__)
+
+
+class VXLANBridge(object):
+    cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5"
+    cmd_up_dev = "{sudo}ip link set {dev} up"
+    cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge"
+    cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up"
+    cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}"
+    cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}"
+
+    # VXLAN ids are at maximum 24 bit - use a /104
+    multicast_network = ipaddress.IPv6Network("ff05::/104")
+    max_vni = (2**24)-1
+
+    def __init__(self,
+                 vni,
+                 uplinkdev,
+                 route=None,
+                 use_sudo=False):
+        self.config = {}
+
+        if vni > self.max_vni:
+            raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni))
+
+        if use_sudo:
+            self.config['sudo'] = 'sudo '
+        else:
+            self.config['sudo'] = ''
+
+        self.config['vni_dec'] = vni
+        self.config['vni_hex'] = "{:x}".format(vni)
+        self.config['multicast_address'] = self.multicast_network[vni]
+
+        self.config['route_network'] = ipaddress.IPv6Network(route)
+        self.config['route'] = route
+
+        self.config['uplinkdev'] = uplinkdev
+        self.config['vxlandev'] = "vx{}".format(self.config['vni_hex'])
+        self.config['bridgedev'] = "br{}".format(self.config['vni_hex'])
+
+
+    def setup_networking(self):
+        pass
+
+    def _setup_vxlan(self):
+        self._execute_cmd(self.cmd_create_vxlan)
+        self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev'])
+
+    def _setup_bridge(self):
+        self._execute_cmd(self.cmd_create_bridge)
+        self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev'])
+
+    def _route_network(self):
+        self._execute_cmd(self.cmd_add_route_dev)
+
+    def _add_vxlan_to_bridge(self):
+        self._execute_cmd(self.cmd_add_to_bridge)
+
+    def _execute_cmd(self, cmd_string, **kwargs):
+        cmd = cmd_string.format(**self.config, **kwargs)
+        log.info("Executing: {}".format(cmd))
+        subprocess.run(cmd.split())
+
+class ManagementBridge(VXLANBridge):
+    pass
+
+
+class DNSRA(object):
+    # VXLAN ids are at maximum 24 bit
+    max_vni = (2**24)-1
+
+
+    # Command to start dnsmasq
+    cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra"
+
+    def __init__(self,
+                 vni,
+                 route=None,
+                 use_sudo=False):
+        self.config = {}
+
+        if vni > self.max_vni:
+            raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni))
+
+        if use_sudo:
+            self.config['sudo'] = 'sudo '
+        else:
+            self.config['sudo'] = ''
+
+        #TODO: remove if not needed
+        #self.config['vni_dec'] = vni
+        self.config['vni_hex'] = "{:x}".format(vni)
+
+        # dnsmasq only wants the network without the prefix, therefore, cut it off
+        self.config['route'] = ipaddress.IPv6Network(route).network_address
+        self.config['bridgedev'] = "br{}".format(self.config['vni_hex'])
+
+    def _setup_dnsmasq(self):
+         self._execute_cmd(self.cmd_start_dnsmasq)
+
+    def _execute_cmd(self, cmd_string, **kwargs):
+        cmd = cmd_string.format(**self.config, **kwargs)
+        log.info("Executing: {}".format(cmd))
+        print("Executing: {}".format(cmd))
+        subprocess.run(cmd.split())
+
+class Firewall(object):
+    pass

uncloud/hack/uncloud-hack-init-host

@@ -0,0 +1,26 @@
+id=100
+rawdev=eth0
+
+# create vxlan
+ip -6 link add vxlan${id} type vxlan \
+    id ${id} \
+    dstport 4789 \
+    group ff05::${id} \
+    dev ${rawdev} \
+    ttl 5
+
+ip link set vxlan${id} up
+
+# create bridge
+ip link set vxlan${id} up
+ip link set br${id} up
+
+# Add vxlan into bridge
+ip link set vxlan${id} master br${id}
+
+
+#  useradd -m uncloud
+# [18:05] tablett.place10:~# id uncloud
+# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu)
+# apk add qemu-system-x86_64
+# also needs group netdev

uncloud/hack/uncloud-run-vm

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+if [ $# -ne 1 ]; then
+    echo $0 vmid
+    exit 1
+fi
+
+id=$1; shift
+
+memory=512
+macaddress=02:00:b9:cb:70:${id}
+netname=net${id}-1
+
+qemu-system-x86_64 \
+    -name uncloud-${id} \
+    -accel kvm \
+    -m ${memory} \
+    -smp 2,sockets=2,cores=1,threads=1 \
+    -device virtio-net-pci,netdev=net0,mac=$macaddress \
+    -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \
+    -vnc [::]:0
+
+# To be changed:
+# -vnc to unix path
+# or -spice

uncloud/hack/vm.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
+#
+# This file is part of uncloud.
+#
+# uncloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uncloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+
+# This module is directly called from the hack module, and can be used as follow:
+#
+#   Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix.
+#   `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2`
+#
+#   List running VMs (returns a list of UUIDs).
+#   `uncloud hack --hackprefix /tmp/hackcloud --list-vms
+#
+#   Get VM status:
+#   `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid`
+#
+#   Stop a VM:
+#   `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid`
+#    ``
+
+import subprocess
+import uuid
+import os
+import logging
+
+from uncloud.hack.db import DB
+from uncloud.hack.mac import MAC
+from uncloud.vmm import VMM
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.DEBUG)
+
+class VM(object):
+    def __init__(self, config):
+        self.config = config
+
+        #TODO: Enable etcd lookup
+        self.no_db = self.config.arguments['no_db']
+        if not self.no_db:
+             self.db = DB(self.config, prefix="/vm")
+
+        # General CLI arguments.
+        self.hackprefix = self.config.arguments['hackprefix']
+        self.uuid = self.config.arguments['uuid']
+        self.memory = self.config.arguments['memory'] or '1024M'
+        self.cores = self.config.arguments['cores'] or 1
+        if self.config.arguments['image']:
+            self.image = os.path.join(self.hackprefix, self.config.arguments['image'])
+        else:
+            self.image = None
+
+        # External components.
+        self.vmm = VMM(vmm_backend=self.hackprefix)
+        self.mac = MAC(self.config)
+
+        # Harcoded & generated values.
+        self.owner = 'uncoud'
+        self.image_format='qcow2'
+        self.accel = 'kvm'
+        self.threads = 1
+        self.ifup = os.path.join(self.hackprefix, "ifup.sh")
+        self.ifdown = os.path.join(self.hackprefix, "ifdown.sh")
+        self.ifname = "uc{}".format(self.mac.to_str_format())
+
+    def get_qemu_args(self):
+        command = (
+            "-name {owner}-{name}"
+            " -machine pc,accel={accel}"
+            " -drive file={image},format={image_format},if=virtio"
+            " -device virtio-rng-pci"
+            " -m {memory} -smp cores={cores},threads={threads}"
+            " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}"
+            " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}"
+        ).format(
+            owner=self.owner, name=self.uuid,
+            accel=self.accel,
+            image=self.image, image_format=self.image_format,
+            memory=self.memory, cores=self.cores, threads=self.threads,
+            ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname,
+            mac=self.mac
+        )
+
+        return command.split(" ")
+
+    def create(self):
+        # New VM: new UUID, new MAC.
+        self.uuid = str(uuid.uuid4())
+        self.mac.create()
+
+        qemu_args = self.get_qemu_args()
+        log.debug("QEMU args passed to VMM: {}".format(qemu_args))
+        self.vmm.start(
+                uuid=self.uuid,
+                migration=False,
+                *qemu_args
+                )
+
+    def stop(self):
+        if not self.uuid:
+            print("Please specific an UUID with the --uuid flag.")
+            exit(1)
+
+        self.vmm.stop(self.uuid)
+
+    def status(self):
+        if not self.uuid:
+            print("Please specific an UUID with the --uuid flag.")
+            exit(1)
+
+        print(self.vmm.get_status(self.uuid))
+
+    def vnc_addr(self):
+        if not self.uuid:
+            print("Please specific an UUID with the --uuid flag.")
+            exit(1)
+
+        print(self.vmm.get_vnc(self.uuid))
+
+    def list(self):
+        print(self.vmm.discover())
+

uncloud/host/main.py

@@ -1,17 +1,20 @@
 import argparse
 import multiprocessing as mp
 import time
+
 from uuid import uuid4
 
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
 from uncloud.common.vm import VMStatus
 from uncloud.vmm import VMM
 from os.path import join as join_path
 
 from . import virtualmachine, logger
 
+arg_parser = argparse.ArgumentParser('host', add_help=False)
+arg_parser.add_argument('--hostname', required=True)
+
 
 def update_heartbeat(hostname):
     """Update Last HeartBeat Time for :param hostname: in etcd"""
@@ -29,10 +32,10 @@ def maintenance(host):
     vmm = VMM()
     running_vms = vmm.discover()
     for vm_uuid in running_vms:
-        if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running":
+        if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
             logger.debug('VM {} is running on {}'.format(vm_uuid, host))
             vm = shared.vm_pool.get(
-                join_path(settings["etcd"]["vm_prefix"], vm_uuid)
+                join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
             )
             vm.status = VMStatus.running
             vm.vnc_socket = vmm.get_vnc(vm_uuid)
@@ -40,77 +43,81 @@ def maintenance(host):
             shared.vm_pool.put(vm)
 
 
-def main(hostname):
+def main(arguments):
+    hostname = arguments['hostname']
     host_pool = shared.host_pool
     host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
 
     # Does not yet exist, create it
     if not host:
         host_key = join_path(
-            settings["etcd"]["host_prefix"], uuid4().hex
+            shared.settings['etcd']['host_prefix'], uuid4().hex
         )
         host_entry = {
-            "specs": "",
+            'specs': '',
-            "hostname": hostname,
+            'hostname': hostname,
-            "status": "DEAD",
+            'status': 'DEAD',
-            "last_heartbeat": "",
+            'last_heartbeat': '',
         }
         shared.etcd_client.put(
             host_key, host_entry, value_in_json=True
         )
 
+    # update, get ourselves now for sure
+    host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
+
     try:
         heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,))
         heartbeat_updating_process.start()
     except Exception as e:
-        raise Exception("uncloud-host heartbeat updating mechanism is not working") from e
+        raise Exception('uncloud-host heartbeat updating mechanism is not working') from e
 
-    for events_iterator in [
+    # The below while True is neccessary for gracefully handling leadership transfer and temporary
-        shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True),
+    # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
-        shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True)
+    # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
-    ]:
+    # which ends the loop immediately. So, having it inside infinite loop we try again and again to
-        for request_event in events_iterator:
+    # get prefix until either success or deamon death comes.
-            request_event = RequestEntry(request_event)
+    while True:
+        for events_iterator in [
+            shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
+                                          raise_exception=False),
+            shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
+                                            raise_exception=False)
+        ]:
+            for request_event in events_iterator:
+                request_event = RequestEntry(request_event)
 
-            if request_event.type == "TIMEOUT":
                 maintenance(host.key)
 
-            elif request_event.hostname == host.key:
+                if request_event.hostname == host.key:
-                logger.debug("VM Request: %s on Host %s", request_event, host.hostname)
+                    logger.debug('VM Request: %s on Host %s', request_event, host.hostname)
-                shared.request_pool.client.client.delete(request_event.key)
-                vm_entry = shared.etcd_client.get(
-                    join_path(settings["etcd"]["vm_prefix"], request_event.uuid)
-                )
-                logger.debug("VM hostname: {}".format(vm_entry.value))
-                vm = virtualmachine.VM(vm_entry)
-                if request_event.type == RequestType.StartVM:
-                    vm.start()
 
-                elif request_event.type == RequestType.StopVM:
+                    shared.request_pool.client.client.delete(request_event.key)
-                    vm.stop()
+                    vm_entry = shared.etcd_client.get(
+                        join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid)
+                    )
 
-                elif request_event.type == RequestType.DeleteVM:
+                    logger.debug('VM hostname: {}'.format(vm_entry.value))
-                    vm.delete()
 
-                elif request_event.type == RequestType.InitVMMigration:
+                    vm = virtualmachine.VM(vm_entry)
-                    vm.start(destination_host_key=host.key)
+                    if request_event.type == RequestType.StartVM:
+                        vm.start()
 
-                elif request_event.type == RequestType.TransferVM:
+                    elif request_event.type == RequestType.StopVM:
-                    destination_host = host_pool.get(request_event.destination_host_key)
+                        vm.stop()
-                    if destination_host:
-                        vm.migrate(
-                            destination_host=destination_host.hostname,
-                            destination_sock_path=request_event.destination_sock_path,
-                        )
-                    else:
-                        logger.error("Host %s not found!", request_event.destination_host_key)
 
+                    elif request_event.type == RequestType.DeleteVM:
+                        vm.delete()
 
-if __name__ == "__main__":
+                    elif request_event.type == RequestType.InitVMMigration:
-    argparser = argparse.ArgumentParser()
+                        vm.start(destination_host_key=host.key)
-    argparser.add_argument(
+
-        "hostname", help="Name of this host. e.g uncloud1.ungleich.ch"
+                    elif request_event.type == RequestType.TransferVM:
-    )
+                        destination_host = host_pool.get(request_event.destination_host_key)
-    args = argparser.parse_args()
+                        if destination_host:
-    mp.set_start_method("spawn")
+                            vm.migrate(
-    main(args.hostname)
+                                destination_host=destination_host.hostname,
+                                destination_sock_path=request_event.destination_sock_path,
+                            )
+                        else:
+                            logger.error('Host %s not found!', request_event.destination_host_key)

uncloud/host/virtualmachine.py

@@ -16,8 +16,7 @@ from uncloud.common.vm import VMStatus, declare_stopped
 from uncloud.common.network import create_dev, delete_network_interface
 from uncloud.common.schemas import VMSchema, NetworkSchema
 from uncloud.host import logger
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
 from uncloud.vmm import VMM
 
 from marshmallow import ValidationError
@@ -42,7 +41,7 @@ class VM:
 
     def get_qemu_args(self):
         command = (
-            "-drive file={file},format=raw,if=virtio,cache=none"
+            "-drive file={file},format=raw,if=virtio"
             " -device virtio-rng-pci"
             " -m {memory} -smp cores={cores},threads={threads}"
             " -name {owner}_{name}"
@@ -91,7 +90,7 @@ class VM:
                         self.vmm.socket_dir, self.uuid
                     ),
                     destination_host_key=destination_host_key,  # Where source host transfer VM
-                    request_prefix=settings["etcd"]["request_prefix"],
+                    request_prefix=shared.settings["etcd"]["request_prefix"],
                 )
                 shared.request_pool.put(r)
             else:
@@ -119,7 +118,7 @@ class VM:
             network_name, mac, tap = network_mac_and_tap
 
             _key = os.path.join(
-                settings["etcd"]["network_prefix"],
+                shared.settings["etcd"]["network_prefix"],
                 self.vm["owner"],
                 network_name,
             )
@@ -133,13 +132,13 @@ class VM:
             if network["type"] == "vxlan":
                 tap = create_vxlan_br_tap(
                     _id=network["id"],
-                    _dev=settings["network"]["vxlan_phy_dev"],
+                    _dev=shared.settings["network"]["vxlan_phy_dev"],
                     tap_id=tap,
                     ip=network["ipv6"],
                 )
 
                 all_networks = shared.etcd_client.get_prefix(
-                    settings["etcd"]["network_prefix"],
+                    shared.settings["etcd"]["network_prefix"],
                     value_in_json=True,
                 )
 
@@ -153,7 +152,10 @@ class VM:
                 )
             )
 
-        return command.split(" ")
+        if command:
+            command = command.split(' ')
+
+        return command
 
     def delete_network_dev(self):
         try:
@@ -226,7 +228,7 @@ class VM:
 def resolve_network(network_name, network_owner):
     network = shared.etcd_client.get(
         join_path(
-            settings["etcd"]["network_prefix"],
+            shared.settings["etcd"]["network_prefix"],
             network_owner,
             network_name,
         ),

uncloud/imagescanner/main.py

@@ -1,13 +1,16 @@
 import json
 import os
+import argparse
 import subprocess as sp
 
 from os.path import join as join_path
-from uncloud.settings import settings
+from uncloud.common.shared import shared
-from uncloud.shared import shared
 from uncloud.imagescanner import logger
 
 
+arg_parser = argparse.ArgumentParser('imagescanner', add_help=False)
+
+
 def qemu_img_type(path):
     qemu_img_info_command = [
         "qemu-img",
@@ -26,10 +29,10 @@ def qemu_img_type(path):
         return qemu_img_info["format"]
 
 
-def main():
+def main(arguments):
     # We want to get images entries that requests images to be created
     images = shared.etcd_client.get_prefix(
-        settings["etcd"]["image_prefix"], value_in_json=True
+        shared.settings["etcd"]["image_prefix"], value_in_json=True
     )
     images_to_be_created = list(
         filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
@@ -42,13 +45,13 @@ def main():
             image_filename = image.value["filename"]
             image_store_name = image.value["store_name"]
             image_full_path = join_path(
-                settings["storage"]["file_dir"],
+                shared.settings["storage"]["file_dir"],
                 image_owner,
                 image_filename,
             )
 
             image_stores = shared.etcd_client.get_prefix(
-                settings["etcd"]["image_store_prefix"],
+                shared.settings["etcd"]["image_store_prefix"],
                 value_in_json=True,
             )
             user_image_store = next(

uncloud/metadata/main.py

@@ -1,17 +1,22 @@
 import os
+import argparse
 
 from flask import Flask, request
 from flask_restful import Resource, Api
 from werkzeug.exceptions import HTTPException
 
-from uncloud.settings import settings
+from uncloud.common.shared import shared
-from uncloud.shared import shared
 
 app = Flask(__name__)
 api = Api(app)
 
 app.logger.handlers.clear()
 
+DEFAULT_PORT=1234
+
+arg_parser = argparse.ArgumentParser('metadata', add_help=False)
+arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT))
+
 
 @app.errorhandler(Exception)
 def handle_exception(e):
@@ -68,7 +73,7 @@ class Root(Resource):
             )
         else:
             etcd_key = os.path.join(
-                settings["etcd"]["user_prefix"],
+                shared.settings["etcd"]["user_prefix"],
                 data.value["owner_realm"],
                 data.value["owner"],
                 "key",
@@ -80,40 +85,11 @@ class Root(Resource):
             data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys
             return data.value["metadata"], 200
 
-    @staticmethod
-    def post():
-        return {"message": "Previous Implementation is deprecated."}
-        # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True)
-        # print(data)
-        # if data:
-        #     for k in request.json:
-        #         if k not in data.value:
-        #             data.value[k] = request.json[k]
-        #             if k.endswith("-list"):
-        #                 data.value[k] = [request.json[k]]
-        #         else:
-        #             if k.endswith("-list"):
-        #                 data.value[k].append(request.json[k])
-        #             else:
-        #                 data.value[k] = request.json[k]
-        #     etcd_client.put("/v1/metadata/{}".format(request.remote_addr),
-        #                     data.value, value_in_json=True)
-        # else:
-        #     data = {}
-        #     for k in request.json:
-        #         data[k] = request.json[k]
-        #         if k.endswith("-list"):
-        #             data[k] = [request.json[k]]
-        #     etcd_client.put("/v1/metadata/{}".format(request.remote_addr),
-        #                     data, value_in_json=True)
-
 
 api.add_resource(Root, "/")
 
 
-def main():
+def main(arguments):
-    app.run(debug=True, host="::", port="80")
+    port = arguments['port']
-
+    debug = arguments['debug']
-
+    app.run(debug=debug, host="::", port=port)
-if __name__ == "__main__":
-    main()

uncloud/oneshot/__init__.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger(__name__)

uncloud/oneshot/main.py

@@ -0,0 +1,123 @@
+import argparse
+import os
+
+
+from pathlib import Path
+from uncloud.vmm import VMM
+from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap
+
+from . import virtualmachine, logger
+
+###
+# Argument parser loaded by scripts/uncloud.
+arg_parser = argparse.ArgumentParser('oneshot', add_help=False)
+
+# Actions.
+arg_parser.add_argument('--list', action='store_true',
+        help='list UUID and name of running VMs')
+arg_parser.add_argument('--start', nargs=3,
+        metavar=('IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'),
+        help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix')
+arg_parser.add_argument('--stop', metavar='UUID',
+        help='stop a VM')
+arg_parser.add_argument('--get-status', metavar='UUID',
+        help='return the status of the VM')
+arg_parser.add_argument('--get-vnc', metavar='UUID',
+        help='return the path of the VNC socket of the VM')
+arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK',
+        help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix')
+
+# Arguments.
+arg_parser.add_argument('--workdir', default=Path.home(),
+        help='Working directory, defaulting to $HOME')
+arg_parser.add_argument('--mac',
+        help='MAC address of the VM to create (--start)')
+arg_parser.add_argument('--memory', type=int,
+        help='Memory (MB) to allocate (--start)')
+arg_parser.add_argument('--cores', type=int,
+        help='Number of cores to allocate (--start)')
+arg_parser.add_argument('--threads', type=int,
+        help='Number of threads to allocate (--start)')
+arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'],
+        help='Format of OS image (--start)')
+arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm',
+        help='QEMU acceleration to use (--start)')
+arg_parser.add_argument('--upstream-interface', default='eth0',
+        help='Name of upstream interface (--start)')
+
+###
+# Helpers.
+
+# XXX: check if it is possible to use the type returned by ETCD queries.
+class UncloudEntryWrapper:
+    def __init__(self, value):
+        self.value = value
+
+    def value(self):
+        return self.value
+
+def status_line(vm):
+     return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status())
+
+###
+# Entrypoint.
+
+def main(arguments):
+    # Initialize VMM.
+    workdir = arguments['workdir']
+    vmm = VMM(vmm_backend=workdir)
+
+    # Harcoded debug values.
+    net_id = 0
+
+    # Build VM configuration.
+    vm_config = {}
+    vm_options = [
+            'mac', 'memory', 'cores', 'threads', 'image', 'image_format',
+            '--upstream_interface', 'upstream_interface', 'network', 'accel'
+            ]
+    for option in vm_options:
+        if arguments.get(option):
+            vm_config[option] = arguments[option]
+
+    vm_config['net_id'] = net_id
+
+    # Execute requested VM action.
+    if arguments['reconfigure_radvd']:
+        # TODO: check that RADVD is available.
+        prefix = arguments['reconfigure_radvd']
+        network = UncloudEntryWrapper({
+                'id': net_id,
+                'ipv6': prefix
+                })
+
+        # Make use of uncloud.host.virtualmachine for network configuration.
+        update_radvd_conf([network])
+    elif arguments['start']:
+        # Extract from --start positional arguments. Quite fragile.
+        vm_config['image'] = arguments['start'][0]
+        vm_config['network'] = arguments['start'][1]
+        vm_config['upstream_interface'] = arguments['start'][2]
+
+        vm_config['tap_interface'] = "uc{}".format(len(vmm.discover()))
+        vm = virtualmachine.VM(vmm, vm_config)
+        vm.start()
+    elif arguments['stop']:
+        vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']})
+        vm = virtualmachine.VM(vmm, vm_config)
+        vm.stop()
+    elif arguments['get_status']:
+        vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']})
+        print(status_line(vm))
+    elif arguments['get_vnc']:
+        vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']})
+        print(vm.get_vnc_addr())
+    elif arguments['list']:
+        vms = vmm.discover()
+        print("Found {} VMs.".format(len(vms)))
+        for uuid in vms:
+            vm = virtualmachine.VM(vmm, {'uuid': uuid})
+            print(status_line(vm))
+    else:
+        print('Please specify an action: --start, --stop, --list,\
+--get-status, --get-vnc, --reconfigure-radvd')

uncloud/oneshot/virtualmachine.py

@@ -0,0 +1,81 @@
+import uuid
+import os
+
+from uncloud.host.virtualmachine import create_vxlan_br_tap
+from uncloud.oneshot import logger
+
+class VM(object):
+    def __init__(self, vmm, config):
+        self.config = config
+        self.vmm = vmm
+
+        # Extract VM specs/metadata from configuration.
+        self.name         = config.get('name', 'no-name')
+        self.memory       = config.get('memory', 1024)
+        self.cores        = config.get('cores', 1)
+        self.threads      = config.get('threads', 1)
+        self.image_format = config.get('image_format', 'qcow2')
+        self.image        = config.get('image')
+        self.uuid         = config.get('uuid', str(uuid.uuid4()))
+        self.mac          = config.get('mac')
+        self.accel        = config.get('accel', 'kvm')
+
+        self.net_id             = config.get('net_id', 0)
+        self.upstream_interface = config.get('upstream_interface', 'eth0')
+        self.tap_interface      = config.get('tap_interface', 'uc0')
+        self.network            = config.get('network')
+
+    def get_qemu_args(self):
+        command = (
+            "-uuid {uuid} -name {name} -machine pc,accel={accel}"
+            " -drive file={image},format={image_format},if=virtio"
+            " -device virtio-rng-pci"
+            " -m {memory} -smp cores={cores},threads={threads}"
+            " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no"
+            " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}"
+        ).format(
+            uuid=self.uuid, name=self.name, accel=self.accel,
+            image=self.image, image_format=self.image_format,
+            memory=self.memory, cores=self.cores, threads=self.threads,
+            net_id=self.net_id, tap=self.tap_interface, mac=self.mac
+        )
+
+        return command.split(" ")
+
+    def start(self):
+        # Check that VM image is available.
+        if not os.path.isfile(self.image):
+            logger.error("Image {} does not exist. Aborting.".format(self.image))
+
+        # Create Bridge, VXLAN and tap interface for VM.
+        create_vxlan_br_tap(
+                self.net_id, self.upstream_interface, self.tap_interface, self.network
+                )
+
+        # Generate config for and run QEMU.
+        qemu_args = self.get_qemu_args()
+        logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args))
+        self.vmm.start(
+                uuid=self.uuid,
+                migration=False,
+                *qemu_args
+                )
+
+    def stop(self):
+        self.vmm.stop(self.uuid)
+
+    def get_status(self):
+        return self.vmm.get_status(self.uuid)
+
+    def get_vnc_addr(self):
+        return self.vmm.get_vnc(self.uuid)
+
+    def get_uuid(self):
+        return self.uuid
+
+    def get_name(self):
+        success, json = self.vmm.execute_command(uuid, 'query-name')
+        if success:
+            return json['return']['name']
+
+        return None

uncloud/scheduler/helper.py

@@ -6,8 +6,7 @@ import bitmath
 from uncloud.common.host import HostStatus
 from uncloud.common.request import RequestEntry, RequestType
 from uncloud.common.vm import VMStatus
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
 
 
 def accumulated_specs(vms_specs):
@@ -130,7 +129,7 @@ def assign_host(vm):
         type=RequestType.StartVM,
         uuid=vm.uuid,
         hostname=vm.hostname,
-        request_prefix=settings["etcd"]["request_prefix"],
+        request_prefix=shared.settings["etcd"]["request_prefix"],
     )
     shared.request_pool.put(r)
 

uncloud/scheduler/main.py

@@ -4,72 +4,48 @@
 #  2. Introduce a status endpoint of the scheduler -
 #     maybe expose a prometheus compatible output
 
+import argparse
+
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.shared import shared
+from uncloud.common.shared import shared
-from uncloud.settings import settings
+from uncloud.scheduler import logger
-from .helper import (
+from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection,
-    dead_host_mitigation,
+                                      assign_host, NoSuitableHostFound)
-    dead_host_detection,
+
-    assign_host,
+arg_parser = argparse.ArgumentParser('scheduler', add_help=False)
-    NoSuitableHostFound,
-)
-from . import logger
 
 
-def main():
+def main(arguments):
-    for request_iterator in [
+    # The below while True is neccessary for gracefully handling leadership transfer and temporary
-        shared.etcd_client.get_prefix(
+    # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
-            settings["etcd"]["request_prefix"], value_in_json=True
+    # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
-        ),
+    # which ends the loop immediately. So, having it inside infinite loop we try again and again to
-        shared.etcd_client.watch_prefix(
+    # get prefix until either success or deamon death comes.
-            settings["etcd"]["request_prefix"],
+    while True:
-            timeout=5,
+        for request_iterator in [
-            value_in_json=True,
+            shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
-        ),
+                                          raise_exception=False),
-    ]:
+            shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
-        for request_event in request_iterator:
+                                            raise_exception=False),
-            request_entry = RequestEntry(request_event)
+        ]:
-            # Never Run time critical mechanism inside timeout
+            for request_event in request_iterator:
-            # mechanism because timeout mechanism only comes
+                dead_host_mitigation(dead_host_detection())
-            # when no other event is happening. It means under
+                request_entry = RequestEntry(request_event)
-            # heavy load there would not be a timeout event.
-            if request_entry.type == "TIMEOUT":
 
-                # Detect hosts that are dead and set their status
+                if request_entry.type == RequestType.ScheduleVM:
-                # to "DEAD", and their VMs' status to "KILLED"
+                    logger.debug('%s, %s', request_entry.key, request_entry.value)
-                dead_hosts = dead_host_detection()
-                if dead_hosts:
-                    logger.debug("Dead hosts: %s", dead_hosts)
-                    dead_host_mitigation(dead_hosts)
 
-            elif request_entry.type == RequestType.ScheduleVM:
+                    vm_entry = shared.vm_pool.get(request_entry.uuid)
-                print(request_event.value)
+                    if vm_entry is None:
-                logger.debug(
+                        logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid))
-                    "%s, %s", request_entry.key, request_entry.value
+                        continue
-                )
 
-                vm_entry = shared.vm_pool.get(request_entry.uuid)
+                    shared.etcd_client.client.delete(request_entry.key)  # consume Request
-                if vm_entry is None:
-                    logger.info(
-                        "Trying to act on {} but it is deleted".format(
-                            request_entry.uuid
-                        )
-                    )
-                    continue
-                shared.etcd_client.client.delete(
-                    request_entry.key
-                )  # consume Request
 
-                try:
+                    try:
-                    assign_host(vm_entry)
+                        assign_host(vm_entry)
-                except NoSuitableHostFound:
+                    except NoSuitableHostFound:
-                    vm_entry.add_log(
+                        vm_entry.add_log('Can\'t schedule VM. No Resource Left.')
-                        "Can't schedule VM. No Resource Left."
+                        shared.vm_pool.put(vm_entry)
-                    )
-                    shared.vm_pool.put(vm_entry)
 
-                    logger.info("No Resource Left. Emailing admin....")
+                        logger.info('No Resource Left. Emailing admin....')
-
-
-if __name__ == "__main__":
-    main()

uncloud/settings/__init__.py

@@ -1,128 +0,0 @@
-import configparser
-import logging
-import sys
-import os
-
-from uncloud.common.etcd_wrapper import Etcd3Wrapper
-
-logger = logging.getLogger(__name__)
-
-
-class CustomConfigParser(configparser.RawConfigParser):
-    def __getitem__(self, key):
-        try:
-            result = super().__getitem__(key)
-        except KeyError as err:
-            raise KeyError(
-                "Key '{}' not found in configuration. Make sure you configure uncloud.".format(
-                    key
-                )
-            ) from err
-        else:
-            return result
-
-
-class Settings(object):
-    def __init__(self, config_key="/uncloud/config/"):
-        conf_name = "uncloud.conf"
-        conf_dir = os.environ.get(
-            "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/")
-        )
-        self.config_file = os.path.join(conf_dir, conf_name)
-
-        self.config_parser = CustomConfigParser(allow_no_value=True)
-        self.config_key = config_key
-
-        self.read_internal_values()
-        try:
-            self.config_parser.read(self.config_file)
-        except Exception as err:
-            logger.error("%s", err)
-
-    def get_etcd_client(self):
-        args = tuple()
-        try:
-            kwargs = {
-                "host": self.config_parser.get("etcd", "url"),
-                "port": self.config_parser.get("etcd", "port"),
-                "ca_cert": self.config_parser.get("etcd", "ca_cert"),
-                "cert_cert": self.config_parser.get(
-                    "etcd", "cert_cert"
-                ),
-                "cert_key": self.config_parser.get("etcd", "cert_key"),
-            }
-        except configparser.Error as err:
-            raise configparser.Error(
-                "{} in config file {}".format(
-                    err.message, self.config_file
-                )
-            ) from err
-        else:
-            try:
-                wrapper = Etcd3Wrapper(*args, **kwargs)
-            except Exception as err:
-                logger.error(
-                    "etcd connection not successfull. Please check your config file."
-                    "\nDetails: %s\netcd connection parameters: %s",
-                    err,
-                    kwargs,
-                )
-                sys.exit(1)
-            else:
-                return wrapper
-
-    def read_internal_values(self):
-        self.config_parser.read_dict(
-            {
-                "etcd": {
-                    "file_prefix": "/files/",
-                    "host_prefix": "/hosts/",
-                    "image_prefix": "/images/",
-                    "image_store_prefix": "/imagestore/",
-                    "network_prefix": "/networks/",
-                    "request_prefix": "/requests/",
-                    "user_prefix": "/users/",
-                    "vm_prefix": "/vms/",
-                }
-            }
-        )
-
-    def read_config_file_values(self, config_file):
-        try:
-            # Trying to read configuration file
-            with open(config_file, "r") as config_file_handle:
-                self.config_parser.read_file(config_file_handle)
-        except FileNotFoundError:
-            sys.exit(
-                "Configuration file {} not found!".format(config_file)
-            )
-        except Exception as err:
-            logger.exception(err)
-            sys.exit("Error occurred while reading configuration file")
-
-    def read_values_from_etcd(self):
-        etcd_client = self.get_etcd_client()
-        config_from_etcd = etcd_client.get(
-            self.config_key, value_in_json=True
-        )
-        if config_from_etcd:
-            self.config_parser.read_dict(config_from_etcd.value)
-        else:
-            raise KeyError(
-                "Key '{}' not found in etcd. Please configure uncloud.".format(
-                    self.config_key
-                )
-            )
-
-    def __getitem__(self, key):
-        # Allow failing to read from etcd if we have
-        # it locally
-        try:
-            self.read_values_from_etcd()
-        except KeyError as e:
-            pass
-
-        return self.config_parser[key]
-
-
-settings = Settings()

uncloud/shared/__init__.py

@@ -1,34 +0,0 @@
-from uncloud.settings import settings
-from uncloud.common.vm import VmPool
-from uncloud.common.host import HostPool
-from uncloud.common.request import RequestPool
-from uncloud.common.storage_handlers import get_storage_handler
-
-
-class Shared:
-    @property
-    def etcd_client(self):
-        return settings.get_etcd_client()
-
-    @property
-    def host_pool(self):
-        return HostPool(
-            self.etcd_client, settings["etcd"]["host_prefix"]
-        )
-
-    @property
-    def vm_pool(self):
-        return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"])
-
-    @property
-    def request_pool(self):
-        return RequestPool(
-            self.etcd_client, settings["etcd"]["request_prefix"]
-        )
-
-    @property
-    def storage_handler(self):
-        return get_storage_handler()
-
-
-shared = Shared()

uncloud/vmm/__init__.py

@@ -100,9 +100,9 @@ class TransferVM(Process):
 class VMM:
     # Virtual Machine Manager
     def __init__(
-        self,
+            self,
-        qemu_path="/usr/bin/qemu-system-x86_64",
+            qemu_path="/usr/bin/qemu-system-x86_64",
-        vmm_backend=os.path.expanduser("~/uncloud/vmm/"),
+            vmm_backend=os.path.expanduser("~/uncloud/vmm/"),
     ):
         self.qemu_path = qemu_path
         self.vmm_backend = vmm_backend
@@ -125,7 +125,7 @@ class VMM:
             os.makedirs(self.socket_dir, exist_ok=True)
 
     def is_running(self, uuid):
-        sock_path = os.path.join(self.vmm_backend, uuid)
+        sock_path = os.path.join(self.socket_dir, uuid)
         try:
             sock = socket.socket(socket.AF_UNIX)
             sock.connect(sock_path)
@@ -163,7 +163,7 @@ class VMM:
             qmp_arg = (
                 "-qmp",
                 "unix:{},server,nowait".format(
-                    join_path(self.vmm_backend, uuid)
+                    join_path(self.socket_dir, uuid)
                 ),
             )
             vnc_arg = (
@@ -190,18 +190,10 @@ class VMM:
                     err.stderr.decode("utf-8"),
                 )
             else:
-                with suppress(sp.CalledProcessError):
+                sp.check_output(
-                    sp.check_output(
+                    ["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory",
-                        [
+                     "chmod", "-R", "o=rwx,g=rwx", self.vmm_backend]
-                            "sudo",
+                )
-                            "-p",
-                            "Enter password to correct permission for uncloud-vmm's directory",
-                            "chmod",
-                            "-R",
-                            "o=rwx,g=rwx",
-                            self.vmm_backend,
-                        ]
-                    )
 
                 # TODO: Find some good way to check whether the virtual machine is up and
                 #       running without relying on non-guarenteed ways.
@@ -220,7 +212,7 @@ class VMM:
     def execute_command(self, uuid, command, **kwargs):
         # execute_command -> sucess?, output
         try:
-            with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as (
+            with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as (
                 sock_handle,
                 file_handle,
             ):
@@ -263,8 +255,8 @@ class VMM:
     def discover(self):
         vms = [
             uuid
-            for uuid in os.listdir(self.vmm_backend)
+            for uuid in os.listdir(self.socket_dir)
-            if not isdir(join_path(self.vmm_backend, uuid))
+            if not isdir(join_path(self.socket_dir, uuid))
         ]
         return vms