diff --git a/.gitignore b/.gitignore index 5c55899..6f0d9df 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ uncloud/version.py build/ venv/ dist/ + +*.iso diff --git a/bin/gen-version b/bin/gen-version index a2e2882..06c3e22 100755 --- a/bin/gen-version +++ b/bin/gen-version @@ -1,22 +1,22 @@ #!/bin/sh # -*- coding: utf-8 -*- # -# 2019 Nico Schottelius (nico-ucloud at schottelius.org) +# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org) # -# This file is part of ucloud. +# This file is part of uncloud. # -# ucloud is free software: you can redistribute it and/or modify +# uncloud is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# ucloud is distributed in the hope that it will be useful, +# uncloud is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with ucloud. If not, see . +# along with uncloud. If not, see . # # @@ -26,4 +26,4 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py +printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py diff --git a/bin/uncloud-run-reinstall b/bin/uncloud-run-reinstall index 18e95c0..b211613 100755 --- a/bin/uncloud-run-reinstall +++ b/bin/uncloud-run-reinstall @@ -24,6 +24,6 @@ dir=${0%/*} ${dir}/gen-version; -pip uninstall -y uncloud -python setup.py install +pip uninstall -y uncloud >/dev/null +python setup.py install >/dev/null ${dir}/uncloud "$@" diff --git a/conf/uncloud.conf b/conf/uncloud.conf index 334bbeb..6a1b500 100644 --- a/conf/uncloud.conf +++ b/conf/uncloud.conf @@ -1,7 +1,13 @@ [etcd] url = localhost port = 2379 - +base_prefix = / ca_cert cert_cert cert_key + +[client] +name = replace_me +realm = replace_me +seed = replace_me +api_server = http://localhost:5000 \ No newline at end of file diff --git a/uncloud/docs/Makefile b/docs/Makefile similarity index 93% rename from uncloud/docs/Makefile rename to docs/Makefile index 5e7ea85..246b56c 100644 --- a/uncloud/docs/Makefile +++ b/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source/ BUILDDIR = build/ -DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/ +DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/ .PHONY: all build clean diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..a5afbaa --- /dev/null +++ b/docs/README.md @@ -0,0 +1,12 @@ +# uncloud docs + +## Requirements +1. Python3 +2. Sphinx + +## Usage +Run `make build` to build docs. + +Run `make clean` to remove build directory. + +Run `make publish` to push build dir to https://ungleich.ch/ucloud/ \ No newline at end of file diff --git a/uncloud/docs/__init__.py b/docs/__init__.py similarity index 100% rename from uncloud/docs/__init__.py rename to docs/__init__.py diff --git a/uncloud/docs/source/__init__.py b/docs/source/__init__.py similarity index 100% rename from uncloud/docs/source/__init__.py rename to docs/source/__init__.py diff --git a/uncloud/docs/source/admin-guide b/docs/source/admin-guide.rst similarity index 72% rename from uncloud/docs/source/admin-guide rename to docs/source/admin-guide.rst index ec6597d..b62808d 100644 --- a/uncloud/docs/source/admin-guide +++ b/docs/source/admin-guide.rst @@ -56,40 +56,13 @@ To start host we created earlier, execute the following command ucloud host ungleich.ch -Create OS Image ---------------- +File & image scanners +-------------------------- -Create ucloud-init ready OS image (Optional) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This step is optional if you just want to test ucloud. However, sooner or later -you want to create OS images with ucloud-init to properly -contexualize VMs. - -1. Start a VM with OS image on which you want to install ucloud-init -2. Execute the following command on the started VM - - .. code-block:: sh - - apk add git - git clone https://code.ungleich.ch/ucloud/ucloud-init.git - cd ucloud-init - sh ./install.sh -3. Congratulations. Your image is now ucloud-init ready. - - -Upload Sample OS Image -~~~~~~~~~~~~~~~~~~~~~~ -Execute the following to get the sample OS image file. - -.. code-block:: sh - - mkdir /var/www/admin - (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download) - -Run File Scanner and Image Scanner -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make -images from tracked files. So, we need to track the file by running File Scanner +Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our +uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by +ucloud. We can only make images from tracked files. So, we need to track the +file by running File Scanner .. code-block:: sh diff --git a/uncloud/docs/source/conf.py b/docs/source/conf.py similarity index 100% rename from uncloud/docs/source/conf.py rename to docs/source/conf.py diff --git a/uncloud/docs/source/diagram-code/ucloud b/docs/source/diagram-code/ucloud similarity index 100% rename from uncloud/docs/source/diagram-code/ucloud rename to docs/source/diagram-code/ucloud diff --git a/docs/source/hacking.rst b/docs/source/hacking.rst new file mode 100644 index 0000000..1c750d6 --- /dev/null +++ b/docs/source/hacking.rst @@ -0,0 +1,36 @@ +Hacking +======= +Using uncloud in hacking (aka development) mode. + + +Get the code +------------ +.. code-block:: sh + :linenos: + + git clone https://code.ungleich.ch/uncloud/uncloud.git + + + +Install python requirements +--------------------------- +You need to have python3 installed. + +.. code-block:: sh + :linenos: + + cd uncloud! + python -m venv venv + . ./venv/bin/activate + ./bin/uncloud-run-reinstall + + + +Install os requirements +----------------------- +Install the following software packages: **dnsmasq**. + +If you already have a working IPv6 SLAAC and DNS setup, +this step can be skipped. + +Note that you need at least one /64 IPv6 network to run uncloud. diff --git a/uncloud/docs/source/images/ucloud.svg b/docs/source/images/ucloud.svg similarity index 100% rename from uncloud/docs/source/images/ucloud.svg rename to docs/source/images/ucloud.svg diff --git a/uncloud/docs/source/index.rst b/docs/source/index.rst similarity index 90% rename from uncloud/docs/source/index.rst rename to docs/source/index.rst index b31cff3..fad1f88 100644 --- a/uncloud/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,14 +11,13 @@ Welcome to ucloud's documentation! :caption: Contents: introduction - user-guide setup-install + vm-images + user-guide admin-guide - user-guide/how-to-create-an-os-image-for-ucloud troubleshooting hacking - Indices and tables ================== diff --git a/uncloud/docs/source/introduction.rst b/docs/source/introduction.rst similarity index 100% rename from uncloud/docs/source/introduction.rst rename to docs/source/introduction.rst diff --git a/uncloud/docs/source/misc/todo.rst b/docs/source/misc/todo.rst similarity index 100% rename from uncloud/docs/source/misc/todo.rst rename to docs/source/misc/todo.rst diff --git a/uncloud/docs/source/setup-install.rst b/docs/source/setup-install.rst similarity index 100% rename from uncloud/docs/source/setup-install.rst rename to docs/source/setup-install.rst diff --git a/uncloud/docs/source/theory/summary.rst b/docs/source/theory/summary.rst similarity index 100% rename from uncloud/docs/source/theory/summary.rst rename to docs/source/theory/summary.rst diff --git a/uncloud/docs/source/troubleshooting.rst b/docs/source/troubleshooting.rst similarity index 100% rename from uncloud/docs/source/troubleshooting.rst rename to docs/source/troubleshooting.rst diff --git a/uncloud/docs/source/user-guide.rst b/docs/source/user-guide.rst similarity index 100% rename from uncloud/docs/source/user-guide.rst rename to docs/source/user-guide.rst diff --git a/uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst b/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst similarity index 100% rename from uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst rename to docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst diff --git a/docs/source/vm-images.rst b/docs/source/vm-images.rst new file mode 100644 index 0000000..4b2758a --- /dev/null +++ b/docs/source/vm-images.rst @@ -0,0 +1,66 @@ +VM images +================================== + +Overview +--------- + +ucloud tries to be least invasise towards VMs and only require +strictly necessary changes for running in a virtualised +environment. This includes configurations for: + +* Configuring the network +* Managing access via ssh keys +* Resizing the attached disk(s) + +Upstream images +--------------- + +The 'official' uncloud images are defined in the `uncloud/images +`_ repository. + +How to make you own Uncloud images +---------------------------------- + +.. note:: + It is fairly easy to create your own images for uncloud, as the common + operations (which are detailed below) can be automatically handled by the + `uncloud/uncloud-init `_ tool. + +Network configuration +~~~~~~~~~~~~~~~~~~~~~ +All VMs in ucloud are required to support IPv6. The primary network +configuration is always done using SLAAC. A VM thus needs only to be +configured to + +* accept router advertisements on all network interfaces +* use the router advertisements to configure the network interfaces +* accept the DNS entries from the router advertisements + + +Configuring SSH keys +~~~~~~~~~~~~~~~~~~~~ + +To be able to access the VM, ucloud support provisioning SSH keys. + +To accept ssh keys in your VM, request the URL +*http://metadata/ssh_keys*. Add the content to the appropriate user's +**authorized_keys** file. Below you find sample code to accomplish +this task: + +.. code-block:: sh + + tmp=$(mktemp) + curl -s http://metadata/ssk_keys > "$tmp" + touch ~/.ssh/authorized_keys # ensure it exists + cat ~/.ssh/authorized_keys >> "$tmp" + sort "$tmp" | uniq > ~/.ssh/authorized_keys + + +Disk resize +~~~~~~~~~~~ +In virtualised environments, the disk sizes might grow. The operating +system should detect disks that are bigger than the existing partition +table and resize accordingly. This task is os specific. + +ucloud does not support shrinking disks due to the complexity and +intra OS dependencies. diff --git a/scripts/uncloud b/scripts/uncloud index 28d8344..7d38e42 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -1,70 +1,88 @@ #!/usr/bin/env python3 -import argparse import logging -import importlib -import multiprocessing as mp import sys +import importlib +import argparse +import os -from logging.handlers import SysLogHandler -from uncloud.configure.main import configure_parser +from etcd3.exceptions import ConnectionFailedError +from uncloud.common import settings from uncloud import UncloudException +from uncloud.common.cli import resolve_otp_credentials -def exception_hook(exc_type, exc_value, exc_traceback): - logging.getLogger(__name__).error( - 'Uncaught exception', - exc_info=(exc_type, exc_value, exc_traceback) - ) +# Components that use etcd +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', + 'imagescanner', 'metadata', 'configure', 'hack'] - -sys.excepthook = exception_hook +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('oneshot') +#ALL_COMPONENTS.append('cli') if __name__ == '__main__': - # Setting up root logger - logger = logging.getLogger() - - logger.setLevel(logging.DEBUG) + arg_parser = argparse.ArgumentParser() + subparsers = arg_parser.add_subparsers(dest='command') parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument("--debug", "-d", action='store_true') + parent_parser.add_argument('--debug', '-d', action='store_true', default=False, + help='More verbose logging') + parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory', + default=os.path.expanduser('~/uncloud')) - arg_parser = argparse.ArgumentParser() + etcd_parser = argparse.ArgumentParser(add_help=False) + etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-port') + etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') + etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') + etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') - subparsers = arg_parser.add_subparsers(dest="command") + for component in ALL_COMPONENTS: + mod = importlib.import_module('uncloud.{}.main'.format(component)) + parser = getattr(mod, 'arg_parser') - api_parser = subparsers.add_parser("api", parents=[parent_parser]) - api_parser.add_argument("--port", "-p") + if component in ETCD_COMPONENTS: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) + else: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - host_parser = subparsers.add_parser("host") - host_parser.add_argument("--hostname", required=True) - - scheduler_parser = subparsers.add_parser("scheduler") - filescanner_parser = subparsers.add_parser("filescanner") - imagescanner_parser = subparsers.add_parser("imagescanner") - metadata_parser = subparsers.add_parser("metadata") - config_parser = subparsers.add_parser("configure") - - configure_parser(config_parser) - args = arg_parser.parse_args() - - if not args.command: + arguments = vars(arg_parser.parse_args()) + etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value] + etcd_arguments = { + 'etcd': { + key.replace('etcd_', ''): arguments[key] + for key in etcd_arguments + } + } + if not arguments['command']: arg_parser.print_help() else: + # Initializing Settings and resolving otp_credentials + # It is neccessary to resolve_otp_credentials after argument parsing is done because + # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and + # providing the default values for --name, --realm and --seed arguments from the values + # we read from file. But, now we are asking user about where the config file lives. So, + # to providing default value is not possible before parsing arguments. So, we are doing + # it after.. +# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) +# resolve_otp_credentials(arguments) - # if we start etcd in seperate process with default settings - # i.e inheriting few things from parent process etcd3 module - # errors out, so the following command configure multiprocessing - # module to not inherit anything from parent. - mp.set_start_method('spawn') + name = arguments.pop('command') + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') + + if arguments['debug']: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + log = logging.getLogger() - arguments = vars(args) try: - name = arguments.pop('command') - mod = importlib.import_module("uncloud.{}.main".format(name)) - main = getattr(mod, "main") - main(**arguments) + main(arguments) except UncloudException as err: - logger.error(err) + log.error(err) +# except ConnectionFailedError as err: +# log.error('Cannot connect to etcd: {}'.format(err)) except Exception as err: - logger.exception(err) + log.exception(err) diff --git a/setup.py b/setup.py index 0764d74..12da6b8 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup( "pynetbox", "colorama", "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3", - "marshmallow", + "marshmallow" ], scripts=["scripts/uncloud"], data_files=[ diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py index 8bcf777..ba9fb37 100755 --- a/uncloud/api/common_fields.py +++ b/uncloud/api/common_fields.py @@ -1,7 +1,6 @@ import os -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared class Optional: @@ -54,9 +53,7 @@ class VmUUIDField(Field): def vm_uuid_validation(self): r = shared.etcd_client.get( - os.path.join(settings["etcd"]["vm_prefix"], self.uuid) + os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid) ) if not r: - self.add_error( - "VM with uuid {} does not exists".format(self.uuid) - ) + self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 73b92f1..90e0f92 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -3,18 +3,17 @@ import os from uuid import uuid4 -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared data = { - "is_public": True, - "type": "ceph", - "name": "images", - "description": "first ever public image-store", - "attributes": {"list": [], "key": [], "pool": "images"}, + 'is_public': True, + 'type': 'ceph', + 'name': 'images', + 'description': 'first ever public image-store', + 'attributes': {'list': [], 'key': [], 'pool': 'images'}, } shared.etcd_client.put( - os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex), + os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data), ) diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index c806814..8ceb3a6 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -1,14 +1,12 @@ import binascii import ipaddress import random -import subprocess as sp import logging import requests from pyotp import TOTP -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared logger = logging.getLogger(__name__) @@ -16,9 +14,9 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": settings["otp"]["auth_name"], - "auth_token": TOTP(settings["otp"]["auth_seed"]).now(), - "auth_realm": settings["otp"]["auth_realm"], + "auth_name": shared.settings["otp"]["auth_name"], + "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), + "auth_realm": shared.settings["otp"]["auth_realm"], "name": name, "realm": realm, "token": token, @@ -26,13 +24,13 @@ def check_otp(name, realm, token): except binascii.Error as err: logger.error( "Cannot compute OTP for seed: {}".format( - settings["otp"]["auth_seed"] + shared.settings["otp"]["auth_seed"] ) ) return 400 response = requests.post( - settings["otp"]["verification_controller_url"], json=data + shared.settings["otp"]["verification_controller_url"], json=data ) return response.status_code @@ -88,7 +86,7 @@ def resolve_image_name(name, etcd_client): ) images = etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) # Try to find image with name == image_name and store_name == store_name @@ -112,9 +110,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -149,3 +145,4 @@ def mac2ipv6(mac, prefix): lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) prefix = ipaddress.IPv6Address(prefix) return str(prefix + int(lower_part)) + diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 93bada7..73e8e21 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -1,6 +1,7 @@ import json import pynetbox import logging +import argparse from uuid import uuid4 from os.path import join as join_path @@ -9,14 +10,13 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException +from uncloud.common.shared import shared + from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType -from uncloud.settings import settings -from uncloud.shared import shared - -from . import schemas -from .helper import generate_mac, mac2ipv6 +from uncloud.api import schemas +from uncloud.api.helper import generate_mac, mac2ipv6 from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -25,6 +25,9 @@ app = Flask(__name__) api = Api(app) app.logger.handlers.clear() +arg_parser = argparse.ArgumentParser('api', add_help=False) +arg_parser.add_argument('--port', '-p') + @app.errorhandler(Exception) def handle_exception(e): @@ -34,7 +37,7 @@ def handle_exception(e): return e # now you're handling non-HTTP exceptions only - return {"message": "Server Error"}, 500 + return {'message': 'Server Error'}, 500 class CreateVM(Resource): @@ -46,33 +49,33 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(settings["etcd"]["vm_prefix"], vm_uuid) + vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) specs = { - "cpu": validator.specs["cpu"], - "ram": validator.specs["ram"], - "os-ssd": validator.specs["os-ssd"], - "hdd": validator.specs["hdd"], + 'cpu': validator.specs['cpu'], + 'ram': validator.specs['ram'], + 'os-ssd': validator.specs['os-ssd'], + 'hdd': validator.specs['hdd'], } - macs = [generate_mac() for _ in range(len(data["network"]))] + macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, "/v1/counter/tap" + shared.etcd_client, shared.settings['etcd']['tap_counter'] ) - for _ in range(len(data["network"])) + for _ in range(len(data['network'])) ] vm_entry = { - "name": data["vm_name"], - "owner": data["name"], - "owner_realm": data["realm"], - "specs": specs, - "hostname": "", - "status": VMStatus.stopped, - "image_uuid": validator.image_uuid, - "log": [], - "vnc_socket": "", - "network": list(zip(data["network"], macs, tap_ids)), - "metadata": {"ssh-keys": []}, - "in_migration": False, + 'name': data['vm_name'], + 'owner': data['name'], + 'owner_realm': data['realm'], + 'specs': specs, + 'hostname': '', + 'status': VMStatus.stopped, + 'image_uuid': validator.image_uuid, + 'log': [], + 'vnc_socket': '', + 'network': list(zip(data['network'], macs, tap_ids)), + 'metadata': {'ssh-keys': []}, + 'in_migration': False, } shared.etcd_client.put(vm_key, vm_entry, value_in_json=True) @@ -80,39 +83,39 @@ class CreateVM(Resource): r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) - return {"message": "VM Creation Queued"}, 200 + return {'message': 'VM Creation Queued'}, 200 return validator.get_errors(), 400 class VmStatus(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], data["uuid"]) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) vm_value = vm.value.copy() - vm_value["ip"] = [] + vm_value['ip'] = [] for network_mac_and_tap in vm.network: network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], - data["name"], + shared.settings['etcd']['network_prefix'], + data['name'], network_name, ), value_in_json=True, ) ipv6_addr = ( - network.value.get("ipv6").split("::")[0] + "::" + network.value.get('ipv6').split('::')[0] + '::' ) - vm_value["ip"].append(mac2ipv6(mac, ipv6_addr)) + vm_value['ip'].append(mac2ipv6(mac, ipv6_addr)) vm.value = vm_value return vm.value else: @@ -126,26 +129,26 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = shared.etcd_client.get( - join_path(settings["etcd"]["file_prefix"], data["uuid"]) + join_path(shared.settings['etcd']['file_prefix'], data['uuid']) ) file_entry_value = json.loads(file_entry.value) image_entry_json = { - "status": "TO_BE_CREATED", - "owner": file_entry_value["owner"], - "filename": file_entry_value["filename"], - "name": data["name"], - "store_name": data["image_store"], - "visibility": "public", + 'status': 'TO_BE_CREATED', + 'owner': file_entry_value['owner'], + 'filename': file_entry_value['filename'], + 'name': data['name'], + 'store_name': data['image_store'], + 'visibility': 'public', } shared.etcd_client.put( join_path( - settings["etcd"]["image_prefix"], data["uuid"] + shared.settings['etcd']['image_prefix'], data['uuid'] ), json.dumps(image_entry_json), ) - return {"message": "Image queued for creation."} + return {'message': 'Image queued for creation.'} return validator.get_errors(), 400 @@ -153,15 +156,15 @@ class ListPublicImages(Resource): @staticmethod def get(): images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings['etcd']['image_prefix'], value_in_json=True ) - r = {"images": []} + r = {'images': []} for image in images: - image_key = "{}:{}".format( - image.value["store_name"], image.value["name"] + image_key = '{}:{}'.format( + image.value['store_name'], image.value['name'] ) - r["images"].append( - {"name": image_key, "status": image.value["status"]} + r['images'].append( + {'name': image_key, 'status': image.value['status']} ) return r, 200 @@ -174,14 +177,14 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], data["uuid"]) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) - action = data["action"] + action = data['action'] - if action == "start": - action = "schedule" + if action == 'start': + action = 'schedule' - if action == "delete" and vm_entry.hostname == "": + if action == 'delete' and vm_entry.hostname == '': if shared.storage_handler.is_vm_image_exists( vm_entry.uuid ): @@ -190,25 +193,25 @@ class VMAction(Resource): ) if r_status: shared.etcd_client.client.delete(vm_entry.key) - return {"message": "VM successfully deleted"} + return {'message': 'VM successfully deleted'} else: logger.error( - "Some Error Occurred while deleting VM" + 'Some Error Occurred while deleting VM' ) - return {"message": "VM deletion unsuccessfull"} + return {'message': 'VM deletion unsuccessfull'} else: shared.etcd_client.client.delete(vm_entry.key) - return {"message": "VM successfully deleted"} + return {'message': 'VM successfully deleted'} r = RequestEntry.from_scratch( - type="{}VM".format(action.title()), - uuid=data["uuid"], + type='{}VM'.format(action.title()), + uuid=data['uuid'], hostname=vm_entry.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( - {"message": "VM {} Queued".format(action.title())}, + {'message': 'VM {} Queued'.format(action.title())}, 200, ) else: @@ -222,20 +225,20 @@ class VMMigration(Resource): validator = schemas.VmMigrationSchema(data) if validator.is_valid(): - vm = shared.vm_pool.get(data["uuid"]) + vm = shared.vm_pool.get(data['uuid']) r = RequestEntry.from_scratch( type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( - settings["etcd"]["host_prefix"], + shared.settings['etcd']['host_prefix'], validator.destination.value, ), - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( - {"message": "VM Migration Initialization Queued"}, + {'message': 'VM Migration Initialization Queued'}, 200, ) else: @@ -244,32 +247,32 @@ class VMMigration(Resource): class ListUserVM(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) if validator.is_valid(): vms = shared.etcd_client.get_prefix( - settings["etcd"]["vm_prefix"], value_in_json=True + shared.settings['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter( - lambda v: v.value["owner"] == data["name"], vms + lambda v: v.value['owner'] == data['name'], vms ) for vm in user_vms: return_vms.append( { - "name": vm.value["name"], - "vm_uuid": vm.key.split("/")[-1], - "specs": vm.value["specs"], - "status": vm.value["status"], - "hostname": vm.value["hostname"], - "vnc_socket": vm.value.get("vnc_socket", None), + 'name': vm.value['name'], + 'vm_uuid': vm.key.split('/')[-1], + 'specs': vm.value['specs'], + 'status': vm.value['status'], + 'hostname': vm.value['hostname'], + 'vnc_socket': vm.value.get('vnc_socket', None), } ) if return_vms: - return {"message": return_vms}, 200 - return {"message": "No VM found"}, 404 + return {'message': return_vms}, 200 + return {'message': 'No VM found'}, 404 else: return validator.get_errors(), 400 @@ -277,28 +280,26 @@ class ListUserVM(Resource): class ListUserFiles(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) if validator.is_valid(): files = shared.etcd_client.get_prefix( - settings["etcd"]["file_prefix"], value_in_json=True + shared.settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] - user_files = list( - filter( - lambda f: f.value["owner"] == data["name"], files - ) - ) + user_files = [f for f in files if f.value['owner'] == data['name']] for file in user_files: - return_files.append( - { - "filename": file.value["filename"], - "uuid": file.key.split("/")[-1], - } - ) - return {"message": return_files}, 200 + file_uuid = file.key.split('/')[-1] + file = file.value + file['uuid'] = file_uuid + + file.pop('sha512sum', None) + file.pop('owner', None) + + return_files.append(file) + return {'message': return_files}, 200 else: return validator.get_errors(), 400 @@ -310,19 +311,19 @@ class CreateHost(Resource): validator = schemas.CreateHostSchema(data) if validator.is_valid(): host_key = join_path( - settings["etcd"]["host_prefix"], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { - "specs": data["specs"], - "hostname": data["hostname"], - "status": "DEAD", - "last_heartbeat": "", + 'specs': data['specs'], + 'hostname': data['hostname'], + 'status': 'DEAD', + 'last_heartbeat': '', } shared.etcd_client.put( host_key, host_entry, value_in_json=True ) - return {"message": "Host Created"}, 200 + return {'message': 'Host Created'}, 200 return validator.get_errors(), 400 @@ -333,9 +334,9 @@ class ListHost(Resource): hosts = shared.host_pool.hosts r = { host.key: { - "status": host.status, - "specs": host.specs, - "hostname": host.hostname, + 'status': host.status, + 'specs': host.specs, + 'hostname': host.hostname, } for host in hosts } @@ -344,7 +345,7 @@ class ListHost(Resource): class GetSSHKeys(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.GetSSHSchema(data) if validator.is_valid(): @@ -352,29 +353,29 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', ) etcd_entry = shared.etcd_client.get_prefix( etcd_key, value_in_json=True ) keys = { - key.key.split("/")[-1]: key.value + key.key.split('/')[-1]: key.value for key in etcd_entry } - return {"keys": keys} + return {'keys': keys} else: # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True @@ -382,14 +383,14 @@ class GetSSHKeys(Resource): if etcd_entry: return { - "keys": { - etcd_entry.key.split("/")[ + 'keys': { + etcd_entry.key.split('/')[ -1 ]: etcd_entry.value } } else: - return {"keys": {}} + return {'keys': {}} else: return validator.get_errors(), 400 @@ -403,56 +404,56 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True ) if etcd_entry: return { - "message": "Key with name '{}' already exists".format( - data["key_name"] + 'message': 'Key with name "{}" already exists'.format( + data['key_name'] ) } else: # Key Not Found. It implies user' haven't added any key yet. shared.etcd_client.put( - etcd_key, data["key"], value_in_json=True + etcd_key, data['key'], value_in_json=True ) - return {"message": "Key added successfully"} + return {'message': 'Key added successfully'} else: return validator.get_errors(), 400 class RemoveSSHKey(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.RemoveSSHSchema(data) if validator.is_valid(): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True ) if etcd_entry: shared.etcd_client.client.delete(etcd_key) - return {"message": "Key successfully removed."} + return {'message': 'Key successfully removed.'} else: return { - "message": "No Key with name '{}' Exists at all.".format( - data["key_name"] + 'message': 'No Key with name "{}" Exists at all.'.format( + data['key_name'] ) } else: @@ -468,104 +469,107 @@ class CreateNetwork(Resource): if validator.is_valid(): network_entry = { - "id": counters.increment_etcd_counter( - shared.etcd_client, "/v1/counter/vxlan" + 'id': counters.increment_etcd_counter( + shared.etcd_client, shared.settings['etcd']['vxlan_counter'] ), - "type": data["type"], + 'type': data['type'], } if validator.user.value: try: nb = pynetbox.api( - url=settings["netbox"]["url"], - token=settings["netbox"]["token"], + url=shared.settings['netbox']['url'], + token=shared.settings['netbox']['token'], ) nb_prefix = nb.ipam.prefixes.get( - prefix=settings["network"]["prefix"] + prefix=shared.settings['network']['prefix'] ) prefix = nb_prefix.available_prefixes.create( data={ - "prefix_length": int( - settings["network"]["prefix_length"] + 'prefix_length': int( + shared.settings['network']['prefix_length'] ), - "description": '{}\'s network "{}"'.format( - data["name"], data["network_name"] + 'description': '{}\'s network "{}"'.format( + data['name'], data['network_name'] ), - "is_pool": True, + 'is_pool': True, } ) except Exception as err: app.logger.error(err) return { - "message": "Error occured while creating network." + 'message': 'Error occured while creating network.' } else: - network_entry["ipv6"] = prefix["prefix"] + network_entry['ipv6'] = prefix['prefix'] else: - network_entry["ipv6"] = "fd00::/64" + network_entry['ipv6'] = 'fd00::/64' network_key = join_path( - settings["etcd"]["network_prefix"], - data["name"], - data["network_name"], + shared.settings['etcd']['network_prefix'], + data['name'], + data['network_name'], ) shared.etcd_client.put( network_key, network_entry, value_in_json=True ) - return {"message": "Network successfully added."} + return {'message': 'Network successfully added.'} else: return validator.get_errors(), 400 class ListUserNetwork(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) if validator.is_valid(): prefix = join_path( - settings["etcd"]["network_prefix"], data["name"] + shared.settings['etcd']['network_prefix'], data['name'] ) networks = shared.etcd_client.get_prefix( prefix, value_in_json=True ) user_networks = [] for net in networks: - net.value["name"] = net.key.split("/")[-1] + net.value['name'] = net.key.split('/')[-1] user_networks.append(net.value) - return {"networks": user_networks}, 200 + return {'networks': user_networks}, 200 else: return validator.get_errors(), 400 -api.add_resource(CreateVM, "/vm/create") -api.add_resource(VmStatus, "/vm/status") +api.add_resource(CreateVM, '/vm/create') +api.add_resource(VmStatus, '/vm/status') -api.add_resource(VMAction, "/vm/action") -api.add_resource(VMMigration, "/vm/migrate") +api.add_resource(VMAction, '/vm/action') +api.add_resource(VMMigration, '/vm/migrate') -api.add_resource(CreateImage, "/image/create") -api.add_resource(ListPublicImages, "/image/list-public") +api.add_resource(CreateImage, '/image/create') +api.add_resource(ListPublicImages, '/image/list-public') -api.add_resource(ListUserVM, "/user/vms") -api.add_resource(ListUserFiles, "/user/files") -api.add_resource(ListUserNetwork, "/user/networks") +api.add_resource(ListUserVM, '/user/vms') +api.add_resource(ListUserFiles, '/user/files') +api.add_resource(ListUserNetwork, '/user/networks') -api.add_resource(AddSSHKey, "/user/add-ssh") -api.add_resource(RemoveSSHKey, "/user/remove-ssh") -api.add_resource(GetSSHKeys, "/user/get-ssh") +api.add_resource(AddSSHKey, '/user/add-ssh') +api.add_resource(RemoveSSHKey, '/user/remove-ssh') +api.add_resource(GetSSHKeys, '/user/get-ssh') -api.add_resource(CreateHost, "/host/create") -api.add_resource(ListHost, "/host/list") +api.add_resource(CreateHost, '/host/create') +api.add_resource(ListHost, '/host/list') -api.add_resource(CreateNetwork, "/network/create") +api.add_resource(CreateNetwork, '/network/create') -def main(debug=False, port=None): +def main(arguments): + debug = arguments['debug'] + port = arguments['port'] + try: image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], value_in_json=True + shared.settings['etcd']['image_store_prefix'], value_in_json=True ) ) except KeyError: @@ -576,27 +580,21 @@ def main(debug=False, port=None): # # if not image_stores: # data = { - # "is_public": True, - # "type": "ceph", - # "name": "images", - # "description": "first ever public image-store", - # "attributes": {"list": [], "key": [], "pool": "images"}, + # 'is_public': True, + # 'type': 'ceph', + # 'name': 'images', + # 'description': 'first ever public image-store', + # 'attributes': {'list': [], 'key': [], 'pool': 'images'}, # } # shared.etcd_client.put( # join_path( - # settings["etcd"]["image_store_prefix"], uuid4().hex + # shared.settings['etcd']['image_store_prefix'], uuid4().hex # ), # json.dumps(data), # ) try: - app.run(host="::", - port=port, - debug=debug) + app.run(host='::', port=port, debug=debug) except OSError as e: - raise UncloudException("Failed to start Flask: {}".format(e)) - - -if __name__ == "__main__": - main() + raise UncloudException('Failed to start Flask: {}'.format(e)) diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index 65055c4..87f20c9 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -21,8 +21,7 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema): def file_uuid_validation(self): file_entry = shared.etcd_client.get( os.path.join( - settings["etcd"]["file_prefix"], self.uuid.value + shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value ) ) if file_entry is None: @@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema): def image_store_name_validation(self): image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"] + shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] ) ) @@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema): for net in _network: network = shared.etcd_client.get( os.path.join( - settings["etcd"]["network_prefix"], + shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, net, ), @@ -322,7 +321,7 @@ class CreateVMSchema(OTPSchema): "Your specified OS-SSD is not in correct units" ) - if _cpu < 1: + if int(_cpu) < 1: self.add_error("CPU must be atleast 1") if parsed_ram < bitmath.GB(1): @@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema): self.add_error("Can't migrate non-running VM") if vm.hostname == os.path.join( - settings["etcd"]["host_prefix"], self.destination.value + shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value ): self.add_error( "Destination host couldn't be same as Source Host" @@ -528,9 +527,7 @@ class GetSSHSchema(OTPSchema): class CreateNetwork(OTPSchema): def __init__(self, data): - self.network_name = Field( - "network_name", str, data.get("network_name", KeyError) - ) + self.network_name = Field("network_name", str, data.get("network_name", KeyError)) self.type = Field("type", str, data.get("type", KeyError)) self.user = Field("user", bool, bool(data.get("user", False))) @@ -541,14 +538,8 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = shared.etcd_client.get( - os.path.join( - settings["etcd"]["network_prefix"], - self.name.value, - self.network_name.value, - ), - value_in_json=True, - ) + key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) + network = shared.etcd_client.get(key, value_in_json=True) if network: self.add_error( "Network with name {} already exists".format( diff --git a/uncloud/cli/__init__.py b/uncloud/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py new file mode 100644 index 0000000..51a4355 --- /dev/null +++ b/uncloud/cli/helper.py @@ -0,0 +1,46 @@ +import requests +import json +import argparse +import binascii + +from pyotp import TOTP +from os.path import join as join_path +from uncloud.common.shared import shared + + +def get_otp_parser(): + otp_parser = argparse.ArgumentParser('otp') + otp_parser.add_argument('--name') + otp_parser.add_argument('--realm') + otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') + + return otp_parser + + +def load_dump_pretty(content): + if isinstance(content, bytes): + content = content.decode('utf-8') + parsed = json.loads(content) + return json.dumps(parsed, indent=4, sort_keys=True) + + +def make_request(*args, data=None, request_method=requests.post): + try: + r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) + except requests.exceptions.RequestException: + print('Error occurred while connecting to API server.') + else: + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except binascii.Error: + raise argparse.ArgumentTypeError('Invalid seed') + else: + return token diff --git a/uncloud/cli/host.py b/uncloud/cli/host.py new file mode 100644 index 0000000..e912567 --- /dev/null +++ b/uncloud/cli/host.py @@ -0,0 +1,45 @@ +import requests + +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class HostParser(BaseParser): + def __init__(self): + super().__init__('host') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs) + p.add_argument('--hostname', required=True) + p.add_argument('--cpu', required=True, type=int) + p.add_argument('--ram', required=True) + p.add_argument('--os-ssd', required=True) + p.add_argument('--hdd', default=list()) + + def list(self, **kwargs): + self.subparser.add_parser('list', **kwargs) + + +parser = HostParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('host_subcommand') + if not subcommand: + arg_parser.print_help() + else: + request_method = requests.post + data = None + if subcommand == 'create': + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + data = kwargs + elif subcommand == 'list': + request_method = requests.get + + make_request('host', subcommand, data=data, request_method=request_method) diff --git a/uncloud/cli/image.py b/uncloud/cli/image.py new file mode 100644 index 0000000..2f59c32 --- /dev/null +++ b/uncloud/cli/image.py @@ -0,0 +1,38 @@ +import requests + +from uncloud.cli.helper import make_request +from uncloud.common.parser import BaseParser + + +class ImageParser(BaseParser): + def __init__(self): + super().__init__('image') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', **kwargs) + p.add_argument('--name', required=True) + p.add_argument('--uuid', required=True) + p.add_argument('--image-store', required=True, dest='image_store') + + def list(self, **kwargs): + self.subparser.add_parser('list', **kwargs) + + +parser = ImageParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('image_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = None + request_method = requests.post + if subcommand == 'list': + subcommand = 'list-public' + request_method = requests.get + elif subcommand == 'create': + data = kwargs + + make_request('image', subcommand, data=data, request_method=request_method) diff --git a/uncloud/cli/main.py b/uncloud/cli/main.py new file mode 100644 index 0000000..9a42497 --- /dev/null +++ b/uncloud/cli/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import argparse +import importlib + +arg_parser = argparse.ArgumentParser('cli', add_help=False) +subparser = arg_parser.add_subparsers(dest='subcommand') + +for component in ['user', 'host', 'image', 'network', 'vm']: + module = importlib.import_module('uncloud.cli.{}'.format(component)) + parser = getattr(module, 'arg_parser') + subparser.add_parser(name=parser.prog, parents=[parser]) + + +def main(arguments): + if not arguments['subcommand']: + arg_parser.print_help() + else: + name = arguments.pop('subcommand') + arguments.pop('debug') + mod = importlib.import_module('uncloud.cli.{}'.format(name)) + _main = getattr(mod, 'main') + _main(**arguments) diff --git a/uncloud/cli/network.py b/uncloud/cli/network.py new file mode 100644 index 0000000..55798bf --- /dev/null +++ b/uncloud/cli/network.py @@ -0,0 +1,32 @@ +import requests + +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class NetworkParser(BaseParser): + def __init__(self): + super().__init__('network') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs) + p.add_argument('--network-name', required=True) + p.add_argument('--network-type', required=True, dest='type') + p.add_argument('--user', action='store_true') + + +parser = NetworkParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('network_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = None + request_method = requests.post + if subcommand == 'create': + data = kwargs + + make_request('network', subcommand, data=data, request_method=request_method) diff --git a/uncloud/cli/user.py b/uncloud/cli/user.py new file mode 100755 index 0000000..3a4cc4e --- /dev/null +++ b/uncloud/cli/user.py @@ -0,0 +1,41 @@ +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class UserParser(BaseParser): + def __init__(self): + super().__init__('user') + + def files(self, **kwargs): + self.subparser.add_parser('files', parents=[get_otp_parser()], **kwargs) + + def vms(self, **kwargs): + self.subparser.add_parser('vms', parents=[get_otp_parser()], **kwargs) + + def networks(self, **kwargs): + self.subparser.add_parser('networks', parents=[get_otp_parser()], **kwargs) + + def add_ssh(self, **kwargs): + p = self.subparser.add_parser('add-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', required=True) + p.add_argument('--key', required=True) + + def get_ssh(self, **kwargs): + p = self.subparser.add_parser('get-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', default='') + + def remove_ssh(self, **kwargs): + p = self.subparser.add_parser('remove-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', required=True) + + +parser = UserParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('user_subcommand') + if not subcommand: + arg_parser.print_help() + else: + make_request('user', subcommand, data=kwargs) diff --git a/uncloud/cli/vm.py b/uncloud/cli/vm.py new file mode 100644 index 0000000..396530e --- /dev/null +++ b/uncloud/cli/vm.py @@ -0,0 +1,62 @@ +from uncloud.common.parser import BaseParser +from uncloud.cli.helper import make_request, get_otp_parser + + +class VMParser(BaseParser): + def __init__(self): + super().__init__('vm') + + def start(self, **args): + p = self.subparser.add_parser('start', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def stop(self, **args): + p = self.subparser.add_parser('stop', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def status(self, **args): + p = self.subparser.add_parser('status', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def delete(self, **args): + p = self.subparser.add_parser('delete', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def migrate(self, **args): + p = self.subparser.add_parser('migrate', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + p.add_argument('--destination', required=True) + + def create(self, **args): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **args) + p.add_argument('--cpu', required=True) + p.add_argument('--ram', required=True) + p.add_argument('--os-ssd', required=True) + p.add_argument('--hdd', action='append', default=list()) + p.add_argument('--image', required=True) + p.add_argument('--network', action='append', default=[]) + p.add_argument('--vm-name', required=True) + + +parser = VMParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('vm_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = kwargs + endpoint = subcommand + if subcommand in ['start', 'stop', 'delete']: + endpoint = 'action' + data['action'] = subcommand + elif subcommand == 'create': + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + make_request('vm', endpoint, data=data) diff --git a/uncloud/common/cli.py b/uncloud/common/cli.py new file mode 100644 index 0000000..3d3c248 --- /dev/null +++ b/uncloud/common/cli.py @@ -0,0 +1,26 @@ +from uncloud.common.shared import shared +from pyotp import TOTP + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except Exception: + raise Exception('Invalid seed') + else: + return token + + +def resolve_otp_credentials(kwargs): + d = { + 'name': shared.settings['client']['name'], + 'realm': shared.settings['client']['realm'], + 'token': get_token(shared.settings['client']['seed']) + } + + for k, v in d.items(): + if k in kwargs and kwargs[k] is None: + kwargs.update({k: v}) + + return d diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index 6a979ba..38471ab 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -1,24 +1,21 @@ import etcd3 import json -import queue -import copy -from uncloud import UncloudException -from collections import namedtuple from functools import wraps -from . import logger - -PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) +from uncloud import UncloudException +from uncloud.common import logger class EtcdEntry: - # key: str - # value: str - - def __init__(self, meta, value, value_in_json=False): - self.key = meta.key.decode("utf-8") - self.value = value.decode("utf-8") + def __init__(self, meta_or_key, value, value_in_json=False): + if hasattr(meta_or_key, 'key'): + # if meta has attr 'key' then get it + self.key = meta_or_key.key.decode('utf-8') + else: + # otherwise meta is the 'key' + self.key = meta_or_key + self.value = value.decode('utf-8') if value_in_json: self.value = json.loads(self.value) @@ -29,18 +26,12 @@ def readable_errors(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except etcd3.exceptions.ConnectionFailedError as err: - raise UncloudException( - "Cannot connect to etcd: is etcd running as configured in uncloud.conf?" - ) + except etcd3.exceptions.ConnectionFailedError: + raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') except etcd3.exceptions.ConnectionTimeoutError as err: - raise etcd3.exceptions.ConnectionTimeoutError( - "etcd connection timeout." - ) from err + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err except Exception: - logger.exception( - "Some etcd error occured. See syslog for details." - ) + logger.exception('Some etcd error occured. See syslog for details.') return wrapper @@ -64,55 +55,21 @@ class Etcd3Wrapper: _value = json.dumps(_value) if not isinstance(_key, str): - _key = _key.decode("utf-8") + _key = _key.decode('utf-8') return self.client.put(_key, _value, **kwargs) @readable_errors - def get_prefix(self, *args, value_in_json=False, **kwargs): - r = self.client.get_prefix(*args, **kwargs) - for entry in r: - e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) - if e.value: - yield e + def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) @readable_errors - def watch_prefix(self, key, timeout=0, value_in_json=False): - timeout_event = EtcdEntry( - PseudoEtcdMeta(key=b"TIMEOUT"), - value=str.encode( - json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"}) - ), - value_in_json=value_in_json, - ) - - event_queue = queue.Queue() - - def add_event_to_queue(event): - if hasattr(event, "events"): - for e in event.events: - if e.value: - event_queue.put( - EtcdEntry( - e, e.value, value_in_json=value_in_json - ) - ) - - self.client.add_watch_prefix_callback(key, add_event_to_queue) - - while True: - try: - while True: - v = event_queue.get(timeout=timeout) - yield v - except queue.Empty: - event_queue.put(copy.deepcopy(timeout_event)) - - -class PsuedoEtcdEntry(EtcdEntry): - def __init__(self, key, value, value_in_json=False): - super().__init__( - PseudoEtcdMeta(key=key.encode("utf-8")), - value, - value_in_json=value_in_json, - ) + def watch_prefix(self, key, raise_exception=True, value_in_json=False): + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = e._event + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) diff --git a/uncloud/common/network.py b/uncloud/common/network.py index adba108..32f6951 100644 --- a/uncloud/common/network.py +++ b/uncloud/common/network.py @@ -1,8 +1,6 @@ import subprocess as sp import random import logging -import socket -from contextlib import closing logger = logging.getLogger(__name__) diff --git a/uncloud/common/parser.py b/uncloud/common/parser.py new file mode 100644 index 0000000..576f0e7 --- /dev/null +++ b/uncloud/common/parser.py @@ -0,0 +1,13 @@ +import argparse + + +class BaseParser: + def __init__(self, command): + self.arg_parser = argparse.ArgumentParser(command, add_help=False) + self.subparser = self.arg_parser.add_subparsers(dest='{}_subcommand'.format(command)) + self.common_args = {'add_help': False} + + methods = [attr for attr in dir(self) if not attr.startswith('__') + and type(getattr(self, attr)).__name__ == 'method'] + for method in methods: + getattr(self, method)(**self.common_args) diff --git a/uncloud/common/request.py b/uncloud/common/request.py index a8c2d0a..cb0add5 100644 --- a/uncloud/common/request.py +++ b/uncloud/common/request.py @@ -2,8 +2,8 @@ import json from os.path import join from uuid import uuid4 -from .etcd_wrapper import PsuedoEtcdEntry -from .classes import SpecificEtcdEntryBase +from uncloud.common.etcd_wrapper import EtcdEntry +from uncloud.common.classes import SpecificEtcdEntryBase class RequestType: @@ -29,11 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase): @classmethod def from_scratch(cls, request_prefix, **kwargs): - e = PsuedoEtcdEntry( - join(request_prefix, uuid4().hex), - value=json.dumps(kwargs).encode("utf-8"), - value_in_json=True, - ) + e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex), + value=json.dumps(kwargs).encode('utf-8'), value_in_json=True) return cls(e) diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py new file mode 100644 index 0000000..8503f42 --- /dev/null +++ b/uncloud/common/settings.py @@ -0,0 +1,136 @@ +import configparser +import logging +import sys +import os + +from datetime import datetime +from uncloud.common.etcd_wrapper import Etcd3Wrapper +from os.path import join as join_path + +logger = logging.getLogger(__name__) +settings = None + + +class CustomConfigParser(configparser.RawConfigParser): + def __getitem__(self, key): + try: + result = super().__getitem__(key) + except KeyError as err: + raise KeyError( + 'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format( + key + ) + ) from err + else: + return result + + +class Settings(object): + def __init__(self, conf_dir, seed_value=None): + conf_name = 'uncloud.conf' + self.config_file = join_path(conf_dir, conf_name) + + # this is used to cache config from etcd for 1 minutes. Without this we + # would make a lot of requests to etcd which slows down everything. + self.last_config_update = datetime.fromtimestamp(0) + + self.config_parser = CustomConfigParser(allow_no_value=True) + self.config_parser.add_section('etcd') + self.config_parser.set('etcd', 'base_prefix', '/') + + if os.access(self.config_file, os.R_OK): + self.config_parser.read(self.config_file) + else: + raise FileNotFoundError('Config file %s not found!', self.config_file) + self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') + + self.read_internal_values() + + if seed_value is None: + seed_value = dict() + + self.config_parser.read_dict(seed_value) + + def get_etcd_client(self): + args = tuple() + try: + kwargs = { + 'host': self.config_parser.get('etcd', 'url'), + 'port': self.config_parser.get('etcd', 'port'), + 'ca_cert': self.config_parser.get('etcd', 'ca_cert'), + 'cert_cert': self.config_parser.get('etcd', 'cert_cert'), + 'cert_key': self.config_parser.get('etcd', 'cert_key'), + } + except configparser.Error as err: + raise configparser.Error( + '{} in config file {}'.format( + err.message, self.config_file + ) + ) from err + else: + try: + wrapper = Etcd3Wrapper(*args, **kwargs) + except Exception as err: + logger.error( + 'etcd connection not successfull. Please check your config file.' + '\nDetails: %s\netcd connection parameters: %s', + err, + kwargs, + ) + sys.exit(1) + else: + return wrapper + + def read_internal_values(self): + base_prefix = self['etcd']['base_prefix'] + self.config_parser.read_dict( + { + 'etcd': { + 'file_prefix': join_path(base_prefix, 'files/'), + 'host_prefix': join_path(base_prefix, 'hosts/'), + 'image_prefix': join_path(base_prefix, 'images/'), + 'image_store_prefix': join_path(base_prefix, 'imagestore/'), + 'network_prefix': join_path(base_prefix, 'networks/'), + 'request_prefix': join_path(base_prefix, 'requests/'), + 'user_prefix': join_path(base_prefix, 'users/'), + 'vm_prefix': join_path(base_prefix, 'vms/'), + 'vxlan_counter': join_path(base_prefix, 'counters/vxlan'), + 'tap_counter': join_path(base_prefix, 'counters/tap') + } + } + ) + + def read_config_file_values(self, config_file): + try: + # Trying to read configuration file + with open(config_file) as config_file_handle: + self.config_parser.read_file(config_file_handle) + except FileNotFoundError: + sys.exit('Configuration file {} not found!'.format(config_file)) + except Exception as err: + logger.exception(err) + sys.exit('Error occurred while reading configuration file') + + def read_values_from_etcd(self): + etcd_client = self.get_etcd_client() + if (datetime.utcnow() - self.last_config_update).total_seconds() > 60: + config_from_etcd = etcd_client.get(self.config_key, value_in_json=True) + if config_from_etcd: + self.config_parser.read_dict(config_from_etcd.value) + self.last_config_update = datetime.utcnow() + else: + raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key)) + + def __getitem__(self, key): + # Allow failing to read from etcd if we have + # it locally + if key not in self.config_parser.sections(): + try: + self.read_values_from_etcd() + except KeyError: + pass + return self.config_parser[key] + + +def get_settings(): + return settings diff --git a/uncloud/common/shared.py b/uncloud/common/shared.py new file mode 100644 index 0000000..aea7cbc --- /dev/null +++ b/uncloud/common/shared.py @@ -0,0 +1,34 @@ +from uncloud.common.settings import get_settings +from uncloud.common.vm import VmPool +from uncloud.common.host import HostPool +from uncloud.common.request import RequestPool +import uncloud.common.storage_handlers as storage_handlers + + +class Shared: + @property + def settings(self): + return get_settings() + + @property + def etcd_client(self): + return self.settings.get_etcd_client() + + @property + def host_pool(self): + return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) + + @property + def vm_pool(self): + return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) + + @property + def request_pool(self): + return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) + + @property + def storage_handler(self): + return storage_handlers.get_storage_handler() + + +shared = Shared() diff --git a/uncloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py index 06751c4..58c2dc2 100644 --- a/uncloud/common/storage_handlers.py +++ b/uncloud/common/storage_handlers.py @@ -6,8 +6,7 @@ import stat from abc import ABC from . import logger from os.path import join as join_path - -from uncloud.settings import settings as config +import uncloud.common.shared as shared class ImageStorageHandler(ABC): @@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): def get_storage_handler(): - __storage_backend = config["storage"]["storage_backend"] + __storage_backend = shared.shared.settings["storage"]["storage_backend"] if __storage_backend == "filesystem": return FileSystemBasedImageStorageHandler( - vm_base=config["storage"]["vm_dir"], - image_base=config["storage"]["image_dir"], + vm_base=shared.shared.settings["storage"]["vm_dir"], + image_base=shared.shared.settings["storage"]["image_dir"], ) elif __storage_backend == "ceph": return CEPHBasedImageStorageHandler( - vm_base=config["storage"]["ceph_vm_pool"], - image_base=config["storage"]["ceph_image_pool"], + vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], + image_base=shared.shared.settings["storage"]["ceph_image_pool"], ) else: - raise Exception("Unknown Image Storage Handler") + raise Exception("Unknown Image Storage Handler") \ No newline at end of file diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index a9b4901..87f5752 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,79 +1,57 @@ import os +import argparse -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared + +arg_parser = argparse.ArgumentParser('configure', add_help=False) +configure_subparsers = arg_parser.add_subparsers(dest='subcommand') + +otp_parser = configure_subparsers.add_parser('otp') +otp_parser.add_argument('--verification-controller-url', required=True, metavar='URL') +otp_parser.add_argument('--auth-name', required=True, metavar='OTP-NAME') +otp_parser.add_argument('--auth-realm', required=True, metavar='OTP-REALM') +otp_parser.add_argument('--auth-seed', required=True, metavar='OTP-SEED') + +network_parser = configure_subparsers.add_parser('network') +network_parser.add_argument('--prefix-length', required=True, type=int) +network_parser.add_argument('--prefix', required=True) +network_parser.add_argument('--vxlan-phy-dev', required=True) + +netbox_parser = configure_subparsers.add_parser('netbox') +netbox_parser.add_argument('--url', required=True) +netbox_parser.add_argument('--token', required=True) + +ssh_parser = configure_subparsers.add_parser('ssh') +ssh_parser.add_argument('--username', default='root') +ssh_parser.add_argument('--private-key-path', default=os.path.expanduser('~/.ssh/id_rsa'),) + +storage_parser = configure_subparsers.add_parser('storage') +storage_parser.add_argument('--file-dir', required=True) +storage_parser_subparsers = storage_parser.add_subparsers(dest='storage_backend') + +filesystem_storage_parser = storage_parser_subparsers.add_parser('filesystem') +filesystem_storage_parser.add_argument('--vm-dir', required=True) +filesystem_storage_parser.add_argument('--image-dir', required=True) + +ceph_storage_parser = storage_parser_subparsers.add_parser('ceph') +ceph_storage_parser.add_argument('--ceph-vm-pool', required=True) +ceph_storage_parser.add_argument('--ceph-image-pool', required=True) def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get( - settings.config_key, value_in_json=True - ) + uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - shared.etcd_client.put( - settings.config_key, uncloud_config, value_in_json=True - ) + shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) -def configure_parser(parser): - configure_subparsers = parser.add_subparsers(dest="subcommand") - - otp_parser = configure_subparsers.add_parser("otp") - otp_parser.add_argument( - "--verification-controller-url", required=True, metavar="URL" - ) - otp_parser.add_argument( - "--auth-name", required=True, metavar="OTP-NAME" - ) - otp_parser.add_argument( - "--auth-realm", required=True, metavar="OTP-REALM" - ) - otp_parser.add_argument( - "--auth-seed", required=True, metavar="OTP-SEED" - ) - - network_parser = configure_subparsers.add_parser("network") - network_parser.add_argument( - "--prefix-length", required=True, type=int - ) - network_parser.add_argument("--prefix", required=True) - network_parser.add_argument("--vxlan-phy-dev", required=True) - - netbox_parser = configure_subparsers.add_parser("netbox") - netbox_parser.add_argument("--url", required=True) - netbox_parser.add_argument("--token", required=True) - - ssh_parser = configure_subparsers.add_parser("ssh") - ssh_parser.add_argument("--username", default="root") - ssh_parser.add_argument( - "--private-key-path", - default=os.path.expanduser("~/.ssh/id_rsa"), - ) - - storage_parser = configure_subparsers.add_parser("storage") - storage_parser.add_argument("--file-dir", required=True) - storage_parser_subparsers = storage_parser.add_subparsers( - dest="storage_backend" - ) - - filesystem_storage_parser = storage_parser_subparsers.add_parser( - "filesystem" - ) - filesystem_storage_parser.add_argument("--vm-dir", required=True) - filesystem_storage_parser.add_argument("--image-dir", required=True) - - ceph_storage_parser = storage_parser_subparsers.add_parser("ceph") - ceph_storage_parser.add_argument("--ceph-vm-pool", required=True) - ceph_storage_parser.add_argument("--ceph-image-pool", required=True) - - -def main(**kwargs): - subcommand = kwargs.pop("subcommand") +def main(arguments): + subcommand = arguments['subcommand'] if not subcommand: - pass + arg_parser.print_help() else: - update_config(subcommand, kwargs) + update_config(subcommand, arguments) diff --git a/uncloud/docs/source/hacking.rst b/uncloud/docs/source/hacking.rst deleted file mode 100644 index 2df42a7..0000000 --- a/uncloud/docs/source/hacking.rst +++ /dev/null @@ -1,17 +0,0 @@ -Hacking -======= -How to hack on the code. - -[ to be done by Balazs: - -* make nice -* indent with shell script mode - -] - -* git clone the repo -* cd to the repo -* Setup your venv: python -m venv venv -* . ./venv/bin/activate # you need the leading dot for sourcing! -* Run ./bin/ucloud-run-reinstall - it should print you an error - message on how to use ucloud diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index 7ce8654..046f915 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -3,12 +3,16 @@ import os import pathlib import subprocess as sp import time +import argparse +import bitmath from uuid import uuid4 from . import logger -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared + +arg_parser = argparse.ArgumentParser('filescanner', add_help=False) +arg_parser.add_argument('--hostname', required=True) def sha512sum(file: str): @@ -24,66 +28,58 @@ def sha512sum(file: str): if not isinstance(file, str): raise TypeError try: - output = sp.check_output(["sha512sum", file], stderr=sp.PIPE) + output = sp.check_output(['sha512sum', file], stderr=sp.PIPE) except sp.CalledProcessError as e: - error = e.stderr.decode("utf-8") - if "No such file or directory" in error: + error = e.stderr.decode('utf-8') + if 'No such file or directory' in error: raise FileNotFoundError from None else: - output = output.decode("utf-8").strip() - output = output.split(" ") + output = output.decode('utf-8').strip() + output = output.split(' ') return output[0] return None -def track_file(file, base_dir): - file_id = uuid4() - +def track_file(file, base_dir, host): + file_path = file.relative_to(base_dir) + file_str = str(file) # Get Username - owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] + try: + owner = file_path.parts[0] + except IndexError: + pass + else: + file_path = file_path.relative_to(owner) + creation_date = time.ctime(os.stat(file_str).st_ctime) - # Get Creation Date of File - # Here, we are assuming that ctime is creation time - # which is mostly not true. - creation_date = time.ctime(os.stat(file).st_ctime) + entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) + entry_value = { + 'filename': str(file_path), + 'owner': owner, + 'sha512sum': sha512sum(file_str), + 'creation_date': creation_date, + 'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()), + 'host': host + } - file_path = pathlib.Path(file).parts[-1] + logger.info('Tracking %s', file_str) - # Create Entry - entry_key = os.path.join( - settings["etcd"]["file_prefix"], str(file_id) - ) - entry_value = { - "filename": file_path, - "owner": owner, - "sha512sum": sha512sum(file), - "creation_date": creation_date, - "size": os.path.getsize(file), - } - - logger.info("Tracking %s", file) - - shared.etcd_client.put(entry_key, entry_value, value_in_json=True) - os.setxattr(file, "user.utracked", b"True") + shared.etcd_client.put(entry_key, entry_value, value_in_json=True) -def main(): - base_dir = settings["storage"]["file_dir"] - +def main(arguments): + hostname = arguments['hostname'] + base_dir = shared.settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR - files = glob.glob("{}/**".format(base_dir), recursive=True) + files = glob.glob('{}/**'.format(base_dir), recursive=True) + files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] - # Retain only Files - files = [file for file in files if os.path.isfile(file)] - - untracked_files = [] - for file in files: - try: - os.getxattr(file, "user.utracked") - except OSError: - track_file(file, base_dir) - untracked_files.append(file) - - -if __name__ == "__main__": - main() + # Files that are already tracked + tracked_files = [ + pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) + for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) + if f.value['host'] == hostname + ] + untracked_files = set(files) - set(tracked_files) + for file in untracked_files: + track_file(file, base_dir, hostname) diff --git a/uncloud/hack/__init__.py b/uncloud/hack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/hack/config.py b/uncloud/hack/config.py new file mode 100644 index 0000000..7e2655d --- /dev/null +++ b/uncloud/hack/config.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +class Config(object): + def __init__(self, arguments): + """ read arguments dicts as a base """ + + self.arguments = arguments + + # Split them so *etcd_args can be used and we can + # iterate over etcd_hosts + self.etcd_hosts = [ arguments['etcd_host'] ] + self.etcd_args = { + 'ca_cert': arguments['etcd_ca_cert'], + 'cert_cert': arguments['etcd_cert_cert'], + 'cert_key': arguments['etcd_cert_key'], +# 'user': None, +# 'password': None + } + self.etcd_prefix = '/nicohack/' diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py new file mode 100644 index 0000000..cb5e490 --- /dev/null +++ b/uncloud/hack/db.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +import etcd3 +import json +import logging + +from functools import wraps +from uncloud import UncloudException + +log = logging.getLogger(__name__) + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + + +class DB(object): + def __init__(self, config, prefix="/"): + self.config = config + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + + self.connect() + + @readable_errors + def connect(self): + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) + + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) + + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: + value = json.dumps(value) + + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + @readable_errors + def increment(self, key, **kwargs): + print(self.realkey(key)) + + + print("prelock") + lock = self._db_clients[0].lock('/nicohack/foo') + print("prelockacq") + lock.acquire() + print("prelockrelease") + lock.release() + + with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock: + print("in lock") + pass + +# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs)) +# self.set(self.realkey(key), str(value + 1), **kwargs) + + +if __name__ == '__main__': + endpoints = [ "https://etcd1.ungleich.ch:2379", + "https://etcd2.ungleich.ch:2379", + "https://etcd3.ungleich.ch:2379" ] + + db = DB(url=endpoints) diff --git a/uncloud/hack/hackcloud/.gitignore b/uncloud/hack/hackcloud/.gitignore new file mode 100644 index 0000000..0ad647b --- /dev/null +++ b/uncloud/hack/hackcloud/.gitignore @@ -0,0 +1,3 @@ +*.iso +radvdpid +foo diff --git a/uncloud/hack/hackcloud/__init__.py b/uncloud/hack/hackcloud/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/uncloud/hack/hackcloud/__init__.py @@ -0,0 +1 @@ + diff --git a/uncloud/hack/hackcloud/etcd-client.sh b/uncloud/hack/hackcloud/etcd-client.sh new file mode 100644 index 0000000..ab102a5 --- /dev/null +++ b/uncloud/hack/hackcloud/etcd-client.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \ + --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \ + --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \ + --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@" diff --git a/uncloud/hack/hackcloud/ifdown.sh b/uncloud/hack/hackcloud/ifdown.sh new file mode 100755 index 0000000..5753099 --- /dev/null +++ b/uncloud/hack/hackcloud/ifdown.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo $@ diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh new file mode 100755 index 0000000..e0a3ca0 --- /dev/null +++ b/uncloud/hack/hackcloud/ifup.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +dev=$1; shift + +# bridge is setup from outside +ip link set dev "$dev" master ${bridge} +ip link set dev "$dev" up diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last new file mode 100644 index 0000000..8c5f254 --- /dev/null +++ b/uncloud/hack/hackcloud/mac-last @@ -0,0 +1 @@ +000000000252 diff --git a/uncloud/hack/hackcloud/mac-prefix b/uncloud/hack/hackcloud/mac-prefix new file mode 100644 index 0000000..5084a2f --- /dev/null +++ b/uncloud/hack/hackcloud/mac-prefix @@ -0,0 +1 @@ +02:00 diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh new file mode 100755 index 0000000..4e2bfa1 --- /dev/null +++ b/uncloud/hack/hackcloud/net.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -x + +netid=100 +dev=wlp2s0 +dev=wlp0s20f3 +#dev=wlan0 + +ip=2a0a:e5c1:111:888::48/64 +vxlandev=vxlan${netid} +bridgedev=br${netid} + +ip -6 link add ${vxlandev} type vxlan \ + id ${netid} \ + dstport 4789 \ + group ff05::${netid} \ + dev ${dev} \ + ttl 5 + +ip link set ${vxlandev} up + + +ip link add ${bridgedev} type bridge +ip link set ${bridgedev} up + +ip link set ${vxlandev} master ${bridgedev} up + +ip addr add ${ip} dev ${bridgedev} diff --git a/uncloud/hack/hackcloud/nftrules b/uncloud/hack/hackcloud/nftrules new file mode 100644 index 0000000..636c63d --- /dev/null +++ b/uncloud/hack/hackcloud/nftrules @@ -0,0 +1,31 @@ +flush ruleset + +table bridge filter { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; + + ibrname br100 jump br100 + } + + chain br100 { + # Allow all incoming traffic from outside + iifname vxlan100 accept + + # Default blocks: router advertisements, dhcpv6, dhcpv4 + icmpv6 type nd-router-advert drop + ip6 version 6 udp sport 547 drop + ip version 4 udp sport 67 drop + + jump br100_vmlist + drop + } + chain br100_vmlist { + # VM1 + iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept + + # VM2 + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept + } +} diff --git a/uncloud/hack/hackcloud/radvd.conf b/uncloud/hack/hackcloud/radvd.conf new file mode 100644 index 0000000..3d8ce4d --- /dev/null +++ b/uncloud/hack/hackcloud/radvd.conf @@ -0,0 +1,13 @@ +interface br100 +{ + AdvSendAdvert on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 5; + AdvDefaultLifetime 3600; + + prefix 2a0a:e5c1:111:888::/64 { + }; + + RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; }; + DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ; +}; diff --git a/uncloud/hack/hackcloud/radvd.sh b/uncloud/hack/hackcloud/radvd.sh new file mode 100644 index 0000000..9d0e7d1 --- /dev/null +++ b/uncloud/hack/hackcloud/radvd.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +radvd -C ./radvd.conf -n -p ./radvdpid diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh new file mode 100755 index 0000000..dd9be84 --- /dev/null +++ b/uncloud/hack/hackcloud/vm.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# if [ $# -ne 1 ]; then +# echo "$0: owner" +# exit 1 +# fi + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +#accel=tcg + +memory=1024 +cores=2 +uuid=$(uuidgen) +mac=$(./mac-gen.py) +owner=nico + +export bridge=br100 + +set -x +$qemu -name "uncloud-${uuid}" \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac} diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py new file mode 100755 index 0000000..66286dd --- /dev/null +++ b/uncloud/hack/mac.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2012 Nico Schottelius (nico-cinv at schottelius.org) +# +# This file is part of cinv. +# +# cinv is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cinv is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cinv. If not, see . +# +# + +import argparse +import logging +import os.path +import os +import re +import json + +from uncloud import UncloudException +from uncloud.hack.db import DB + +log = logging.getLogger(__name__) + + +class MAC(object): + def __init__(self, config): + self.config = config + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(config, prefix="/mac") + + self.prefix = 0x420000000000 + self._number = 0 # Not set by default + + @staticmethod + def validate_mac(mac): + if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): + raise Error("Not a valid mac address: %s" % mac) + + def last_used_index(self): + if not self.no_db: + value = self.db.get("last_used_index") + if not value: + self.db.set("last_used_index", "0") + value = self.db.get("last_used_index") + + else: + value = "0" + + return int(value) + + def last_used_mac(self): + return self.int_to_mac(self.prefix + self.last_used_index()) + + def to_colon_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ':'.join(format(s, '02x') for s in b) + + def to_str_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ''.join(format(s, '02x') for s in b) + + def create(self): + last_number = self.last_used_index() + + if last_number == int('0xffffffff', 16): + raise UncloudException("Exhausted all possible mac addresses - try to free some") + + next_number = last_number + 1 + self._number = self.prefix + next_number + + #next_number_string = "{:012x}".format(next_number) + #next_mac = self.int_to_mac(next_mac_number) + # db_entry = {} + # db_entry['vm_uuid'] = vmuuid + # db_entry['index'] = next_number + # db_entry['mac_address'] = next_mac + + # should be one transaction + # self.db.increment("last_used_index") + # self.db.set("used/{}".format(next_mac), + # db_entry, as_json=True) + + def __int__(self): + return self._number + + def __repr__(self): + return self.to_str_format() + + def __str__(self): + return self.to_colon_format() diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py new file mode 100644 index 0000000..9607ec2 --- /dev/null +++ b/uncloud/hack/main.py @@ -0,0 +1,92 @@ +import argparse +import logging + +from uncloud.hack.vm import VM +from uncloud.hack.config import Config +from uncloud.hack.mac import MAC +from uncloud.hack.net import VXLANBridge, DNSRA + +from uncloud import UncloudException + +arg_parser = argparse.ArgumentParser('hack', add_help=False) + #description="Commands that are unfinished - use at own risk") +arg_parser.add_argument('--last-used-mac', action='store_true') +arg_parser.add_argument('--get-new-mac', action='store_true') + +arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true') +arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true') +arg_parser.add_argument('--network', help="/64 IPv6 network") +arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0") +arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) +arg_parser.add_argument('--run-dns-ra', action='store_true', + help="Provide router advertisements and DNS resolution via dnsmasq") +arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') + +arg_parser.add_argument('--create-vm', action='store_true') +arg_parser.add_argument('--destroy-vm', action='store_true') +arg_parser.add_argument('--get-vm-status', action='store_true') +arg_parser.add_argument('--get-vm-vnc', action='store_true') +arg_parser.add_argument('--list-vms', action='store_true') +arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) +arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) +arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") +arg_parser.add_argument('--uuid', help="VM UUID") + +arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') +arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") + + +log = logging.getLogger(__name__) + + +def main(arguments): + config = Config(arguments) + + if arguments['create_vm']: + vm = VM(config) + vm.create() + + if arguments['destroy_vm']: + vm = VM(config) + vm.stop() + + if arguments['get_vm_status']: + vm = VM(config) + vm.status() + + if arguments['get_vm_vnc']: + vm = VM(config) + vm.vnc_addr() + + if arguments['list_vms']: + vm = VM(config) + vm.list() + + if arguments['last_used_mac']: + m = MAC(config) + print(m.last_used_mac()) + + if arguments['get_new_mac']: + print(MAC(config).get_next()) + + #if arguments['init_network']: + if arguments['create_vxlan']: + if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: + raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + vb = VXLANBridge(vni=arguments['vni'], + route=arguments['network'], + uplinkdev=arguments['vxlan_uplink_device'], + use_sudo=arguments['use_sudo']) + vb._setup_vxlan() + vb._setup_bridge() + vb._add_vxlan_to_bridge() + vb._route_network() + + if arguments['run_dns_ra']: + if not arguments['network'] or not arguments['vni']: + raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + + dnsra = DNSRA(route=arguments['network'], + vni=arguments['vni'], + use_sudo=arguments['use_sudo']) + dnsra._setup_dnsmasq() diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py new file mode 100644 index 0000000..f28ab7f --- /dev/null +++ b/uncloud/hack/net.py @@ -0,0 +1,116 @@ +import subprocess +import ipaddress +import logging + + +from uncloud import UncloudException + +log = logging.getLogger(__name__) + + +class VXLANBridge(object): + cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" + cmd_up_dev = "{sudo}ip link set {dev} up" + cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge" + cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up" + cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}" + cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}" + + # VXLAN ids are at maximum 24 bit - use a /104 + multicast_network = ipaddress.IPv6Network("ff05::/104") + max_vni = (2**24)-1 + + def __init__(self, + vni, + uplinkdev, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' + + self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + self.config['multicast_address'] = self.multicast_network[vni] + + self.config['route_network'] = ipaddress.IPv6Network(route) + self.config['route'] = route + + self.config['uplinkdev'] = uplinkdev + self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + + def setup_networking(self): + pass + + def _setup_vxlan(self): + self._execute_cmd(self.cmd_create_vxlan) + self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev']) + + def _setup_bridge(self): + self._execute_cmd(self.cmd_create_bridge) + self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev']) + + def _route_network(self): + self._execute_cmd(self.cmd_add_route_dev) + + def _add_vxlan_to_bridge(self): + self._execute_cmd(self.cmd_add_to_bridge) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + +class ManagementBridge(VXLANBridge): + pass + + +class DNSRA(object): + # VXLAN ids are at maximum 24 bit + max_vni = (2**24)-1 + + + # Command to start dnsmasq + cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra" + + def __init__(self, + vni, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' + + #TODO: remove if not needed + #self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + + # dnsmasq only wants the network without the prefix, therefore, cut it off + self.config['route'] = ipaddress.IPv6Network(route).network_address + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + def _setup_dnsmasq(self): + self._execute_cmd(self.cmd_start_dnsmasq) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + print("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + +class Firewall(object): + pass diff --git a/uncloud/hack/uncloud-hack-init-host b/uncloud/hack/uncloud-hack-init-host new file mode 100644 index 0000000..787ff80 --- /dev/null +++ b/uncloud/hack/uncloud-hack-init-host @@ -0,0 +1,26 @@ +id=100 +rawdev=eth0 + +# create vxlan +ip -6 link add vxlan${id} type vxlan \ + id ${id} \ + dstport 4789 \ + group ff05::${id} \ + dev ${rawdev} \ + ttl 5 + +ip link set vxlan${id} up + +# create bridge +ip link set vxlan${id} up +ip link set br${id} up + +# Add vxlan into bridge +ip link set vxlan${id} master br${id} + + +# useradd -m uncloud +# [18:05] tablett.place10:~# id uncloud +# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu) +# apk add qemu-system-x86_64 +# also needs group netdev diff --git a/uncloud/hack/uncloud-run-vm b/uncloud/hack/uncloud-run-vm new file mode 100644 index 0000000..33e5860 --- /dev/null +++ b/uncloud/hack/uncloud-run-vm @@ -0,0 +1,25 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo $0 vmid + exit 1 +fi + +id=$1; shift + +memory=512 +macaddress=02:00:b9:cb:70:${id} +netname=net${id}-1 + +qemu-system-x86_64 \ + -name uncloud-${id} \ + -accel kvm \ + -m ${memory} \ + -smp 2,sockets=2,cores=1,threads=1 \ + -device virtio-net-pci,netdev=net0,mac=$macaddress \ + -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \ + -vnc [::]:0 + +# To be changed: +# -vnc to unix path +# or -spice diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py new file mode 100755 index 0000000..ac403d8 --- /dev/null +++ b/uncloud/hack/vm.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +# This module is directly called from the hack module, and can be used as follow: +# +# Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix. +# `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2` +# +# List running VMs (returns a list of UUIDs). +# `uncloud hack --hackprefix /tmp/hackcloud --list-vms +# +# Get VM status: +# `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid` +# +# Stop a VM: +# `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` +# `` + +import subprocess +import uuid +import os +import logging + +from uncloud.hack.db import DB +from uncloud.hack.mac import MAC +from uncloud.vmm import VMM + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +class VM(object): + def __init__(self, config): + self.config = config + + #TODO: Enable etcd lookup + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(self.config, prefix="/vm") + + # General CLI arguments. + self.hackprefix = self.config.arguments['hackprefix'] + self.uuid = self.config.arguments['uuid'] + self.memory = self.config.arguments['memory'] or '1024M' + self.cores = self.config.arguments['cores'] or 1 + if self.config.arguments['image']: + self.image = os.path.join(self.hackprefix, self.config.arguments['image']) + else: + self.image = None + + # External components. + self.vmm = VMM(vmm_backend=self.hackprefix) + self.mac = MAC(self.config) + + # Harcoded & generated values. + self.owner = 'uncoud' + self.image_format='qcow2' + self.accel = 'kvm' + self.threads = 1 + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + self.ifname = "uc{}".format(self.mac.to_str_format()) + + def get_qemu_args(self): + command = ( + "-name {owner}-{name}" + " -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}" + " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" + ).format( + owner=self.owner, name=self.uuid, + accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname, + mac=self.mac + ) + + return command.split(" ") + + def create(self): + # New VM: new UUID, new MAC. + self.uuid = str(uuid.uuid4()) + self.mac.create() + + qemu_args = self.get_qemu_args() + log.debug("QEMU args passed to VMM: {}".format(qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + self.vmm.stop(self.uuid) + + def status(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_status(self.uuid)) + + def vnc_addr(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_vnc(self.uuid)) + + def list(self): + print(self.vmm.discover()) + diff --git a/uncloud/host/main.py b/uncloud/host/main.py index d1e7c9a..f680991 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -1,17 +1,20 @@ import argparse import multiprocessing as mp import time + from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared from uncloud.common.vm import VMStatus from uncloud.vmm import VMM from os.path import join as join_path from . import virtualmachine, logger +arg_parser = argparse.ArgumentParser('host', add_help=False) +arg_parser.add_argument('--hostname', required=True) + def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" @@ -29,10 +32,10 @@ def maintenance(host): vmm = VMM() running_vms = vmm.discover() for vm_uuid in running_vms: - if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running": + if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], vm_uuid) + join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) ) vm.status = VMStatus.running vm.vnc_socket = vmm.get_vnc(vm_uuid) @@ -40,77 +43,81 @@ def maintenance(host): shared.vm_pool.put(vm) -def main(hostname): +def main(arguments): + hostname = arguments['hostname'] host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) # Does not yet exist, create it if not host: host_key = join_path( - settings["etcd"]["host_prefix"], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { - "specs": "", - "hostname": hostname, - "status": "DEAD", - "last_heartbeat": "", + 'specs': '', + 'hostname': hostname, + 'status': 'DEAD', + 'last_heartbeat': '', } shared.etcd_client.put( host_key, host_entry, value_in_json=True ) + # update, get ourselves now for sure + host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + try: heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() except Exception as e: - raise Exception("uncloud-host heartbeat updating mechanism is not working") from e + raise Exception('uncloud-host heartbeat updating mechanism is not working') from e - for events_iterator in [ - shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True), - shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True) - ]: - for request_event in events_iterator: - request_event = RequestEntry(request_event) + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for events_iterator in [ + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False) + ]: + for request_event in events_iterator: + request_event = RequestEntry(request_event) - if request_event.type == "TIMEOUT": maintenance(host.key) - elif request_event.hostname == host.key: - logger.debug("VM Request: %s on Host %s", request_event, host.hostname) - shared.request_pool.client.client.delete(request_event.key) - vm_entry = shared.etcd_client.get( - join_path(settings["etcd"]["vm_prefix"], request_event.uuid) - ) - logger.debug("VM hostname: {}".format(vm_entry.value)) - vm = virtualmachine.VM(vm_entry) - if request_event.type == RequestType.StartVM: - vm.start() + if request_event.hostname == host.key: + logger.debug('VM Request: %s on Host %s', request_event, host.hostname) - elif request_event.type == RequestType.StopVM: - vm.stop() + shared.request_pool.client.client.delete(request_event.key) + vm_entry = shared.etcd_client.get( + join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) + ) - elif request_event.type == RequestType.DeleteVM: - vm.delete() + logger.debug('VM hostname: {}'.format(vm_entry.value)) - elif request_event.type == RequestType.InitVMMigration: - vm.start(destination_host_key=host.key) + vm = virtualmachine.VM(vm_entry) + if request_event.type == RequestType.StartVM: + vm.start() - elif request_event.type == RequestType.TransferVM: - destination_host = host_pool.get(request_event.destination_host_key) - if destination_host: - vm.migrate( - destination_host=destination_host.hostname, - destination_sock_path=request_event.destination_sock_path, - ) - else: - logger.error("Host %s not found!", request_event.destination_host_key) + elif request_event.type == RequestType.StopVM: + vm.stop() + elif request_event.type == RequestType.DeleteVM: + vm.delete() -if __name__ == "__main__": - argparser = argparse.ArgumentParser() - argparser.add_argument( - "hostname", help="Name of this host. e.g uncloud1.ungleich.ch" - ) - args = argparser.parse_args() - mp.set_start_method("spawn") - main(args.hostname) + elif request_event.type == RequestType.InitVMMigration: + vm.start(destination_host_key=host.key) + + elif request_event.type == RequestType.TransferVM: + destination_host = host_pool.get(request_event.destination_host_key) + if destination_host: + vm.migrate( + destination_host=destination_host.hostname, + destination_sock_path=request_event.destination_sock_path, + ) + else: + logger.error('Host %s not found!', request_event.destination_host_key) diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index 0bd20bf..a592efc 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -16,8 +16,7 @@ from uncloud.common.vm import VMStatus, declare_stopped from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared from uncloud.vmm import VMM from marshmallow import ValidationError @@ -42,7 +41,7 @@ class VM: def get_qemu_args(self): command = ( - "-drive file={file},format=raw,if=virtio,cache=none" + "-drive file={file},format=raw,if=virtio" " -device virtio-rng-pci" " -m {memory} -smp cores={cores},threads={threads}" " -name {owner}_{name}" @@ -91,7 +90,7 @@ class VM: self.vmm.socket_dir, self.uuid ), destination_host_key=destination_host_key, # Where source host transfer VM - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) else: @@ -119,7 +118,7 @@ class VM: network_name, mac, tap = network_mac_and_tap _key = os.path.join( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], self.vm["owner"], network_name, ) @@ -133,13 +132,13 @@ class VM: if network["type"] == "vxlan": tap = create_vxlan_br_tap( _id=network["id"], - _dev=settings["network"]["vxlan_phy_dev"], + _dev=shared.settings["network"]["vxlan_phy_dev"], tap_id=tap, ip=network["ipv6"], ) all_networks = shared.etcd_client.get_prefix( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], value_in_json=True, ) @@ -153,7 +152,10 @@ class VM: ) ) - return command.split(" ") + if command: + command = command.split(' ') + + return command def delete_network_dev(self): try: @@ -226,7 +228,7 @@ class VM: def resolve_network(network_name, network_owner): network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], network_owner, network_name, ), diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index 93e4dd5..ee9da2e 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -1,13 +1,16 @@ import json import os +import argparse import subprocess as sp from os.path import join as join_path -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.imagescanner import logger +arg_parser = argparse.ArgumentParser('imagescanner', add_help=False) + + def qemu_img_type(path): qemu_img_info_command = [ "qemu-img", @@ -26,10 +29,10 @@ def qemu_img_type(path): return qemu_img_info["format"] -def main(): +def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) images_to_be_created = list( filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) @@ -42,13 +45,13 @@ def main(): image_filename = image.value["filename"] image_store_name = image.value["store_name"] image_full_path = join_path( - settings["storage"]["file_dir"], + shared.settings["storage"]["file_dir"], image_owner, image_filename, ) image_stores = shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], + shared.settings["etcd"]["image_store_prefix"], value_in_json=True, ) user_image_store = next( diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index da993ae..374260e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -1,17 +1,22 @@ import os +import argparse from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared app = Flask(__name__) api = Api(app) app.logger.handlers.clear() +DEFAULT_PORT=1234 + +arg_parser = argparse.ArgumentParser('metadata', add_help=False) +arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT)) + @app.errorhandler(Exception) def handle_exception(e): @@ -68,7 +73,7 @@ class Root(Resource): ) else: etcd_key = os.path.join( - settings["etcd"]["user_prefix"], + shared.settings["etcd"]["user_prefix"], data.value["owner_realm"], data.value["owner"], "key", @@ -80,40 +85,11 @@ class Root(Resource): data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys return data.value["metadata"], 200 - @staticmethod - def post(): - return {"message": "Previous Implementation is deprecated."} - # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True) - # print(data) - # if data: - # for k in request.json: - # if k not in data.value: - # data.value[k] = request.json[k] - # if k.endswith("-list"): - # data.value[k] = [request.json[k]] - # else: - # if k.endswith("-list"): - # data.value[k].append(request.json[k]) - # else: - # data.value[k] = request.json[k] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data.value, value_in_json=True) - # else: - # data = {} - # for k in request.json: - # data[k] = request.json[k] - # if k.endswith("-list"): - # data[k] = [request.json[k]] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data, value_in_json=True) - api.add_resource(Root, "/") -def main(): - app.run(debug=True, host="::", port="80") - - -if __name__ == "__main__": - main() +def main(arguments): + port = arguments['port'] + debug = arguments['debug'] + app.run(debug=debug, host="::", port=port) diff --git a/uncloud/oneshot/__init__.py b/uncloud/oneshot/__init__.py new file mode 100644 index 0000000..eea436a --- /dev/null +++ b/uncloud/oneshot/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/uncloud/oneshot/main.py b/uncloud/oneshot/main.py new file mode 100644 index 0000000..5b9b61c --- /dev/null +++ b/uncloud/oneshot/main.py @@ -0,0 +1,123 @@ +import argparse +import os + + +from pathlib import Path +from uncloud.vmm import VMM +from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap + +from . import virtualmachine, logger + +### +# Argument parser loaded by scripts/uncloud. +arg_parser = argparse.ArgumentParser('oneshot', add_help=False) + +# Actions. +arg_parser.add_argument('--list', action='store_true', + help='list UUID and name of running VMs') +arg_parser.add_argument('--start', nargs=3, + metavar=('IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'), + help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix') +arg_parser.add_argument('--stop', metavar='UUID', + help='stop a VM') +arg_parser.add_argument('--get-status', metavar='UUID', + help='return the status of the VM') +arg_parser.add_argument('--get-vnc', metavar='UUID', + help='return the path of the VNC socket of the VM') +arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK', + help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix') + +# Arguments. +arg_parser.add_argument('--workdir', default=Path.home(), + help='Working directory, defaulting to $HOME') +arg_parser.add_argument('--mac', + help='MAC address of the VM to create (--start)') +arg_parser.add_argument('--memory', type=int, + help='Memory (MB) to allocate (--start)') +arg_parser.add_argument('--cores', type=int, + help='Number of cores to allocate (--start)') +arg_parser.add_argument('--threads', type=int, + help='Number of threads to allocate (--start)') +arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], + help='Format of OS image (--start)') +arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm', + help='QEMU acceleration to use (--start)') +arg_parser.add_argument('--upstream-interface', default='eth0', + help='Name of upstream interface (--start)') + +### +# Helpers. + +# XXX: check if it is possible to use the type returned by ETCD queries. +class UncloudEntryWrapper: + def __init__(self, value): + self.value = value + + def value(self): + return self.value + +def status_line(vm): + return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status()) + +### +# Entrypoint. + +def main(arguments): + # Initialize VMM. + workdir = arguments['workdir'] + vmm = VMM(vmm_backend=workdir) + + # Harcoded debug values. + net_id = 0 + + # Build VM configuration. + vm_config = {} + vm_options = [ + 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', + '--upstream_interface', 'upstream_interface', 'network', 'accel' + ] + for option in vm_options: + if arguments.get(option): + vm_config[option] = arguments[option] + + vm_config['net_id'] = net_id + + # Execute requested VM action. + if arguments['reconfigure_radvd']: + # TODO: check that RADVD is available. + prefix = arguments['reconfigure_radvd'] + network = UncloudEntryWrapper({ + 'id': net_id, + 'ipv6': prefix + }) + + # Make use of uncloud.host.virtualmachine for network configuration. + update_radvd_conf([network]) + elif arguments['start']: + # Extract from --start positional arguments. Quite fragile. + vm_config['image'] = arguments['start'][0] + vm_config['network'] = arguments['start'][1] + vm_config['upstream_interface'] = arguments['start'][2] + + vm_config['tap_interface'] = "uc{}".format(len(vmm.discover())) + vm = virtualmachine.VM(vmm, vm_config) + vm.start() + elif arguments['stop']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']}) + vm = virtualmachine.VM(vmm, vm_config) + vm.stop() + elif arguments['get_status']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']}) + print(status_line(vm)) + elif arguments['get_vnc']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']}) + print(vm.get_vnc_addr()) + elif arguments['list']: + vms = vmm.discover() + print("Found {} VMs.".format(len(vms))) + for uuid in vms: + vm = virtualmachine.VM(vmm, {'uuid': uuid}) + print(status_line(vm)) + else: + print('Please specify an action: --start, --stop, --list,\ +--get-status, --get-vnc, --reconfigure-radvd') diff --git a/uncloud/oneshot/virtualmachine.py b/uncloud/oneshot/virtualmachine.py new file mode 100644 index 0000000..c8c2909 --- /dev/null +++ b/uncloud/oneshot/virtualmachine.py @@ -0,0 +1,81 @@ +import uuid +import os + +from uncloud.host.virtualmachine import create_vxlan_br_tap +from uncloud.oneshot import logger + +class VM(object): + def __init__(self, vmm, config): + self.config = config + self.vmm = vmm + + # Extract VM specs/metadata from configuration. + self.name = config.get('name', 'no-name') + self.memory = config.get('memory', 1024) + self.cores = config.get('cores', 1) + self.threads = config.get('threads', 1) + self.image_format = config.get('image_format', 'qcow2') + self.image = config.get('image') + self.uuid = config.get('uuid', str(uuid.uuid4())) + self.mac = config.get('mac') + self.accel = config.get('accel', 'kvm') + + self.net_id = config.get('net_id', 0) + self.upstream_interface = config.get('upstream_interface', 'eth0') + self.tap_interface = config.get('tap_interface', 'uc0') + self.network = config.get('network') + + def get_qemu_args(self): + command = ( + "-uuid {uuid} -name {name} -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" + ).format( + uuid=self.uuid, name=self.name, accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + net_id=self.net_id, tap=self.tap_interface, mac=self.mac + ) + + return command.split(" ") + + def start(self): + # Check that VM image is available. + if not os.path.isfile(self.image): + logger.error("Image {} does not exist. Aborting.".format(self.image)) + + # Create Bridge, VXLAN and tap interface for VM. + create_vxlan_br_tap( + self.net_id, self.upstream_interface, self.tap_interface, self.network + ) + + # Generate config for and run QEMU. + qemu_args = self.get_qemu_args() + logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + self.vmm.stop(self.uuid) + + def get_status(self): + return self.vmm.get_status(self.uuid) + + def get_vnc_addr(self): + return self.vmm.get_vnc(self.uuid) + + def get_uuid(self): + return self.uuid + + def get_name(self): + success, json = self.vmm.execute_command(uuid, 'query-name') + if success: + return json['return']['name'] + + return None diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index 7edf623..79db322 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -6,8 +6,7 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared def accumulated_specs(vms_specs): @@ -130,7 +129,7 @@ def assign_host(vm): type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 5a4014f..38c07bf 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -4,72 +4,48 @@ # 2. Introduce a status endpoint of the scheduler - # maybe expose a prometheus compatible output +import argparse + from uncloud.common.request import RequestEntry, RequestType -from uncloud.shared import shared -from uncloud.settings import settings -from .helper import ( - dead_host_mitigation, - dead_host_detection, - assign_host, - NoSuitableHostFound, -) -from . import logger +from uncloud.common.shared import shared +from uncloud.scheduler import logger +from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection, + assign_host, NoSuitableHostFound) + +arg_parser = argparse.ArgumentParser('scheduler', add_help=False) -def main(): - for request_iterator in [ - shared.etcd_client.get_prefix( - settings["etcd"]["request_prefix"], value_in_json=True - ), - shared.etcd_client.watch_prefix( - settings["etcd"]["request_prefix"], - timeout=5, - value_in_json=True, - ), - ]: - for request_event in request_iterator: - request_entry = RequestEntry(request_event) - # Never Run time critical mechanism inside timeout - # mechanism because timeout mechanism only comes - # when no other event is happening. It means under - # heavy load there would not be a timeout event. - if request_entry.type == "TIMEOUT": +def main(arguments): + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for request_iterator in [ + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + ]: + for request_event in request_iterator: + dead_host_mitigation(dead_host_detection()) + request_entry = RequestEntry(request_event) - # Detect hosts that are dead and set their status - # to "DEAD", and their VMs' status to "KILLED" - dead_hosts = dead_host_detection() - if dead_hosts: - logger.debug("Dead hosts: %s", dead_hosts) - dead_host_mitigation(dead_hosts) + if request_entry.type == RequestType.ScheduleVM: + logger.debug('%s, %s', request_entry.key, request_entry.value) - elif request_entry.type == RequestType.ScheduleVM: - print(request_event.value) - logger.debug( - "%s, %s", request_entry.key, request_entry.value - ) + vm_entry = shared.vm_pool.get(request_entry.uuid) + if vm_entry is None: + logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid)) + continue - vm_entry = shared.vm_pool.get(request_entry.uuid) - if vm_entry is None: - logger.info( - "Trying to act on {} but it is deleted".format( - request_entry.uuid - ) - ) - continue - shared.etcd_client.client.delete( - request_entry.key - ) # consume Request + shared.etcd_client.client.delete(request_entry.key) # consume Request - try: - assign_host(vm_entry) - except NoSuitableHostFound: - vm_entry.add_log( - "Can't schedule VM. No Resource Left." - ) - shared.vm_pool.put(vm_entry) + try: + assign_host(vm_entry) + except NoSuitableHostFound: + vm_entry.add_log('Can\'t schedule VM. No Resource Left.') + shared.vm_pool.put(vm_entry) - logger.info("No Resource Left. Emailing admin....") - - -if __name__ == "__main__": - main() + logger.info('No Resource Left. Emailing admin....') diff --git a/uncloud/settings/__init__.py b/uncloud/settings/__init__.py deleted file mode 100644 index 629660e..0000000 --- a/uncloud/settings/__init__.py +++ /dev/null @@ -1,128 +0,0 @@ -import configparser -import logging -import sys -import os - -from uncloud.common.etcd_wrapper import Etcd3Wrapper - -logger = logging.getLogger(__name__) - - -class CustomConfigParser(configparser.RawConfigParser): - def __getitem__(self, key): - try: - result = super().__getitem__(key) - except KeyError as err: - raise KeyError( - "Key '{}' not found in configuration. Make sure you configure uncloud.".format( - key - ) - ) from err - else: - return result - - -class Settings(object): - def __init__(self, config_key="/uncloud/config/"): - conf_name = "uncloud.conf" - conf_dir = os.environ.get( - "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/") - ) - self.config_file = os.path.join(conf_dir, conf_name) - - self.config_parser = CustomConfigParser(allow_no_value=True) - self.config_key = config_key - - self.read_internal_values() - try: - self.config_parser.read(self.config_file) - except Exception as err: - logger.error("%s", err) - - def get_etcd_client(self): - args = tuple() - try: - kwargs = { - "host": self.config_parser.get("etcd", "url"), - "port": self.config_parser.get("etcd", "port"), - "ca_cert": self.config_parser.get("etcd", "ca_cert"), - "cert_cert": self.config_parser.get( - "etcd", "cert_cert" - ), - "cert_key": self.config_parser.get("etcd", "cert_key"), - } - except configparser.Error as err: - raise configparser.Error( - "{} in config file {}".format( - err.message, self.config_file - ) - ) from err - else: - try: - wrapper = Etcd3Wrapper(*args, **kwargs) - except Exception as err: - logger.error( - "etcd connection not successfull. Please check your config file." - "\nDetails: %s\netcd connection parameters: %s", - err, - kwargs, - ) - sys.exit(1) - else: - return wrapper - - def read_internal_values(self): - self.config_parser.read_dict( - { - "etcd": { - "file_prefix": "/files/", - "host_prefix": "/hosts/", - "image_prefix": "/images/", - "image_store_prefix": "/imagestore/", - "network_prefix": "/networks/", - "request_prefix": "/requests/", - "user_prefix": "/users/", - "vm_prefix": "/vms/", - } - } - ) - - def read_config_file_values(self, config_file): - try: - # Trying to read configuration file - with open(config_file, "r") as config_file_handle: - self.config_parser.read_file(config_file_handle) - except FileNotFoundError: - sys.exit( - "Configuration file {} not found!".format(config_file) - ) - except Exception as err: - logger.exception(err) - sys.exit("Error occurred while reading configuration file") - - def read_values_from_etcd(self): - etcd_client = self.get_etcd_client() - config_from_etcd = etcd_client.get( - self.config_key, value_in_json=True - ) - if config_from_etcd: - self.config_parser.read_dict(config_from_etcd.value) - else: - raise KeyError( - "Key '{}' not found in etcd. Please configure uncloud.".format( - self.config_key - ) - ) - - def __getitem__(self, key): - # Allow failing to read from etcd if we have - # it locally - try: - self.read_values_from_etcd() - except KeyError as e: - pass - - return self.config_parser[key] - - -settings = Settings() diff --git a/uncloud/shared/__init__.py b/uncloud/shared/__init__.py deleted file mode 100644 index db2093f..0000000 --- a/uncloud/shared/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -from uncloud.settings import settings -from uncloud.common.vm import VmPool -from uncloud.common.host import HostPool -from uncloud.common.request import RequestPool -from uncloud.common.storage_handlers import get_storage_handler - - -class Shared: - @property - def etcd_client(self): - return settings.get_etcd_client() - - @property - def host_pool(self): - return HostPool( - self.etcd_client, settings["etcd"]["host_prefix"] - ) - - @property - def vm_pool(self): - return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) - - @property - def request_pool(self): - return RequestPool( - self.etcd_client, settings["etcd"]["request_prefix"] - ) - - @property - def storage_handler(self): - return get_storage_handler() - - -shared = Shared() diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 6cdd938..6db61eb 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -100,9 +100,9 @@ class TransferVM(Process): class VMM: # Virtual Machine Manager def __init__( - self, - qemu_path="/usr/bin/qemu-system-x86_64", - vmm_backend=os.path.expanduser("~/uncloud/vmm/"), + self, + qemu_path="/usr/bin/qemu-system-x86_64", + vmm_backend=os.path.expanduser("~/uncloud/vmm/"), ): self.qemu_path = qemu_path self.vmm_backend = vmm_backend @@ -125,7 +125,7 @@ class VMM: os.makedirs(self.socket_dir, exist_ok=True) def is_running(self, uuid): - sock_path = os.path.join(self.vmm_backend, uuid) + sock_path = os.path.join(self.socket_dir, uuid) try: sock = socket.socket(socket.AF_UNIX) sock.connect(sock_path) @@ -163,7 +163,7 @@ class VMM: qmp_arg = ( "-qmp", "unix:{},server,nowait".format( - join_path(self.vmm_backend, uuid) + join_path(self.socket_dir, uuid) ), ) vnc_arg = ( @@ -190,18 +190,10 @@ class VMM: err.stderr.decode("utf-8"), ) else: - with suppress(sp.CalledProcessError): - sp.check_output( - [ - "sudo", - "-p", - "Enter password to correct permission for uncloud-vmm's directory", - "chmod", - "-R", - "o=rwx,g=rwx", - self.vmm_backend, - ] - ) + sp.check_output( + ["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory", + "chmod", "-R", "o=rwx,g=rwx", self.vmm_backend] + ) # TODO: Find some good way to check whether the virtual machine is up and # running without relying on non-guarenteed ways. @@ -220,7 +212,7 @@ class VMM: def execute_command(self, uuid, command, **kwargs): # execute_command -> sucess?, output try: - with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as ( + with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as ( sock_handle, file_handle, ): @@ -263,8 +255,8 @@ class VMM: def discover(self): vms = [ uuid - for uuid in os.listdir(self.vmm_backend) - if not isdir(join_path(self.vmm_backend, uuid)) + for uuid in os.listdir(self.socket_dir) + if not isdir(join_path(self.socket_dir, uuid)) ] return vms