diff --git a/.gitignore b/.gitignore index 6f0d9df..5c55899 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,3 @@ uncloud/version.py build/ venv/ dist/ - -*.iso diff --git a/bin/gen-version b/bin/gen-version index 06c3e22..a2e2882 100755 --- a/bin/gen-version +++ b/bin/gen-version @@ -1,22 +1,22 @@ #!/bin/sh # -*- coding: utf-8 -*- # -# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org) +# 2019 Nico Schottelius (nico-ucloud at schottelius.org) # -# This file is part of uncloud. +# This file is part of ucloud. # -# uncloud is free software: you can redistribute it and/or modify +# ucloud is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# uncloud is distributed in the hope that it will be useful, +# ucloud is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with uncloud. If not, see . +# along with ucloud. If not, see . # # @@ -26,4 +26,4 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py +printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py diff --git a/bin/uncloud-run-reinstall b/bin/uncloud-run-reinstall index b211613..18e95c0 100755 --- a/bin/uncloud-run-reinstall +++ b/bin/uncloud-run-reinstall @@ -24,6 +24,6 @@ dir=${0%/*} ${dir}/gen-version; -pip uninstall -y uncloud >/dev/null -python setup.py install >/dev/null +pip uninstall -y uncloud +python setup.py install ${dir}/uncloud "$@" diff --git a/docs/source/hacking.rst b/docs/source/hacking.rst deleted file mode 100644 index 1c750d6..0000000 --- a/docs/source/hacking.rst +++ /dev/null @@ -1,36 +0,0 @@ -Hacking -======= -Using uncloud in hacking (aka development) mode. - - -Get the code ------------- -.. code-block:: sh - :linenos: - - git clone https://code.ungleich.ch/uncloud/uncloud.git - - - -Install python requirements ---------------------------- -You need to have python3 installed. - -.. code-block:: sh - :linenos: - - cd uncloud! - python -m venv venv - . ./venv/bin/activate - ./bin/uncloud-run-reinstall - - - -Install os requirements ------------------------ -Install the following software packages: **dnsmasq**. - -If you already have a working IPv6 SLAAC and DNS setup, -this step can be skipped. - -Note that you need at least one /64 IPv6 network to run uncloud. diff --git a/docs/source/vm-images.rst b/docs/source/vm-images.rst deleted file mode 100644 index 4b2758a..0000000 --- a/docs/source/vm-images.rst +++ /dev/null @@ -1,66 +0,0 @@ -VM images -================================== - -Overview ---------- - -ucloud tries to be least invasise towards VMs and only require -strictly necessary changes for running in a virtualised -environment. This includes configurations for: - -* Configuring the network -* Managing access via ssh keys -* Resizing the attached disk(s) - -Upstream images ---------------- - -The 'official' uncloud images are defined in the `uncloud/images -`_ repository. - -How to make you own Uncloud images ----------------------------------- - -.. note:: - It is fairly easy to create your own images for uncloud, as the common - operations (which are detailed below) can be automatically handled by the - `uncloud/uncloud-init `_ tool. - -Network configuration -~~~~~~~~~~~~~~~~~~~~~ -All VMs in ucloud are required to support IPv6. The primary network -configuration is always done using SLAAC. A VM thus needs only to be -configured to - -* accept router advertisements on all network interfaces -* use the router advertisements to configure the network interfaces -* accept the DNS entries from the router advertisements - - -Configuring SSH keys -~~~~~~~~~~~~~~~~~~~~ - -To be able to access the VM, ucloud support provisioning SSH keys. - -To accept ssh keys in your VM, request the URL -*http://metadata/ssh_keys*. Add the content to the appropriate user's -**authorized_keys** file. Below you find sample code to accomplish -this task: - -.. code-block:: sh - - tmp=$(mktemp) - curl -s http://metadata/ssk_keys > "$tmp" - touch ~/.ssh/authorized_keys # ensure it exists - cat ~/.ssh/authorized_keys >> "$tmp" - sort "$tmp" | uniq > ~/.ssh/authorized_keys - - -Disk resize -~~~~~~~~~~~ -In virtualised environments, the disk sizes might grow. The operating -system should detect disks that are bigger than the existing partition -table and resize accordingly. This task is os specific. - -ucloud does not support shrinking disks due to the complexity and -intra OS dependencies. diff --git a/scripts/uncloud b/scripts/uncloud index 7d38e42..1ca9c68 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -3,86 +3,56 @@ import logging import sys import importlib import argparse -import os +import multiprocessing as mp -from etcd3.exceptions import ConnectionFailedError - -from uncloud.common import settings from uncloud import UncloudException -from uncloud.common.cli import resolve_otp_credentials +from contextlib import suppress -# Components that use etcd -ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', - 'imagescanner', 'metadata', 'configure', 'hack'] -ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('oneshot') -#ALL_COMPONENTS.append('cli') +def exception_hook(exc_type, exc_value, exc_traceback): + logging.getLogger(__name__).error( + 'Uncaught exception', + exc_info=(exc_type, exc_value, exc_traceback) + ) +sys.excepthook = exception_hook + if __name__ == '__main__': + # Setting up root logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + arg_parser = argparse.ArgumentParser() subparsers = arg_parser.add_subparsers(dest='command') parent_parser = argparse.ArgumentParser(add_help=False) parent_parser.add_argument('--debug', '-d', action='store_true', default=False, help='More verbose logging') - parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory', - default=os.path.expanduser('~/uncloud')) - etcd_parser = argparse.ArgumentParser(add_help=False) - etcd_parser.add_argument('--etcd-host') - etcd_parser.add_argument('--etcd-port') - etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') - etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') - etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') - - for component in ALL_COMPONENTS: + for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', + 'metadata', 'configure', 'cli']: mod = importlib.import_module('uncloud.{}.main'.format(component)) parser = getattr(mod, 'arg_parser') + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - if component in ETCD_COMPONENTS: - subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) - else: - subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - - arguments = vars(arg_parser.parse_args()) - etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value] - etcd_arguments = { - 'etcd': { - key.replace('etcd_', ''): arguments[key] - for key in etcd_arguments - } - } - if not arguments['command']: + args = arg_parser.parse_args() + if not args.command: arg_parser.print_help() else: - # Initializing Settings and resolving otp_credentials - # It is neccessary to resolve_otp_credentials after argument parsing is done because - # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and - # providing the default values for --name, --realm and --seed arguments from the values - # we read from file. But, now we are asking user about where the config file lives. So, - # to providing default value is not possible before parsing arguments. So, we are doing - # it after.. -# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) -# resolve_otp_credentials(arguments) - - name = arguments.pop('command') - mod = importlib.import_module('uncloud.{}.main'.format(name)) - main = getattr(mod, 'main') - - if arguments['debug']: - logging.basicConfig(level=logging.DEBUG) - else: - logging.basicConfig(level=logging.INFO) - - log = logging.getLogger() + # if we start etcd in seperate process with default settings + # i.e inheriting few things from parent process etcd3 module + # errors out, so the following command configure multiprocessing + # module to not inherit anything from parent. + # mp.set_start_method('spawn') + arguments = vars(args) try: - main(arguments) + name = arguments.pop('command') + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') + main(**arguments) except UncloudException as err: - log.error(err) -# except ConnectionFailedError as err: -# log.error('Cannot connect to etcd: {}'.format(err)) + logger.error(err) except Exception as err: - log.exception(err) + logger.exception(err) diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py deleted file mode 100755 index ba9fb37..0000000 --- a/uncloud/api/common_fields.py +++ /dev/null @@ -1,59 +0,0 @@ -import os - -from uncloud.common.shared import shared - - -class Optional: - pass - - -class Field: - def __init__(self, _name, _type, _value=None): - self.name = _name - self.value = _value - self.type = _type - self.__errors = [] - - def validation(self): - return True - - def is_valid(self): - if self.value == KeyError: - self.add_error( - "'{}' field is a required field".format(self.name) - ) - else: - if isinstance(self.value, Optional): - pass - elif not isinstance(self.value, self.type): - self.add_error( - "Incorrect Type for '{}' field".format(self.name) - ) - else: - self.validation() - - if self.__errors: - return False - return True - - def get_errors(self): - return self.__errors - - def add_error(self, error): - self.__errors.append(error) - - -class VmUUIDField(Field): - def __init__(self, data): - self.uuid = data.get("uuid", KeyError) - - super().__init__("uuid", str, self.uuid) - - self.validation = self.vm_uuid_validation - - def vm_uuid_validation(self): - r = shared.etcd_client.get( - os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid) - ) - if not r: - self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 90e0f92..802567f 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -4,6 +4,7 @@ import os from uuid import uuid4 from uncloud.common.shared import shared +from uncloud.common.settings import settings data = { 'is_public': True, @@ -13,7 +14,4 @@ data = { 'attributes': {'list': [], 'key': [], 'pool': 'images'}, } -shared.etcd_client.put( - os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex), - json.dumps(data), -) +shared.etcd_client.put(os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index 8ceb3a6..c0205c8 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -1,12 +1,11 @@ import binascii -import ipaddress -import random import logging import requests from pyotp import TOTP from uncloud.common.shared import shared +from uncloud.common.settings import settings logger = logging.getLogger(__name__) @@ -14,25 +13,19 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": shared.settings["otp"]["auth_name"], - "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), - "auth_realm": shared.settings["otp"]["auth_realm"], - "name": name, - "realm": realm, - "token": token, + 'auth_name': settings['otp']['auth_name'], + 'auth_token': TOTP(settings['otp']['auth_seed']).now(), + 'auth_realm': settings['otp']['auth_realm'], + 'name': name, + 'realm': realm, + 'token': token, } - except binascii.Error as err: - logger.error( - "Cannot compute OTP for seed: {}".format( - shared.settings["otp"]["auth_seed"] - ) - ) + except binascii.Error: + logger.error('Cannot compute OTP for seed: {}'.format(settings['otp']['auth_seed'])) return 400 - - response = requests.post( - shared.settings["otp"]["verification_controller_url"], json=data - ) - return response.status_code + else: + response = requests.post(settings['otp']['verification_controller_url'], json=data) + return response.status_code def resolve_vm_name(name, owner): @@ -42,29 +35,24 @@ def resolve_vm_name(name, owner): Output: uuid of vm if found otherwise None """ result = next( - filter( - lambda vm: vm.value["owner"] == owner - and vm.value["name"] == name, - shared.vm_pool.vms, - ), - None, + filter(lambda vm: vm.value['owner'] == owner and vm.value['name'] == name, shared.vm_pool.vms), + None ) if result: - return result.key.split("/")[-1] + return result.key.split('/')[-1] return None -def resolve_image_name(name, etcd_client): +def resolve_image_name(name): """Return image uuid given its name and its store * If the provided name is not in correct format i.e {store_name}:{image_name} return ValueError * If no such image found then return KeyError - """ - seperator = ":" + seperator = ':' # Ensure, user/program passed valid name that is of type string try: @@ -72,77 +60,35 @@ def resolve_image_name(name, etcd_client): """ Examples, where it would work and where it would raise exception - "images:alpine" --> ["images", "alpine"] + 'images:alpine' --> ['images', 'alpine'] - "images" --> ["images"] it would raise Exception as non enough value to unpack + 'images' --> ['images'] it would raise Exception as non enough value to unpack - "images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception + 'images:alpine:meow' --> ['images', 'alpine', 'meow'] it would raise Exception as too many values to unpack """ store_name, image_name = store_name_and_image_name except Exception: - raise ValueError( - "Image name not in correct format i.e {store_name}:{image_name}" - ) + raise ValueError('Image name not in correct format i.e {store_name}:{image_name}') - images = etcd_client.get_prefix( - shared.settings["etcd"]["image_prefix"], value_in_json=True - ) + images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True) # Try to find image with name == image_name and store_name == store_name try: image = next( filter( - lambda im: im.value["name"] == image_name - and im.value["store_name"] == store_name, + lambda im: im.value['name'] == image_name + and im.value['store_name'] == store_name, images, ) ) except StopIteration: - raise KeyError("No image with name {} found.".format(name)) + raise KeyError('No image with name {} found.'.format(name)) else: - image_uuid = image.key.split("/")[-1] + image_uuid = image.key.split('/')[-1] return image_uuid -def random_bytes(num=6): - return [random.randrange(256) for _ in range(num)] - - -def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): - mac = random_bytes() - if oui: - if type(oui) == str: - oui = [int(chunk) for chunk in oui.split(separator)] - mac = oui + random_bytes(num=6 - len(oui)) - else: - if multicast: - mac[0] |= 1 # set bit 0 - else: - mac[0] &= ~1 # clear bit 0 - if uaa: - mac[0] &= ~(1 << 1) # clear bit 1 - else: - mac[0] |= 1 << 1 # set bit 1 - return separator.join(byte_fmt % b for b in mac) - - -def mac2ipv6(mac, prefix): - # only accept MACs separated by a colon - parts = mac.split(":") - - # modify parts to match IPv6 value - parts.insert(3, "ff") - parts.insert(4, "fe") - parts[0] = "%x" % (int(parts[0], 16) ^ 2) - - # format output - ipv6_parts = [str(0)] * 4 - for i in range(0, len(parts), 2): - ipv6_parts.append("".join(parts[i : i + 2])) - - lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) - prefix = ipaddress.IPv6Address(prefix) - return str(prefix + int(lower_part)) - +def make_return_message(err, status_code=200): + return {'message': str(err)}, status_code diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 73e8e21..032864c 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -14,9 +14,13 @@ from uncloud.common.shared import shared from uncloud.common import counters from uncloud.common.vm import VMStatus +from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.settings import settings from uncloud.api import schemas -from uncloud.api.helper import generate_mac, mac2ipv6 +from uncloud.api.schemas import ValidationException +from uncloud.common.network import generate_mac, mac2ipv6 +from uncloud.api.helper import make_return_message from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -29,51 +33,52 @@ arg_parser = argparse.ArgumentParser('api', add_help=False) arg_parser.add_argument('--port', '-p') -@app.errorhandler(Exception) -def handle_exception(e): - app.logger.error(e) - # pass through HTTP errors - if isinstance(e, HTTPException): - return e - - # now you're handling non-HTTP exceptions only - return {'message': 'Server Error'}, 500 +# @app.errorhandler(Exception) +# def handle_exception(e): +# app.logger.error(e) +# +# # pass through HTTP errors +# if isinstance(e, HTTPException): +# return e +# +# # now you're handling non-HTTP exceptions only +# return {'message': 'Server Error'}, 500 class CreateVM(Resource): - """API Request to Handle Creation of VM""" - @staticmethod def post(): data = request.json - validator = schemas.CreateVMSchema(data) - if validator.is_valid(): + try: + validator = schemas.CreateVMSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: vm_uuid = uuid4().hex - vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) + vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) specs = { 'cpu': validator.specs['cpu'], 'ram': validator.specs['ram'], 'os-ssd': validator.specs['os-ssd'], 'hdd': validator.specs['hdd'], } - macs = [generate_mac() for _ in range(len(data['network']))] + macs = [generate_mac() for _ in range(len(validator.network))] tap_ids = [ - counters.increment_etcd_counter( - shared.etcd_client, shared.settings['etcd']['tap_counter'] - ) - for _ in range(len(data['network'])) + counters.increment_etcd_counter(settings['etcd']['tap_counter']) + for _ in range(len(validator.network)) ] vm_entry = { - 'name': data['vm_name'], - 'owner': data['name'], - 'owner_realm': data['realm'], + 'name': validator.vm_name, + 'owner': validator.name, + 'owner_realm': validator.realm, 'specs': specs, 'hostname': '', 'status': VMStatus.stopped, 'image_uuid': validator.image_uuid, 'log': [], 'vnc_socket': '', - 'network': list(zip(data['network'], macs, tap_ids)), + 'network': list(zip(validator.network, macs, tap_ids)), 'metadata': {'ssh-keys': []}, 'in_migration': False, } @@ -83,89 +88,79 @@ class CreateVM(Resource): r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=shared.settings['etcd']['request_prefix'], + request_prefix=settings['etcd']['request_prefix'], ) shared.request_pool.put(r) - - return {'message': 'VM Creation Queued'}, 200 - return validator.get_errors(), 400 + return make_return_message('VM Creation Queued') -class VmStatus(Resource): +class GetVMStatus(Resource): @staticmethod def post(): data = request.json - validator = schemas.VMStatusSchema(data) - if validator.is_valid(): - vm = shared.vm_pool.get( - join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) - ) + try: + validator = schemas.VMStatusSchema(data) + validator.is_valid() + except (ValidationException, Exception) as err: + return make_return_message(err, 400) + else: + vm = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], validator.uuid)) vm_value = vm.value.copy() vm_value['ip'] = [] for network_mac_and_tap in vm.network: network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( - join_path( - shared.settings['etcd']['network_prefix'], - data['name'], - network_name, - ), + join_path(settings['etcd']['network_prefix'], validator.name, network_name), value_in_json=True, ) - ipv6_addr = ( - network.value.get('ipv6').split('::')[0] + '::' - ) + ipv6_addr = (network.value.get('ipv6').split('::')[0] + '::') vm_value['ip'].append(mac2ipv6(mac, ipv6_addr)) vm.value = vm_value - return vm.value - else: - return validator.get_errors(), 400 + return vm.value, 200 class CreateImage(Resource): @staticmethod def post(): data = request.json - validator = schemas.CreateImageSchema(data) - if validator.is_valid(): - file_entry = shared.etcd_client.get( - join_path(shared.settings['etcd']['file_prefix'], data['uuid']) - ) - file_entry_value = json.loads(file_entry.value) + try: + validator = schemas.CreateImageSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + try: + file_entry = shared.etcd_client.get( + join_path(settings['etcd']['file_prefix'], validator.uuid), value_in_json=True + ) + except KeyError: + # TODO: Add some message + pass + else: + image_entry_json = { + 'status': 'TO_BE_CREATED', + 'owner': file_entry.value['owner'], + 'filename': file_entry.value['filename'], + 'name': validator.name, + 'store_name': validator.image_store, + 'visibility': 'public', + } + shared.etcd_client.put( + join_path(settings['etcd']['image_prefix'], validator.uuid), + json.dumps(image_entry_json), + ) - image_entry_json = { - 'status': 'TO_BE_CREATED', - 'owner': file_entry_value['owner'], - 'filename': file_entry_value['filename'], - 'name': data['name'], - 'store_name': data['image_store'], - 'visibility': 'public', - } - shared.etcd_client.put( - join_path( - shared.settings['etcd']['image_prefix'], data['uuid'] - ), - json.dumps(image_entry_json), - ) - - return {'message': 'Image queued for creation.'} - return validator.get_errors(), 400 + return {'message': 'Image queued for creation.'}, 200 class ListPublicImages(Resource): @staticmethod def get(): - images = shared.etcd_client.get_prefix( - shared.settings['etcd']['image_prefix'], value_in_json=True - ) + images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True) r = {'images': []} for image in images: - image_key = '{}:{}'.format( - image.value['store_name'], image.value['name'] - ) - r['images'].append( - {'name': image_key, 'status': image.value['status']} - ) + image_key = '{}:{}'.format(image.value['store_name'], image.value['name']) + r['images'].append({'name': image_key, 'status': image.value['status']}) return r, 200 @@ -173,92 +168,79 @@ class VMAction(Resource): @staticmethod def post(): data = request.json - validator = schemas.VmActionSchema(data) - - if validator.is_valid(): - vm_entry = shared.vm_pool.get( - join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) - ) - action = data['action'] + try: + validator = schemas.VmActionSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + vm_entry = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], validator.uuid)) + action = validator.action if action == 'start': action = 'schedule' if action == 'delete' and vm_entry.hostname == '': - if shared.storage_handler.is_vm_image_exists( - vm_entry.uuid - ): - r_status = shared.storage_handler.delete_vm_image( - vm_entry.uuid - ) + if shared.storage_handler.is_vm_image_exists(vm_entry.uuid): + r_status = shared.storage_handler.delete_vm_image(vm_entry.uuid) if r_status: shared.etcd_client.client.delete(vm_entry.key) - return {'message': 'VM successfully deleted'} + return make_return_message('VM successfully deleted') else: - logger.error( - 'Some Error Occurred while deleting VM' - ) - return {'message': 'VM deletion unsuccessfull'} + logger.error('Some Error Occurred while deleting VM') + return make_return_message('VM deletion unsuccessfull') else: shared.etcd_client.client.delete(vm_entry.key) - return {'message': 'VM successfully deleted'} + return make_return_message('VM successfully deleted') r = RequestEntry.from_scratch( type='{}VM'.format(action.title()), - uuid=data['uuid'], + uuid=validator.uuid, hostname=vm_entry.hostname, - request_prefix=shared.settings['etcd']['request_prefix'], + request_prefix=settings['etcd']['request_prefix'], ) shared.request_pool.put(r) - return ( - {'message': 'VM {} Queued'.format(action.title())}, - 200, - ) - else: - return validator.get_errors(), 400 + return make_return_message('VM {} Queued'.format(action.title())) class VMMigration(Resource): @staticmethod def post(): data = request.json - validator = schemas.VmMigrationSchema(data) - - if validator.is_valid(): - vm = shared.vm_pool.get(data['uuid']) + try: + validator = schemas.VmMigrationSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err), 400 + else: + vm = shared.vm_pool.get(validator.uuid) r = RequestEntry.from_scratch( type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( - shared.settings['etcd']['host_prefix'], - validator.destination.value, + settings['etcd']['host_prefix'], + validator.destination, ), - request_prefix=shared.settings['etcd']['request_prefix'], + request_prefix=settings['etcd']['request_prefix'], ) shared.request_pool.put(r) - return ( - {'message': 'VM Migration Initialization Queued'}, - 200, - ) - else: - return validator.get_errors(), 400 + return make_return_message('VM Migration Initialization Queued') class ListUserVM(Resource): @staticmethod def post(): data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - vms = shared.etcd_client.get_prefix( - shared.settings['etcd']['vm_prefix'], value_in_json=True - ) + try: + validator = schemas.OTPSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + vms = shared.etcd_client.get_prefix(settings['etcd']['vm_prefix'], value_in_json=True) return_vms = [] - user_vms = filter( - lambda v: v.value['owner'] == data['name'], vms - ) + user_vms = filter(lambda v: v.value['owner'] == validator.name, vms) for vm in user_vms: return_vms.append( { @@ -270,26 +252,22 @@ class ListUserVM(Resource): 'vnc_socket': vm.value.get('vnc_socket', None), } ) - if return_vms: - return {'message': return_vms}, 200 - return {'message': 'No VM found'}, 404 - - else: - return validator.get_errors(), 400 + return make_return_message(return_vms) class ListUserFiles(Resource): @staticmethod def post(): data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - files = shared.etcd_client.get_prefix( - shared.settings['etcd']['file_prefix'], value_in_json=True - ) + try: + validator = schemas.OTPSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + files = shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) return_files = [] - user_files = [f for f in files if f.value['owner'] == data['name']] + user_files = [f for f in files if f.value['owner'] == validator.name] for file in user_files: file_uuid = file.key.split('/')[-1] file = file.value @@ -299,33 +277,28 @@ class ListUserFiles(Resource): file.pop('owner', None) return_files.append(file) - return {'message': return_files}, 200 - else: - return validator.get_errors(), 400 + return make_return_message(return_files) class CreateHost(Resource): @staticmethod def post(): data = request.json - validator = schemas.CreateHostSchema(data) - if validator.is_valid(): - host_key = join_path( - shared.settings['etcd']['host_prefix'], uuid4().hex - ) + try: + validator = schemas.CreateHostSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + host_key = join_path(settings['etcd']['host_prefix'], uuid4().hex) host_entry = { - 'specs': data['specs'], - 'hostname': data['hostname'], - 'status': 'DEAD', + 'specs': validator.specs, + 'hostname': validator.hostname, + 'status': HostStatus.dead, 'last_heartbeat': '', } - shared.etcd_client.put( - host_key, host_entry, value_in_json=True - ) - - return {'message': 'Host Created'}, 200 - - return validator.get_errors(), 400 + shared.etcd_client.put(host_key, host_entry, value_in_json=True) + return make_return_message('Host Created.') class ListHost(Resource): @@ -347,200 +320,142 @@ class GetSSHKeys(Resource): @staticmethod def post(): data = request.json - validator = schemas.GetSSHSchema(data) - if validator.is_valid(): - if not validator.key_name.value: - - # {user_prefix}/{realm}/{name}/key/ - etcd_key = join_path( - shared.settings['etcd']['user_prefix'], - data['realm'], - data['name'], - 'key', - ) - etcd_entry = shared.etcd_client.get_prefix( - etcd_key, value_in_json=True - ) - + try: + validator = schemas.GetSSHSchema(data) + except ValidationException as err: + return make_return_message(err, 400) + else: + etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm, + validator.name, 'key') + if not validator.key_name: + etcd_entry = shared.etcd_client.get_prefix(etcd_key, value_in_json=True) keys = { key.key.split('/')[-1]: key.value for key in etcd_entry } return {'keys': keys} else: - - # {user_prefix}/{realm}/{name}/key/{key_name} - etcd_key = join_path( - shared.settings['etcd']['user_prefix'], - data['realm'], - data['name'], - 'key', - data['key_name'], - ) - etcd_entry = shared.etcd_client.get( - etcd_key, value_in_json=True - ) - - if etcd_entry: - return { - 'keys': { - etcd_entry.key.split('/')[ - -1 - ]: etcd_entry.value - } - } + etcd_key = join_path(validator.key_name) + try: + etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) + except KeyError: + return make_return_message('No such key found.', 400) else: - return {'keys': {}} - else: - return validator.get_errors(), 400 + return { + 'keys': {etcd_entry.key.split('/')[-1]: etcd_entry.value} + } class AddSSHKey(Resource): @staticmethod def post(): data = request.json - validator = schemas.AddSSHSchema(data) - if validator.is_valid(): - + try: + validator = schemas.AddSSHSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - shared.settings['etcd']['user_prefix'], - data['realm'], - data['name'], - 'key', - data['key_name'], + settings['etcd']['user_prefix'], validator.realm, + validator.name, 'key', validator.key_name ) - etcd_entry = shared.etcd_client.get( - etcd_key, value_in_json=True - ) - if etcd_entry: - return { - 'message': 'Key with name "{}" already exists'.format( - data['key_name'] - ) - } - else: + try: + shared.etcd_client.get(etcd_key, value_in_json=True) + except KeyError: # Key Not Found. It implies user' haven't added any key yet. - shared.etcd_client.put( - etcd_key, data['key'], value_in_json=True - ) - return {'message': 'Key added successfully'} - else: - return validator.get_errors(), 400 + shared.etcd_client.put(etcd_key, validator.key, value_in_json=True) + return make_return_message('Key added successfully') + else: + return make_return_message('Key "{}" already exists'.format(validator.key_name)) class RemoveSSHKey(Resource): @staticmethod def post(): data = request.json - validator = schemas.RemoveSSHSchema(data) - if validator.is_valid(): - + try: + validator = schemas.RemoveSSHSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: # {user_prefix}/{realm}/{name}/key/{key_name} - etcd_key = join_path( - shared.settings['etcd']['user_prefix'], - data['realm'], - data['name'], - 'key', - data['key_name'], - ) - etcd_entry = shared.etcd_client.get( - etcd_key, value_in_json=True - ) + etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm, + validator.name, 'key', validator.key_name) + try: + etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) + except KeyError: + return make_return_message('No Key "{}" exists.'.format(validator.key_name)) if etcd_entry: shared.etcd_client.client.delete(etcd_key) return {'message': 'Key successfully removed.'} - else: - return { - 'message': 'No Key with name "{}" Exists at all.'.format( - data['key_name'] - ) - } - else: - return validator.get_errors(), 400 class CreateNetwork(Resource): @staticmethod def post(): data = request.json - validator = schemas.CreateNetwork(data) - - if validator.is_valid(): - + try: + validator = schemas.CreateNetwork(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: network_entry = { - 'id': counters.increment_etcd_counter( - shared.etcd_client, shared.settings['etcd']['vxlan_counter'] - ), - 'type': data['type'], + 'id': counters.increment_etcd_counter(settings['etcd']['vxlan_counter']), + 'type': validator.type, } - if validator.user.value: + if validator.user: try: - nb = pynetbox.api( - url=shared.settings['netbox']['url'], - token=shared.settings['netbox']['token'], - ) - nb_prefix = nb.ipam.prefixes.get( - prefix=shared.settings['network']['prefix'] - ) + nb = pynetbox.api(url=settings['netbox']['url'], token=settings['netbox']['token']) + nb_prefix = nb.ipam.prefixes.get(prefix=settings['network']['prefix']) prefix = nb_prefix.available_prefixes.create( data={ - 'prefix_length': int( - shared.settings['network']['prefix_length'] - ), + 'prefix_length': int(settings['network']['prefix_length']), 'description': '{}\'s network "{}"'.format( - data['name'], data['network_name'] + validator.name, + validator.network_name ), 'is_pool': True, } ) except Exception as err: app.logger.error(err) - return { - 'message': 'Error occured while creating network.' - } + return make_return_message('Error occured while creating network.', 400) else: network_entry['ipv6'] = prefix['prefix'] else: network_entry['ipv6'] = 'fd00::/64' - network_key = join_path( - shared.settings['etcd']['network_prefix'], - data['name'], - data['network_name'], - ) - shared.etcd_client.put( - network_key, network_entry, value_in_json=True - ) - return {'message': 'Network successfully added.'} - else: - return validator.get_errors(), 400 + network_key = join_path(settings['etcd']['network_prefix'], validator.name, + validator.network_name) + shared.etcd_client.put(network_key, network_entry, value_in_json=True) + return make_return_message('Network successfully added.') class ListUserNetwork(Resource): @staticmethod def post(): data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - prefix = join_path( - shared.settings['etcd']['network_prefix'], data['name'] - ) - networks = shared.etcd_client.get_prefix( - prefix, value_in_json=True - ) + try: + validator = schemas.OTPSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + prefix = join_path(settings['etcd']['network_prefix'], validator.name) + networks = shared.etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] for net in networks: net.value['name'] = net.key.split('/')[-1] user_networks.append(net.value) return {'networks': user_networks}, 200 - else: - return validator.get_errors(), 400 api.add_resource(CreateVM, '/vm/create') -api.add_resource(VmStatus, '/vm/status') +api.add_resource(GetVMStatus, '/vm/status') api.add_resource(VMAction, '/vm/action') api.add_resource(VMMigration, '/vm/migrate') @@ -562,39 +477,12 @@ api.add_resource(ListHost, '/host/list') api.add_resource(CreateNetwork, '/network/create') -def main(arguments): - debug = arguments['debug'] - port = arguments['port'] - - try: - image_stores = list( - shared.etcd_client.get_prefix( - shared.settings['etcd']['image_store_prefix'], value_in_json=True - ) - ) - except KeyError: - image_stores = False - - # Do not inject default values that might be very wrong - # fail when required, not before - # - # if not image_stores: - # data = { - # 'is_public': True, - # 'type': 'ceph', - # 'name': 'images', - # 'description': 'first ever public image-store', - # 'attributes': {'list': [], 'key': [], 'pool': 'images'}, - # } - - # shared.etcd_client.put( - # join_path( - # shared.settings['etcd']['image_store_prefix'], uuid4().hex - # ), - # json.dumps(data), - # ) - +def main(debug=False, port=None): try: app.run(host='::', port=port, debug=debug) except OSError as e: raise UncloudException('Failed to start Flask: {}'.format(e)) + + +if __name__ == '__main__': + main() diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index 87f20c9..7d8969b 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -1,19 +1,3 @@ -""" -This module contain classes thats validates and intercept/modify -data coming from uncloud-cli (user) - -It was primarily developed as an alternative to argument parser -of Flask_Restful which is going to be deprecated. I also tried -marshmallow for that purpose but it was an overkill (because it -do validation + serialization + deserialization) and little -inflexible for our purpose. -""" - -# TODO: Fix error message when user's mentioned VM (referred by name) -# does not exists. -# -# Currently, it says uuid is a required field. - import json import os @@ -22,19 +6,54 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from . import helper, logger -from .common_fields import Field, VmUUIDField -from .helper import check_otp, resolve_vm_name +from uncloud.common.settings import settings +from uncloud.api import helper +from uncloud.api.helper import check_otp, resolve_vm_name + + +class ValidationException(Exception): + """Validation Error""" + + +class Field: + def __init__(self, _name, _type, _value=None, validators=None): + if validators is None: + validators = [] + + assert isinstance(validators, list) + + self.name = _name + self.value = _value + self.type = _type + self.validators = validators + + def is_valid(self): + if not isinstance(self.value, self.type): + raise ValidationException("Incorrect Type for '{}' field".format(self.name)) + + for validator in self.validators: + validator() + + def __repr__(self): + return self.name + + +class VmUUIDField(Field): + def __init__(self, data): + self.uuid = data.get('uuid', KeyError) + + super().__init__('uuid', str, self.uuid, validators=[self.vm_uuid_validation]) + + def vm_uuid_validation(self): + try: + shared.etcd_client.get(os.path.join(settings['etcd']['vm_prefix'], self.uuid)) + except KeyError: + raise ValidationException('VM with uuid {} does not exists'.format(self.uuid)) class BaseSchema: - def __init__(self, data, fields=None): - _ = data # suppress linter warning - self.__errors = [] - if fields is None: - self.fields = [] - else: - self.fields = fields + def __init__(self): + self.fields = [getattr(self, field) for field in dir(self) if isinstance(getattr(self, field), Field)] def validation(self): # custom validation is optional @@ -43,515 +62,340 @@ class BaseSchema: def is_valid(self): for field in self.fields: field.is_valid() - self.add_field_errors(field) for parent in self.__class__.__bases__: - try: - parent.validation(self) - except AttributeError: - pass - if not self.__errors: - self.validation() + parent.validation(self) - if self.__errors: - return False - return True + self.validation() - def get_errors(self): - return {"message": self.__errors} + for field in self.fields: + setattr(self, field.name, field.value) - def add_field_errors(self, field: Field): - self.__errors += field.get_errors() - def add_error(self, error): - self.__errors.append(error) +def get(dictionary: dict, key: str, return_default=False, default=None): + if dictionary is None: + raise ValidationException('No data provided at all.') + try: + value = dictionary[key] + except KeyError: + if return_default: + return default + raise ValidationException("Missing data for '{}' field.".format(key)) + else: + return value class OTPSchema(BaseSchema): - def __init__(self, data: dict, fields=None): - self.name = Field("name", str, data.get("name", KeyError)) - self.realm = Field("realm", str, data.get("realm", KeyError)) - self.token = Field("token", str, data.get("token", KeyError)) - - _fields = [self.name, self.realm, self.token] - if fields: - _fields += fields - super().__init__(data=data, fields=_fields) + def __init__(self, data: dict): + self.name = Field('name', str, get(data, 'name')) + self.realm = Field('realm', str, get(data, 'realm')) + self.token = Field('token', str, get(data, 'token')) + super().__init__() def validation(self): - if ( - check_otp( - self.name.value, self.realm.value, self.token.value - ) - != 200 - ): - self.add_error("Wrong Credentials") - - -########################## Image Operations ############################################### + if check_otp(self.name.value, self.realm.value, self.token.value) != 200: + raise ValidationException('Wrong Credentials') class CreateImageSchema(BaseSchema): def __init__(self, data): - # Fields - self.uuid = Field("uuid", str, data.get("uuid", KeyError)) - self.name = Field("name", str, data.get("name", KeyError)) - self.image_store = Field( - "image_store", str, data.get("image_store", KeyError) - ) - - # Validations - self.uuid.validation = self.file_uuid_validation - self.image_store.validation = self.image_store_name_validation - - # All Fields - fields = [self.uuid, self.name, self.image_store] - super().__init__(data, fields) + self.uuid = Field('uuid', str, get(data, 'uuid'), validators=[self.file_uuid_validation]) + self.name = Field('name', str, get(data, 'name')) + self.image_store = Field('image_store', str, get(data, 'image_store'), + validators=[self.image_store_name_validation]) + super().__init__() def file_uuid_validation(self): - file_entry = shared.etcd_client.get( - os.path.join( - shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value - ) - ) - if file_entry is None: - self.add_error( - "Image File with uuid '{}' Not Found".format( - self.uuid.value - ) - ) + try: + shared.etcd_client.get(os.path.join(settings['etcd']['file_prefix'], self.uuid.value)) + except KeyError: + raise ValidationException("Image File with uuid '{}' Not Found".format(self.uuid.value)) def image_store_name_validation(self): - image_stores = list( - shared.etcd_client.get_prefix( - shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] - ) - ) - - image_store = next( - filter( - lambda s: json.loads(s.value)["name"] - == self.image_store.value, - image_stores, - ), - None, - ) - if not image_store: - self.add_error( - "Store '{}' does not exists".format( - self.image_store.value - ) - ) - - -# Host Operations + image_stores = list(shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix'])) + try: + next(filter(lambda s: json.loads(s.value)['name'] == self.image_store.value, image_stores)) + except StopIteration: + raise ValidationException("Store '{}' does not exists".format(self.image_store.value)) class CreateHostSchema(OTPSchema): def __init__(self, data): - # Fields - self.specs = Field("specs", dict, data.get("specs", KeyError)) - self.hostname = Field( - "hostname", str, data.get("hostname", KeyError) - ) + self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation]) + self.hostname = Field('hostname', str, get(data, 'hostname')) - # Validation - self.specs.validation = self.specs_validation - - fields = [self.hostname, self.specs] - - super().__init__(data=data, fields=fields) + super().__init__(data) def specs_validation(self): - ALLOWED_BASE = 10 + allowed_base = 10 - _cpu = self.specs.value.get("cpu", KeyError) - _ram = self.specs.value.get("ram", KeyError) - _os_ssd = self.specs.value.get("os-ssd", KeyError) - _hdd = self.specs.value.get("hdd", KeyError) + _cpu = self.specs.value.get('cpu', KeyError) + _ram = self.specs.value.get('ram', KeyError) + _os_ssd = self.specs.value.get('os-ssd', KeyError) + _hdd = self.specs.value.get('hdd', KeyError) - if KeyError in [_cpu, _ram, _os_ssd, _hdd]: - self.add_error( - "You must specify CPU, RAM and OS-SSD in your specs" - ) - return None + if KeyError in [_cpu, _ram, _os_ssd]: + raise ValidationException('You must specify CPU, RAM and OS-SSD in your specs') try: parsed_ram = bitmath.parse_string_unsafe(_ram) parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) - if parsed_ram.base != ALLOWED_BASE: - self.add_error( - "Your specified RAM is not in correct units" - ) - if parsed_os_ssd.base != ALLOWED_BASE: - self.add_error( - "Your specified OS-SSD is not in correct units" - ) + if parsed_ram.base != allowed_base: + raise ValidationException('Your specified RAM is not in correct units') + + if parsed_os_ssd.base != allowed_base: + raise ValidationException('Your specified OS-SSD is not in correct units') if _cpu < 1: - self.add_error("CPU must be atleast 1") + raise ValidationException('CPU must be atleast 1') if parsed_ram < bitmath.GB(1): - self.add_error("RAM must be atleast 1 GB") + raise ValidationException('RAM must be atleast 1 GB') if parsed_os_ssd < bitmath.GB(10): - self.add_error("OS-SSD must be atleast 10 GB") + raise ValidationException('OS-SSD must be atleast 10 GB') parsed_hdd = [] for hdd in _hdd: _parsed_hdd = bitmath.parse_string_unsafe(hdd) - if _parsed_hdd.base != ALLOWED_BASE: - self.add_error( - "Your specified HDD is not in correct units" - ) - break + if _parsed_hdd.base != allowed_base: + raise ValidationException('Your specified HDD is not in correct units') else: parsed_hdd.append(str(_parsed_hdd)) except ValueError: - # TODO: Find some good error message - self.add_error("Specs are not correct.") + raise ValidationException('Specs are not correct.') else: - if self.get_errors(): - self.specs = { - "cpu": _cpu, - "ram": str(parsed_ram), - "os-ssd": str(parsed_os_ssd), - "hdd": parsed_hdd, - } + self.specs = { + 'cpu': _cpu, + 'ram': str(parsed_ram), + 'os-ssd': str(parsed_os_ssd), + 'hdd': parsed_hdd, + } def validation(self): - if self.realm.value != "ungleich-admin": - self.add_error( - "Invalid Credentials/Insufficient Permission" - ) - - -# VM Operations + if self.realm.value != 'ungleich-admin': + raise ValidationException('Invalid Credentials/Insufficient Permission') class CreateVMSchema(OTPSchema): def __init__(self, data): - # Fields - self.specs = Field("specs", dict, data.get("specs", KeyError)) - self.vm_name = Field( - "vm_name", str, data.get("vm_name", KeyError) - ) - self.image = Field("image", str, data.get("image", KeyError)) - self.network = Field( - "network", list, data.get("network", KeyError) - ) + self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation]) + self.vm_name = Field('vm_name', str, get(data, 'vm_name'), validators=[self.vm_name_validation]) + self.image = Field('image', str, get(data, 'image'), validators=[self.image_validation]) + self.network = Field('network', list, get(data, 'network', return_default=True, default=[]), + validators=[self.network_validation]) + self.image_uuid = None - # Validation - self.image.validation = self.image_validation - self.vm_name.validation = self.vm_name_validation - self.specs.validation = self.specs_validation - self.network.validation = self.network_validation - - fields = [self.vm_name, self.image, self.specs, self.network] - - super().__init__(data=data, fields=fields) + super().__init__(data=data) def image_validation(self): try: - image_uuid = helper.resolve_image_name( - self.image.value, shared.etcd_client - ) - except Exception as e: - logger.exception( - "Cannot resolve image name = %s", self.image.value - ) - self.add_error(str(e)) + image_uuid = helper.resolve_image_name(self.image.value) + except Exception: + raise ValidationException('No image of name \'{}\' found'.format(self.image.value)) else: self.image_uuid = image_uuid def vm_name_validation(self): - if resolve_vm_name( - name=self.vm_name.value, owner=self.name.value - ): - self.add_error( - 'VM with same name "{}" already exists'.format( - self.vm_name.value - ) - ) + if resolve_vm_name(name=self.vm_name.value, owner=self.name.value): + raise ValidationException("VM with same name '{}' already exists".format(self.vm_name.value)) def network_validation(self): _network = self.network.value if _network: for net in _network: - network = shared.etcd_client.get( - os.path.join( - shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], - self.name.value, - net, - ), - value_in_json=True, - ) - if not network: - self.add_error( - "Network with name {} does not exists".format( - net - ) + try: + shared.etcd_client.get( + os.path.join(settings['etcd']['network_prefix'], self.name.value, net), + value_in_json=True ) + except KeyError: + raise ValidationException('Network with name {} does not exists'.format(net)) def specs_validation(self): - ALLOWED_BASE = 10 + allowed_base = 10 - _cpu = self.specs.value.get("cpu", KeyError) - _ram = self.specs.value.get("ram", KeyError) - _os_ssd = self.specs.value.get("os-ssd", KeyError) - _hdd = self.specs.value.get("hdd", KeyError) - - if KeyError in [_cpu, _ram, _os_ssd, _hdd]: - self.add_error( - "You must specify CPU, RAM and OS-SSD in your specs" - ) - return None try: - parsed_ram = bitmath.parse_string_unsafe(_ram) - parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) - - if parsed_ram.base != ALLOWED_BASE: - self.add_error( - "Your specified RAM is not in correct units" - ) - if parsed_os_ssd.base != ALLOWED_BASE: - self.add_error( - "Your specified OS-SSD is not in correct units" - ) - - if int(_cpu) < 1: - self.add_error("CPU must be atleast 1") - - if parsed_ram < bitmath.GB(1): - self.add_error("RAM must be atleast 1 GB") - - if parsed_os_ssd < bitmath.GB(1): - self.add_error("OS-SSD must be atleast 1 GB") - - parsed_hdd = [] - for hdd in _hdd: - _parsed_hdd = bitmath.parse_string_unsafe(hdd) - if _parsed_hdd.base != ALLOWED_BASE: - self.add_error( - "Your specified HDD is not in correct units" - ) - break - else: - parsed_hdd.append(str(_parsed_hdd)) - - except ValueError: - # TODO: Find some good error message - self.add_error("Specs are not correct.") + _cpu = get(self.specs.value, 'cpu') + _ram = get(self.specs.value, 'ram') + _os_ssd = get(self.specs.value, 'os-ssd') + _hdd = get(self.specs.value, 'hdd', return_default=True, default=[]) + except (KeyError, Exception): + raise ValidationException('You must specify CPU, RAM and OS-SSD in your specs') else: - if self.get_errors(): + try: + parsed_ram = bitmath.parse_string_unsafe(_ram) + parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) + + if parsed_ram.base != allowed_base: + raise ValidationException('Your specified RAM is not in correct units') + + if parsed_os_ssd.base != allowed_base: + raise ValidationException('Your specified OS-SSD is not in correct units') + + if int(_cpu) < 1: + raise ValidationException('CPU must be atleast 1') + + if parsed_ram < bitmath.GB(1): + raise ValidationException('RAM must be atleast 1 GB') + + if parsed_os_ssd < bitmath.GB(1): + raise ValidationException('OS-SSD must be atleast 1 GB') + + parsed_hdd = [] + for hdd in _hdd: + _parsed_hdd = bitmath.parse_string_unsafe(hdd) + if _parsed_hdd.base != allowed_base: + raise ValidationException('Your specified HDD is not in correct units') + else: + parsed_hdd.append(str(_parsed_hdd)) + + except ValueError: + raise ValidationException('Specs are not correct.') + else: self.specs = { - "cpu": _cpu, - "ram": str(parsed_ram), - "os-ssd": str(parsed_os_ssd), - "hdd": parsed_hdd, + 'cpu': _cpu, + 'ram': str(parsed_ram), + 'os-ssd': str(parsed_os_ssd), + 'hdd': parsed_hdd, } class VMStatusSchema(OTPSchema): def __init__(self, data): - data["uuid"] = ( + data['uuid'] = ( resolve_vm_name( - name=data.get("vm_name", None), + name=get(data, 'vm_name', return_default=True), owner=( - data.get("in_support_of", None) - or data.get("name", None) - ), + get(data, 'in_support_of', return_default=True) or + get(data, 'name', return_default=True) + ) ) or KeyError ) self.uuid = VmUUIDField(data) - fields = [self.uuid] - - super().__init__(data, fields) + super().__init__(data) def validation(self): vm = shared.vm_pool.get(self.uuid.value) - if not ( - vm.value["owner"] == self.name.value - or self.realm.value == "ungleich-admin" - ): - self.add_error("Invalid User") + if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'): + raise ValidationException('Invalid User') class VmActionSchema(OTPSchema): def __init__(self, data): - data["uuid"] = ( + data['uuid'] = ( resolve_vm_name( - name=data.get("vm_name", None), + name=get(data, 'vm_name', return_default=True), owner=( - data.get("in_support_of", None) - or data.get("name", None) - ), + get(data, 'in_support_of', return_default=True) or + get(data, 'name', return_default=True) + ) ) or KeyError ) self.uuid = VmUUIDField(data) - self.action = Field("action", str, data.get("action", KeyError)) + self.action = Field('action', str, get(data, 'action'), validators=[self.action_validation]) - self.action.validation = self.action_validation - - _fields = [self.uuid, self.action] - - super().__init__(data=data, fields=_fields) + super().__init__(data=data) def action_validation(self): - allowed_actions = ["start", "stop", "delete"] + allowed_actions = ['start', 'stop', 'delete'] if self.action.value not in allowed_actions: - self.add_error( - "Invalid Action. Allowed Actions are {}".format( - allowed_actions - ) - ) + raise ValidationException('Invalid Action. Allowed Actions are {}'.format(allowed_actions)) def validation(self): vm = shared.vm_pool.get(self.uuid.value) - if not ( - vm.value["owner"] == self.name.value - or self.realm.value == "ungleich-admin" - ): - self.add_error("Invalid User") + if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'): + raise ValidationException('Invalid User.') - if ( - self.action.value == "start" - and vm.status == VMStatus.running - and vm.hostname != "" - ): - self.add_error("VM Already Running") + if self.action.value == 'start' and vm.status == VMStatus.running and vm.hostname != '': + raise ValidationException('VM Already Running') - if self.action.value == "stop": + if self.action.value == 'stop': if vm.status == VMStatus.stopped: - self.add_error("VM Already Stopped") + raise ValidationException('VM Already Stopped') elif vm.status != VMStatus.running: - self.add_error("Cannot stop non-running VM") + raise ValidationException('Cannot stop non-running VM') class VmMigrationSchema(OTPSchema): def __init__(self, data): - data["uuid"] = ( + data['uuid'] = ( resolve_vm_name( - name=data.get("vm_name", None), + name=get(data, 'vm_name', return_default=True), owner=( - data.get("in_support_of", None) - or data.get("name", None) - ), - ) - or KeyError + get(data, 'in_support_of', return_default=True) or + get(data, 'name', return_default=True) + ) + ) or KeyError ) self.uuid = VmUUIDField(data) - self.destination = Field( - "destination", str, data.get("destination", KeyError) - ) + self.destination = Field('destination', str, get(data, 'destination'), + validators=[self.destination_validation]) - self.destination.validation = self.destination_validation - - fields = [self.destination] - super().__init__(data=data, fields=fields) + super().__init__(data=data) def destination_validation(self): hostname = self.destination.value - host = next( - filter( - lambda h: h.hostname == hostname, shared.host_pool.hosts - ), - None, - ) + host = next(filter(lambda h: h.hostname == hostname, shared.host_pool.hosts), None,) if not host: - self.add_error( - "No Such Host ({}) exists".format( - self.destination.value - ) - ) + raise ValidationException('No Such Host ({}) exists'.format(self.destination.value)) elif host.status != HostStatus.alive: - self.add_error("Destination Host is dead") + raise ValidationException('Destination Host is dead') else: self.destination.value = host.key def validation(self): vm = shared.vm_pool.get(self.uuid.value) - if not ( - vm.value["owner"] == self.name.value - or self.realm.value == "ungleich-admin" - ): - self.add_error("Invalid User") + if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'): + raise ValidationException('Invalid User') if vm.status != VMStatus.running: - self.add_error("Can't migrate non-running VM") + raise ValidationException("Can't migrate non-running VM") - if vm.hostname == os.path.join( - shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value - ): - self.add_error( - "Destination host couldn't be same as Source Host" - ) + if vm.hostname == os.path.join(settings['etcd']['host_prefix'], self.destination.value): + raise ValidationException("Destination host couldn't be same as Source Host") class AddSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field( - "key_name", str, data.get("key_name", KeyError) - ) - self.key = Field("key", str, data.get("key_name", KeyError)) - - fields = [self.key_name, self.key] - super().__init__(data=data, fields=fields) + self.key_name = Field('key_name', str, get(data, 'key_name')) + self.key = Field('key', str, get(data, 'key')) + super().__init__(data=data) class RemoveSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field( - "key_name", str, data.get("key_name", KeyError) - ) - - fields = [self.key_name] - super().__init__(data=data, fields=fields) + self.key_name = Field('key_name', str, get(data, 'key_name')) + super().__init__(data=data) class GetSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field( - "key_name", str, data.get("key_name", None) - ) - - fields = [self.key_name] - super().__init__(data=data, fields=fields) + self.key_name = Field('key_name', str, get(data, 'key_name', return_default=True)) + super().__init__(data=data) class CreateNetwork(OTPSchema): def __init__(self, data): - self.network_name = Field("network_name", str, data.get("network_name", KeyError)) - self.type = Field("type", str, data.get("type", KeyError)) - self.user = Field("user", bool, bool(data.get("user", False))) - - self.network_name.validation = self.network_name_validation - self.type.validation = self.network_type_validation - - fields = [self.network_name, self.type, self.user] - super().__init__(data, fields=fields) + self.network_name = Field('network_name', str, get(data, 'name'), + validators=[self.network_name_validation]) + self.type = Field('type', str, get(data, 'type'), validators=[self.network_type_validation]) + self.user = Field('user', bool, bool(get(data, 'user', return_default=True, default=False))) + super().__init__(data) def network_name_validation(self): - key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) + key = os.path.join(settings['etcd']['network_prefix'], self.name.value, self.network_name.value) network = shared.etcd_client.get(key, value_in_json=True) if network: - self.add_error( - "Network with name {} already exists".format( - self.network_name.value - ) - ) + raise ValidationException('Network with name {} already exists'.format(self.network_name.value)) def network_type_validation(self): - supported_network_types = ["vxlan"] + supported_network_types = ['vxlan'] if self.type.value not in supported_network_types: - self.add_error( - "Unsupported Network Type. Supported network types are {}".format( - supported_network_types - ) - ) + raise ValidationException('Unsupported Network Type. Supported network types are {}'.format(supported_network_types)) diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py index 51a4355..0495fac 100644 --- a/uncloud/cli/helper.py +++ b/uncloud/cli/helper.py @@ -5,15 +5,15 @@ import binascii from pyotp import TOTP from os.path import join as join_path -from uncloud.common.shared import shared +from uncloud.common.settings import settings def get_otp_parser(): otp_parser = argparse.ArgumentParser('otp') - otp_parser.add_argument('--name') - otp_parser.add_argument('--realm') - otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') - + otp_parser.add_argument('--name', default=settings['client']['name']) + otp_parser.add_argument('--realm', default=settings['client']['realm']) + otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'], + dest='token', metavar='SEED') return otp_parser @@ -25,15 +25,11 @@ def load_dump_pretty(content): def make_request(*args, data=None, request_method=requests.post): + r = request_method(join_path(settings['client']['api_server'], *args), json=data) try: - r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) - except requests.exceptions.RequestException: - print('Error occurred while connecting to API server.') - else: - try: - print(load_dump_pretty(r.content)) - except Exception: - print('Error occurred while getting output from api server.') + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') def get_token(seed): diff --git a/uncloud/cli/main.py b/uncloud/cli/main.py index 9a42497..7f5e367 100644 --- a/uncloud/cli/main.py +++ b/uncloud/cli/main.py @@ -12,12 +12,12 @@ for component in ['user', 'host', 'image', 'network', 'vm']: subparser.add_parser(name=parser.prog, parents=[parser]) -def main(arguments): - if not arguments['subcommand']: +def main(**kwargs): + if not kwargs['subcommand']: arg_parser.print_help() else: - name = arguments.pop('subcommand') - arguments.pop('debug') + name = kwargs.pop('subcommand') + kwargs.pop('debug') mod = importlib.import_module('uncloud.cli.{}'.format(name)) _main = getattr(mod, 'main') - _main(**arguments) + _main(**kwargs) diff --git a/uncloud/common/cli.py b/uncloud/common/cli.py deleted file mode 100644 index 3d3c248..0000000 --- a/uncloud/common/cli.py +++ /dev/null @@ -1,26 +0,0 @@ -from uncloud.common.shared import shared -from pyotp import TOTP - - -def get_token(seed): - if seed is not None: - try: - token = TOTP(seed).now() - except Exception: - raise Exception('Invalid seed') - else: - return token - - -def resolve_otp_credentials(kwargs): - d = { - 'name': shared.settings['client']['name'], - 'realm': shared.settings['client']['realm'], - 'token': get_token(shared.settings['client']['seed']) - } - - for k, v in d.items(): - if k in kwargs and kwargs[k] is None: - kwargs.update({k: v}) - - return d diff --git a/uncloud/common/counters.py b/uncloud/common/counters.py index 2d4a8e9..dba8a08 100644 --- a/uncloud/common/counters.py +++ b/uncloud/common/counters.py @@ -1,8 +1,8 @@ -from .etcd_wrapper import Etcd3Wrapper +from uncloud.common.shared import shared -def increment_etcd_counter(etcd_client: Etcd3Wrapper, key): - kv = etcd_client.get(key) +def increment_etcd_counter(key): + kv = shared.etcd_client.get(key) if kv: counter = int(kv.value) @@ -10,12 +10,12 @@ def increment_etcd_counter(etcd_client: Etcd3Wrapper, key): else: counter = 1 - etcd_client.put(key, str(counter)) + shared.etcd_client.put(key, str(counter)) return counter -def get_etcd_counter(etcd_client: Etcd3Wrapper, key): - kv = etcd_client.get(key) +def get_etcd_counter(key): + kv = shared.etcd_client.get(key) if kv: return int(kv.value) return None diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index 38471ab..119d0e6 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -5,6 +5,7 @@ from functools import wraps from uncloud import UncloudException from uncloud.common import logger +from typing import Iterator class EtcdEntry: @@ -42,14 +43,30 @@ class Etcd3Wrapper: self.client = etcd3.client(*args, **kwargs) @readable_errors - def get(self, *args, value_in_json=False, **kwargs): + def get(self, *args, value_in_json=False, **kwargs) -> EtcdEntry: + """Get a key/value pair from etcd + + :return: + EtcdEntry: if a key/value pair is found in etcd + :raises: + KeyError: If key is not found in etcd + Exception: Different type of exception can be raised depending on + situation + """ _value, _key = self.client.get(*args, **kwargs) if _key is None or _value is None: - return None + raise KeyError return EtcdEntry(_key, _value, value_in_json=value_in_json) @readable_errors def put(self, *args, value_in_json=False, **kwargs): + """Put key/value pair in etcd + + :return: a response containing a header and the prev_kv + :raises: + Exception: Different type of exception can be raised depending on + situation + """ _key, _value = args if value_in_json: _value = json.dumps(_value) @@ -60,16 +77,28 @@ class Etcd3Wrapper: return self.client.put(_key, _value, **kwargs) @readable_errors - def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): + def get_prefix(self, *args, value_in_json=False, **kwargs) -> \ + Iterator[EtcdEntry]: event_iterator = self.client.get_prefix(*args, **kwargs) for e in event_iterator: yield EtcdEntry(*e[::-1], value_in_json=value_in_json) @readable_errors - def watch_prefix(self, key, raise_exception=True, value_in_json=False): - event_iterator, cancel = self.client.watch_prefix(key) - for e in event_iterator: - if hasattr(e, '_event'): - e = e._event - if e.type == e.PUT: - yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) + def watch_prefix(self, key, raise_exception=True, value_in_json=False) -> Iterator[EtcdEntry]: + try: + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = e._event + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) + except Exception as err: + if raise_exception: + raise Exception('Exception in etcd_wrapper.get_prefix') from err + else: + logger.exception('Error in etcd_wrapper.watch_prefix') + try: + cancel() + except Exception: + pass + return iter([]) diff --git a/uncloud/common/network.py b/uncloud/common/network.py index 32f6951..e74359d 100644 --- a/uncloud/common/network.py +++ b/uncloud/common/network.py @@ -1,6 +1,7 @@ import subprocess as sp import random import logging +import ipaddress logger = logging.getLogger(__name__) @@ -9,9 +10,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -68,3 +67,21 @@ def delete_network_interface(iface): except Exception: logger.exception("Interface %s Deletion failed", iface) + +def mac2ipv6(mac, prefix): + # only accept MACs separated by a colon + parts = mac.split(':') + + # modify parts to match IPv6 value + parts.insert(3, 'ff') + parts.insert(4, 'fe') + parts[0] = '%x' % (int(parts[0], 16) ^ 2) + + # format output + ipv6_parts = [str(0)] * 4 + for i in range(0, len(parts), 2): + ipv6_parts.append(''.join(parts[i: i + 2])) + + lower_part = ipaddress.IPv6Address(':'.join(ipv6_parts)) + prefix = ipaddress.IPv6Address(prefix) + return str(prefix + int(lower_part)) diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 8503f42..47ad5a7 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -8,7 +8,6 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper from os.path import join as join_path logger = logging.getLogger(__name__) -settings = None class CustomConfigParser(configparser.RawConfigParser): @@ -26,8 +25,9 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): - def __init__(self, conf_dir, seed_value=None): + def __init__(self): conf_name = 'uncloud.conf' + conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/')) self.config_file = join_path(conf_dir, conf_name) # this is used to cache config from etcd for 1 minutes. Without this we @@ -38,19 +38,15 @@ class Settings(object): self.config_parser.add_section('etcd') self.config_parser.set('etcd', 'base_prefix', '/') - if os.access(self.config_file, os.R_OK): + try: self.config_parser.read(self.config_file) - else: - raise FileNotFoundError('Config file %s not found!', self.config_file) + except Exception as err: + logger.error('%s', err) + self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') self.read_internal_values() - if seed_value is None: - seed_value = dict() - - self.config_parser.read_dict(seed_value) - def get_etcd_client(self): args = tuple() try: @@ -103,10 +99,12 @@ class Settings(object): def read_config_file_values(self, config_file): try: # Trying to read configuration file - with open(config_file) as config_file_handle: + with open(config_file, 'r') as config_file_handle: self.config_parser.read_file(config_file_handle) except FileNotFoundError: - sys.exit('Configuration file {} not found!'.format(config_file)) + sys.exit( + 'Configuration file {} not found!'.format(config_file) + ) except Exception as err: logger.exception(err) sys.exit('Error occurred while reading configuration file') @@ -132,5 +130,4 @@ class Settings(object): return self.config_parser[key] -def get_settings(): - return settings +settings = Settings() diff --git a/uncloud/common/shared.py b/uncloud/common/shared.py index aea7cbc..918dd0c 100644 --- a/uncloud/common/shared.py +++ b/uncloud/common/shared.py @@ -1,34 +1,34 @@ -from uncloud.common.settings import get_settings +from uncloud.common.settings import settings from uncloud.common.vm import VmPool from uncloud.common.host import HostPool from uncloud.common.request import RequestPool -import uncloud.common.storage_handlers as storage_handlers +from uncloud.common.storage_handlers import get_storage_handler class Shared: - @property - def settings(self): - return get_settings() - @property def etcd_client(self): - return self.settings.get_etcd_client() + return settings.get_etcd_client() @property def host_pool(self): - return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) + return HostPool( + self.etcd_client, settings["etcd"]["host_prefix"] + ) @property def vm_pool(self): - return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) + return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) @property def request_pool(self): - return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) + return RequestPool( + self.etcd_client, settings["etcd"]["request_prefix"] + ) @property def storage_handler(self): - return storage_handlers.get_storage_handler() + return get_storage_handler() shared = Shared() diff --git a/uncloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py index 58c2dc2..6f9b29e 100644 --- a/uncloud/common/storage_handlers.py +++ b/uncloud/common/storage_handlers.py @@ -6,7 +6,8 @@ import stat from abc import ABC from . import logger from os.path import join as join_path -import uncloud.common.shared as shared + +from uncloud.common.settings import settings as config class ImageStorageHandler(ABC): @@ -192,16 +193,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): def get_storage_handler(): - __storage_backend = shared.shared.settings["storage"]["storage_backend"] + __storage_backend = config["storage"]["storage_backend"] if __storage_backend == "filesystem": return FileSystemBasedImageStorageHandler( - vm_base=shared.shared.settings["storage"]["vm_dir"], - image_base=shared.shared.settings["storage"]["image_dir"], + vm_base=config["storage"]["vm_dir"], + image_base=config["storage"]["image_dir"], ) elif __storage_backend == "ceph": return CEPHBasedImageStorageHandler( - vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], - image_base=shared.shared.settings["storage"]["ceph_image_pool"], + vm_base=config["storage"]["ceph_vm_pool"], + image_base=config["storage"]["ceph_image_pool"], ) else: - raise Exception("Unknown Image Storage Handler") \ No newline at end of file + raise Exception("Unknown Image Storage Handler") diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index 87f5752..e190460 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,6 +1,7 @@ import os import argparse +from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('configure', add_help=False) @@ -39,19 +40,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True) def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) + uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) + shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) -def main(arguments): - subcommand = arguments['subcommand'] +def main(**kwargs): + subcommand = kwargs.pop('subcommand') if not subcommand: arg_parser.print_help() else: - update_config(subcommand, arguments) + update_config(subcommand, kwargs) diff --git a/docs/Makefile b/uncloud/docs/Makefile similarity index 93% rename from docs/Makefile rename to uncloud/docs/Makefile index 246b56c..5e7ea85 100644 --- a/docs/Makefile +++ b/uncloud/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source/ BUILDDIR = build/ -DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/ +DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/ .PHONY: all build clean diff --git a/docs/README.md b/uncloud/docs/README.md similarity index 100% rename from docs/README.md rename to uncloud/docs/README.md diff --git a/docs/__init__.py b/uncloud/docs/__init__.py similarity index 100% rename from docs/__init__.py rename to uncloud/docs/__init__.py diff --git a/docs/source/__init__.py b/uncloud/docs/source/__init__.py similarity index 100% rename from docs/source/__init__.py rename to uncloud/docs/source/__init__.py diff --git a/docs/source/admin-guide.rst b/uncloud/docs/source/admin-guide similarity index 72% rename from docs/source/admin-guide.rst rename to uncloud/docs/source/admin-guide index b62808d..ec6597d 100644 --- a/docs/source/admin-guide.rst +++ b/uncloud/docs/source/admin-guide @@ -56,13 +56,40 @@ To start host we created earlier, execute the following command ucloud host ungleich.ch -File & image scanners --------------------------- +Create OS Image +--------------- -Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our -uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by -ucloud. We can only make images from tracked files. So, we need to track the -file by running File Scanner +Create ucloud-init ready OS image (Optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +This step is optional if you just want to test ucloud. However, sooner or later +you want to create OS images with ucloud-init to properly +contexualize VMs. + +1. Start a VM with OS image on which you want to install ucloud-init +2. Execute the following command on the started VM + + .. code-block:: sh + + apk add git + git clone https://code.ungleich.ch/ucloud/ucloud-init.git + cd ucloud-init + sh ./install.sh +3. Congratulations. Your image is now ucloud-init ready. + + +Upload Sample OS Image +~~~~~~~~~~~~~~~~~~~~~~ +Execute the following to get the sample OS image file. + +.. code-block:: sh + + mkdir /var/www/admin + (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download) + +Run File Scanner and Image Scanner +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make +images from tracked files. So, we need to track the file by running File Scanner .. code-block:: sh diff --git a/docs/source/conf.py b/uncloud/docs/source/conf.py similarity index 100% rename from docs/source/conf.py rename to uncloud/docs/source/conf.py diff --git a/docs/source/diagram-code/ucloud b/uncloud/docs/source/diagram-code/ucloud similarity index 100% rename from docs/source/diagram-code/ucloud rename to uncloud/docs/source/diagram-code/ucloud diff --git a/uncloud/docs/source/hacking.rst b/uncloud/docs/source/hacking.rst new file mode 100644 index 0000000..2df42a7 --- /dev/null +++ b/uncloud/docs/source/hacking.rst @@ -0,0 +1,17 @@ +Hacking +======= +How to hack on the code. + +[ to be done by Balazs: + +* make nice +* indent with shell script mode + +] + +* git clone the repo +* cd to the repo +* Setup your venv: python -m venv venv +* . ./venv/bin/activate # you need the leading dot for sourcing! +* Run ./bin/ucloud-run-reinstall - it should print you an error + message on how to use ucloud diff --git a/docs/source/images/ucloud.svg b/uncloud/docs/source/images/ucloud.svg similarity index 100% rename from docs/source/images/ucloud.svg rename to uncloud/docs/source/images/ucloud.svg diff --git a/docs/source/index.rst b/uncloud/docs/source/index.rst similarity index 90% rename from docs/source/index.rst rename to uncloud/docs/source/index.rst index fad1f88..b31cff3 100644 --- a/docs/source/index.rst +++ b/uncloud/docs/source/index.rst @@ -11,13 +11,14 @@ Welcome to ucloud's documentation! :caption: Contents: introduction - setup-install - vm-images user-guide + setup-install admin-guide + user-guide/how-to-create-an-os-image-for-ucloud troubleshooting hacking + Indices and tables ================== diff --git a/docs/source/introduction.rst b/uncloud/docs/source/introduction.rst similarity index 100% rename from docs/source/introduction.rst rename to uncloud/docs/source/introduction.rst diff --git a/docs/source/misc/todo.rst b/uncloud/docs/source/misc/todo.rst similarity index 100% rename from docs/source/misc/todo.rst rename to uncloud/docs/source/misc/todo.rst diff --git a/docs/source/setup-install.rst b/uncloud/docs/source/setup-install.rst similarity index 100% rename from docs/source/setup-install.rst rename to uncloud/docs/source/setup-install.rst diff --git a/docs/source/theory/summary.rst b/uncloud/docs/source/theory/summary.rst similarity index 100% rename from docs/source/theory/summary.rst rename to uncloud/docs/source/theory/summary.rst diff --git a/docs/source/troubleshooting.rst b/uncloud/docs/source/troubleshooting.rst similarity index 100% rename from docs/source/troubleshooting.rst rename to uncloud/docs/source/troubleshooting.rst diff --git a/docs/source/user-guide.rst b/uncloud/docs/source/user-guide.rst similarity index 100% rename from docs/source/user-guide.rst rename to uncloud/docs/source/user-guide.rst diff --git a/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst b/uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst similarity index 100% rename from docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst rename to uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index 046f915..314481f 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -9,6 +9,7 @@ import bitmath from uuid import uuid4 from . import logger +from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('filescanner', add_help=False) @@ -52,7 +53,7 @@ def track_file(file, base_dir, host): file_path = file_path.relative_to(owner) creation_date = time.ctime(os.stat(file_str).st_ctime) - entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) + entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) entry_value = { 'filename': str(file_path), 'owner': owner, @@ -67,9 +68,8 @@ def track_file(file, base_dir, host): shared.etcd_client.put(entry_key, entry_value, value_in_json=True) -def main(arguments): - hostname = arguments['hostname'] - base_dir = shared.settings['storage']['file_dir'] +def main(hostname, debug=False): + base_dir = settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] @@ -77,7 +77,7 @@ def main(arguments): # Files that are already tracked tracked_files = [ pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) - for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) + for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) if f.value['host'] == hostname ] untracked_files = set(files) - set(tracked_files) diff --git a/uncloud/hack/__init__.py b/uncloud/hack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/uncloud/hack/config.py b/uncloud/hack/config.py deleted file mode 100644 index 7e2655d..0000000 --- a/uncloud/hack/config.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) -# -# This file is part of uncloud. -# -# uncloud is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# uncloud is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with uncloud. If not, see . -# -# - -class Config(object): - def __init__(self, arguments): - """ read arguments dicts as a base """ - - self.arguments = arguments - - # Split them so *etcd_args can be used and we can - # iterate over etcd_hosts - self.etcd_hosts = [ arguments['etcd_host'] ] - self.etcd_args = { - 'ca_cert': arguments['etcd_ca_cert'], - 'cert_cert': arguments['etcd_cert_cert'], - 'cert_key': arguments['etcd_cert_key'], -# 'user': None, -# 'password': None - } - self.etcd_prefix = '/nicohack/' diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py deleted file mode 100644 index cb5e490..0000000 --- a/uncloud/hack/db.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) -# -# This file is part of uncloud. -# -# uncloud is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# uncloud is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with uncloud. If not, see . -# -# - -import etcd3 -import json -import logging - -from functools import wraps -from uncloud import UncloudException - -log = logging.getLogger(__name__) - - -def readable_errors(func): - @wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except etcd3.exceptions.ConnectionFailedError as e: - raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) - except etcd3.exceptions.ConnectionTimeoutError as e: - raise UncloudException('etcd connection timeout. {}'.format(e)) - - return wrapper - - -class DB(object): - def __init__(self, config, prefix="/"): - self.config = config - - # Root for everything - self.base_prefix= '/nicohack' - - # Can be set from outside - self.prefix = prefix - - self.connect() - - @readable_errors - def connect(self): - self._db_clients = [] - for endpoint in self.config.etcd_hosts: - client = etcd3.client(host=endpoint, **self.config.etcd_args) - self._db_clients.append(client) - - def realkey(self, key): - return "{}{}/{}".format(self.base_prefix, - self.prefix, - key) - - @readable_errors - def get(self, key, as_json=False, **kwargs): - value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) - - if as_json: - value = json.loads(value) - - return value - - - @readable_errors - def set(self, key, value, as_json=False, **kwargs): - if as_json: - value = json.dumps(value) - - # FIXME: iterate over clients in case of failure ? - return self._db_clients[0].put(self.realkey(key), value, **kwargs) - - @readable_errors - def increment(self, key, **kwargs): - print(self.realkey(key)) - - - print("prelock") - lock = self._db_clients[0].lock('/nicohack/foo') - print("prelockacq") - lock.acquire() - print("prelockrelease") - lock.release() - - with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock: - print("in lock") - pass - -# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs)) -# self.set(self.realkey(key), str(value + 1), **kwargs) - - -if __name__ == '__main__': - endpoints = [ "https://etcd1.ungleich.ch:2379", - "https://etcd2.ungleich.ch:2379", - "https://etcd3.ungleich.ch:2379" ] - - db = DB(url=endpoints) diff --git a/uncloud/hack/hackcloud/.gitignore b/uncloud/hack/hackcloud/.gitignore deleted file mode 100644 index 0ad647b..0000000 --- a/uncloud/hack/hackcloud/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.iso -radvdpid -foo diff --git a/uncloud/hack/hackcloud/__init__.py b/uncloud/hack/hackcloud/__init__.py deleted file mode 100644 index 8b13789..0000000 --- a/uncloud/hack/hackcloud/__init__.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/uncloud/hack/hackcloud/etcd-client.sh b/uncloud/hack/hackcloud/etcd-client.sh deleted file mode 100644 index ab102a5..0000000 --- a/uncloud/hack/hackcloud/etcd-client.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \ - --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \ - --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \ - --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@" diff --git a/uncloud/hack/hackcloud/ifdown.sh b/uncloud/hack/hackcloud/ifdown.sh deleted file mode 100755 index 5753099..0000000 --- a/uncloud/hack/hackcloud/ifdown.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -echo $@ diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh deleted file mode 100755 index e0a3ca0..0000000 --- a/uncloud/hack/hackcloud/ifup.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -dev=$1; shift - -# bridge is setup from outside -ip link set dev "$dev" master ${bridge} -ip link set dev "$dev" up diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last deleted file mode 100644 index 8c5f254..0000000 --- a/uncloud/hack/hackcloud/mac-last +++ /dev/null @@ -1 +0,0 @@ -000000000252 diff --git a/uncloud/hack/hackcloud/mac-prefix b/uncloud/hack/hackcloud/mac-prefix deleted file mode 100644 index 5084a2f..0000000 --- a/uncloud/hack/hackcloud/mac-prefix +++ /dev/null @@ -1 +0,0 @@ -02:00 diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh deleted file mode 100755 index 4e2bfa1..0000000 --- a/uncloud/hack/hackcloud/net.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -set -x - -netid=100 -dev=wlp2s0 -dev=wlp0s20f3 -#dev=wlan0 - -ip=2a0a:e5c1:111:888::48/64 -vxlandev=vxlan${netid} -bridgedev=br${netid} - -ip -6 link add ${vxlandev} type vxlan \ - id ${netid} \ - dstport 4789 \ - group ff05::${netid} \ - dev ${dev} \ - ttl 5 - -ip link set ${vxlandev} up - - -ip link add ${bridgedev} type bridge -ip link set ${bridgedev} up - -ip link set ${vxlandev} master ${bridgedev} up - -ip addr add ${ip} dev ${bridgedev} diff --git a/uncloud/hack/hackcloud/nftrules b/uncloud/hack/hackcloud/nftrules deleted file mode 100644 index 636c63d..0000000 --- a/uncloud/hack/hackcloud/nftrules +++ /dev/null @@ -1,31 +0,0 @@ -flush ruleset - -table bridge filter { - chain prerouting { - type filter hook prerouting priority 0; - policy accept; - - ibrname br100 jump br100 - } - - chain br100 { - # Allow all incoming traffic from outside - iifname vxlan100 accept - - # Default blocks: router advertisements, dhcpv6, dhcpv4 - icmpv6 type nd-router-advert drop - ip6 version 6 udp sport 547 drop - ip version 4 udp sport 67 drop - - jump br100_vmlist - drop - } - chain br100_vmlist { - # VM1 - iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept - - # VM2 - iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept - iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept - } -} diff --git a/uncloud/hack/hackcloud/radvd.conf b/uncloud/hack/hackcloud/radvd.conf deleted file mode 100644 index 3d8ce4d..0000000 --- a/uncloud/hack/hackcloud/radvd.conf +++ /dev/null @@ -1,13 +0,0 @@ -interface br100 -{ - AdvSendAdvert on; - MinRtrAdvInterval 3; - MaxRtrAdvInterval 5; - AdvDefaultLifetime 3600; - - prefix 2a0a:e5c1:111:888::/64 { - }; - - RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; }; - DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ; -}; diff --git a/uncloud/hack/hackcloud/radvd.sh b/uncloud/hack/hackcloud/radvd.sh deleted file mode 100644 index 9d0e7d1..0000000 --- a/uncloud/hack/hackcloud/radvd.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -radvd -C ./radvd.conf -n -p ./radvdpid diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh deleted file mode 100755 index dd9be84..0000000 --- a/uncloud/hack/hackcloud/vm.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -# if [ $# -ne 1 ]; then -# echo "$0: owner" -# exit 1 -# fi - -qemu=/usr/bin/qemu-system-x86_64 - -accel=kvm -#accel=tcg - -memory=1024 -cores=2 -uuid=$(uuidgen) -mac=$(./mac-gen.py) -owner=nico - -export bridge=br100 - -set -x -$qemu -name "uncloud-${uuid}" \ - -machine pc,accel=${accel} \ - -m ${memory} \ - -smp ${cores} \ - -uuid ${uuid} \ - -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ - -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \ - -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac} diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py deleted file mode 100755 index 66286dd..0000000 --- a/uncloud/hack/mac.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2012 Nico Schottelius (nico-cinv at schottelius.org) -# -# This file is part of cinv. -# -# cinv is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cinv is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cinv. If not, see . -# -# - -import argparse -import logging -import os.path -import os -import re -import json - -from uncloud import UncloudException -from uncloud.hack.db import DB - -log = logging.getLogger(__name__) - - -class MAC(object): - def __init__(self, config): - self.config = config - self.no_db = self.config.arguments['no_db'] - if not self.no_db: - self.db = DB(config, prefix="/mac") - - self.prefix = 0x420000000000 - self._number = 0 # Not set by default - - @staticmethod - def validate_mac(mac): - if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): - raise Error("Not a valid mac address: %s" % mac) - - def last_used_index(self): - if not self.no_db: - value = self.db.get("last_used_index") - if not value: - self.db.set("last_used_index", "0") - value = self.db.get("last_used_index") - - else: - value = "0" - - return int(value) - - def last_used_mac(self): - return self.int_to_mac(self.prefix + self.last_used_index()) - - def to_colon_format(self): - b = self._number.to_bytes(6, byteorder="big") - return ':'.join(format(s, '02x') for s in b) - - def to_str_format(self): - b = self._number.to_bytes(6, byteorder="big") - return ''.join(format(s, '02x') for s in b) - - def create(self): - last_number = self.last_used_index() - - if last_number == int('0xffffffff', 16): - raise UncloudException("Exhausted all possible mac addresses - try to free some") - - next_number = last_number + 1 - self._number = self.prefix + next_number - - #next_number_string = "{:012x}".format(next_number) - #next_mac = self.int_to_mac(next_mac_number) - # db_entry = {} - # db_entry['vm_uuid'] = vmuuid - # db_entry['index'] = next_number - # db_entry['mac_address'] = next_mac - - # should be one transaction - # self.db.increment("last_used_index") - # self.db.set("used/{}".format(next_mac), - # db_entry, as_json=True) - - def __int__(self): - return self._number - - def __repr__(self): - return self.to_str_format() - - def __str__(self): - return self.to_colon_format() diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py deleted file mode 100644 index 9607ec2..0000000 --- a/uncloud/hack/main.py +++ /dev/null @@ -1,92 +0,0 @@ -import argparse -import logging - -from uncloud.hack.vm import VM -from uncloud.hack.config import Config -from uncloud.hack.mac import MAC -from uncloud.hack.net import VXLANBridge, DNSRA - -from uncloud import UncloudException - -arg_parser = argparse.ArgumentParser('hack', add_help=False) - #description="Commands that are unfinished - use at own risk") -arg_parser.add_argument('--last-used-mac', action='store_true') -arg_parser.add_argument('--get-new-mac', action='store_true') - -arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true') -arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true') -arg_parser.add_argument('--network', help="/64 IPv6 network") -arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0") -arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) -arg_parser.add_argument('--run-dns-ra', action='store_true', - help="Provide router advertisements and DNS resolution via dnsmasq") -arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') - -arg_parser.add_argument('--create-vm', action='store_true') -arg_parser.add_argument('--destroy-vm', action='store_true') -arg_parser.add_argument('--get-vm-status', action='store_true') -arg_parser.add_argument('--get-vm-vnc', action='store_true') -arg_parser.add_argument('--list-vms', action='store_true') -arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) -arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) -arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") -arg_parser.add_argument('--uuid', help="VM UUID") - -arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') -arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") - - -log = logging.getLogger(__name__) - - -def main(arguments): - config = Config(arguments) - - if arguments['create_vm']: - vm = VM(config) - vm.create() - - if arguments['destroy_vm']: - vm = VM(config) - vm.stop() - - if arguments['get_vm_status']: - vm = VM(config) - vm.status() - - if arguments['get_vm_vnc']: - vm = VM(config) - vm.vnc_addr() - - if arguments['list_vms']: - vm = VM(config) - vm.list() - - if arguments['last_used_mac']: - m = MAC(config) - print(m.last_used_mac()) - - if arguments['get_new_mac']: - print(MAC(config).get_next()) - - #if arguments['init_network']: - if arguments['create_vxlan']: - if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: - raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") - vb = VXLANBridge(vni=arguments['vni'], - route=arguments['network'], - uplinkdev=arguments['vxlan_uplink_device'], - use_sudo=arguments['use_sudo']) - vb._setup_vxlan() - vb._setup_bridge() - vb._add_vxlan_to_bridge() - vb._route_network() - - if arguments['run_dns_ra']: - if not arguments['network'] or not arguments['vni']: - raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") - - dnsra = DNSRA(route=arguments['network'], - vni=arguments['vni'], - use_sudo=arguments['use_sudo']) - dnsra._setup_dnsmasq() diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py deleted file mode 100644 index f28ab7f..0000000 --- a/uncloud/hack/net.py +++ /dev/null @@ -1,116 +0,0 @@ -import subprocess -import ipaddress -import logging - - -from uncloud import UncloudException - -log = logging.getLogger(__name__) - - -class VXLANBridge(object): - cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" - cmd_up_dev = "{sudo}ip link set {dev} up" - cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge" - cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up" - cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}" - cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}" - - # VXLAN ids are at maximum 24 bit - use a /104 - multicast_network = ipaddress.IPv6Network("ff05::/104") - max_vni = (2**24)-1 - - def __init__(self, - vni, - uplinkdev, - route=None, - use_sudo=False): - self.config = {} - - if vni > self.max_vni: - raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) - - if use_sudo: - self.config['sudo'] = 'sudo ' - else: - self.config['sudo'] = '' - - self.config['vni_dec'] = vni - self.config['vni_hex'] = "{:x}".format(vni) - self.config['multicast_address'] = self.multicast_network[vni] - - self.config['route_network'] = ipaddress.IPv6Network(route) - self.config['route'] = route - - self.config['uplinkdev'] = uplinkdev - self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) - self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) - - - def setup_networking(self): - pass - - def _setup_vxlan(self): - self._execute_cmd(self.cmd_create_vxlan) - self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev']) - - def _setup_bridge(self): - self._execute_cmd(self.cmd_create_bridge) - self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev']) - - def _route_network(self): - self._execute_cmd(self.cmd_add_route_dev) - - def _add_vxlan_to_bridge(self): - self._execute_cmd(self.cmd_add_to_bridge) - - def _execute_cmd(self, cmd_string, **kwargs): - cmd = cmd_string.format(**self.config, **kwargs) - log.info("Executing: {}".format(cmd)) - subprocess.run(cmd.split()) - -class ManagementBridge(VXLANBridge): - pass - - -class DNSRA(object): - # VXLAN ids are at maximum 24 bit - max_vni = (2**24)-1 - - - # Command to start dnsmasq - cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra" - - def __init__(self, - vni, - route=None, - use_sudo=False): - self.config = {} - - if vni > self.max_vni: - raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) - - if use_sudo: - self.config['sudo'] = 'sudo ' - else: - self.config['sudo'] = '' - - #TODO: remove if not needed - #self.config['vni_dec'] = vni - self.config['vni_hex'] = "{:x}".format(vni) - - # dnsmasq only wants the network without the prefix, therefore, cut it off - self.config['route'] = ipaddress.IPv6Network(route).network_address - self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) - - def _setup_dnsmasq(self): - self._execute_cmd(self.cmd_start_dnsmasq) - - def _execute_cmd(self, cmd_string, **kwargs): - cmd = cmd_string.format(**self.config, **kwargs) - log.info("Executing: {}".format(cmd)) - print("Executing: {}".format(cmd)) - subprocess.run(cmd.split()) - -class Firewall(object): - pass diff --git a/uncloud/hack/uncloud-hack-init-host b/uncloud/hack/uncloud-hack-init-host deleted file mode 100644 index 787ff80..0000000 --- a/uncloud/hack/uncloud-hack-init-host +++ /dev/null @@ -1,26 +0,0 @@ -id=100 -rawdev=eth0 - -# create vxlan -ip -6 link add vxlan${id} type vxlan \ - id ${id} \ - dstport 4789 \ - group ff05::${id} \ - dev ${rawdev} \ - ttl 5 - -ip link set vxlan${id} up - -# create bridge -ip link set vxlan${id} up -ip link set br${id} up - -# Add vxlan into bridge -ip link set vxlan${id} master br${id} - - -# useradd -m uncloud -# [18:05] tablett.place10:~# id uncloud -# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu) -# apk add qemu-system-x86_64 -# also needs group netdev diff --git a/uncloud/hack/uncloud-run-vm b/uncloud/hack/uncloud-run-vm deleted file mode 100644 index 33e5860..0000000 --- a/uncloud/hack/uncloud-run-vm +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -if [ $# -ne 1 ]; then - echo $0 vmid - exit 1 -fi - -id=$1; shift - -memory=512 -macaddress=02:00:b9:cb:70:${id} -netname=net${id}-1 - -qemu-system-x86_64 \ - -name uncloud-${id} \ - -accel kvm \ - -m ${memory} \ - -smp 2,sockets=2,cores=1,threads=1 \ - -device virtio-net-pci,netdev=net0,mac=$macaddress \ - -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \ - -vnc [::]:0 - -# To be changed: -# -vnc to unix path -# or -spice diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py deleted file mode 100755 index ac403d8..0000000 --- a/uncloud/hack/vm.py +++ /dev/null @@ -1,136 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) -# -# This file is part of uncloud. -# -# uncloud is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# uncloud is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with uncloud. If not, see . - -# This module is directly called from the hack module, and can be used as follow: -# -# Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix. -# `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2` -# -# List running VMs (returns a list of UUIDs). -# `uncloud hack --hackprefix /tmp/hackcloud --list-vms -# -# Get VM status: -# `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid` -# -# Stop a VM: -# `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` -# `` - -import subprocess -import uuid -import os -import logging - -from uncloud.hack.db import DB -from uncloud.hack.mac import MAC -from uncloud.vmm import VMM - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - -class VM(object): - def __init__(self, config): - self.config = config - - #TODO: Enable etcd lookup - self.no_db = self.config.arguments['no_db'] - if not self.no_db: - self.db = DB(self.config, prefix="/vm") - - # General CLI arguments. - self.hackprefix = self.config.arguments['hackprefix'] - self.uuid = self.config.arguments['uuid'] - self.memory = self.config.arguments['memory'] or '1024M' - self.cores = self.config.arguments['cores'] or 1 - if self.config.arguments['image']: - self.image = os.path.join(self.hackprefix, self.config.arguments['image']) - else: - self.image = None - - # External components. - self.vmm = VMM(vmm_backend=self.hackprefix) - self.mac = MAC(self.config) - - # Harcoded & generated values. - self.owner = 'uncoud' - self.image_format='qcow2' - self.accel = 'kvm' - self.threads = 1 - self.ifup = os.path.join(self.hackprefix, "ifup.sh") - self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") - self.ifname = "uc{}".format(self.mac.to_str_format()) - - def get_qemu_args(self): - command = ( - "-name {owner}-{name}" - " -machine pc,accel={accel}" - " -drive file={image},format={image_format},if=virtio" - " -device virtio-rng-pci" - " -m {memory} -smp cores={cores},threads={threads}" - " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}" - " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" - ).format( - owner=self.owner, name=self.uuid, - accel=self.accel, - image=self.image, image_format=self.image_format, - memory=self.memory, cores=self.cores, threads=self.threads, - ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname, - mac=self.mac - ) - - return command.split(" ") - - def create(self): - # New VM: new UUID, new MAC. - self.uuid = str(uuid.uuid4()) - self.mac.create() - - qemu_args = self.get_qemu_args() - log.debug("QEMU args passed to VMM: {}".format(qemu_args)) - self.vmm.start( - uuid=self.uuid, - migration=False, - *qemu_args - ) - - def stop(self): - if not self.uuid: - print("Please specific an UUID with the --uuid flag.") - exit(1) - - self.vmm.stop(self.uuid) - - def status(self): - if not self.uuid: - print("Please specific an UUID with the --uuid flag.") - exit(1) - - print(self.vmm.get_status(self.uuid)) - - def vnc_addr(self): - if not self.uuid: - print("Please specific an UUID with the --uuid flag.") - exit(1) - - print(self.vmm.get_vnc(self.uuid)) - - def list(self): - print(self.vmm.discover()) - diff --git a/uncloud/host/main.py b/uncloud/host/main.py index f680991..695e3d1 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -6,6 +6,7 @@ from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared +from uncloud.common.settings import settings from uncloud.common.vm import VMStatus from uncloud.vmm import VMM from os.path import join as join_path @@ -35,7 +36,7 @@ def maintenance(host): if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( - join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) + join_path(settings['etcd']['vm_prefix'], vm_uuid) ) vm.status = VMStatus.running vm.vnc_socket = vmm.get_vnc(vm_uuid) @@ -43,15 +44,14 @@ def maintenance(host): shared.vm_pool.put(vm) -def main(arguments): - hostname = arguments['hostname'] +def main(hostname, debug=False): host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) # Does not yet exist, create it if not host: host_key = join_path( - shared.settings['etcd']['host_prefix'], uuid4().hex + settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': '', @@ -79,9 +79,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for events_iterator in [ - shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False) ]: for request_event in events_iterator: @@ -94,7 +94,7 @@ def main(arguments): shared.request_pool.client.client.delete(request_event.key) vm_entry = shared.etcd_client.get( - join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) + join_path(settings['etcd']['vm_prefix'], request_event.uuid) ) logger.debug('VM hostname: {}'.format(vm_entry.value)) diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index a592efc..2f6a5e3 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -17,6 +17,7 @@ from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger from uncloud.common.shared import shared +from uncloud.common.settings import settings from uncloud.vmm import VMM from marshmallow import ValidationError @@ -90,7 +91,7 @@ class VM: self.vmm.socket_dir, self.uuid ), destination_host_key=destination_host_key, # Where source host transfer VM - request_prefix=shared.settings["etcd"]["request_prefix"], + request_prefix=settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) else: @@ -118,7 +119,7 @@ class VM: network_name, mac, tap = network_mac_and_tap _key = os.path.join( - shared.settings["etcd"]["network_prefix"], + settings["etcd"]["network_prefix"], self.vm["owner"], network_name, ) @@ -132,13 +133,13 @@ class VM: if network["type"] == "vxlan": tap = create_vxlan_br_tap( _id=network["id"], - _dev=shared.settings["network"]["vxlan_phy_dev"], + _dev=settings["network"]["vxlan_phy_dev"], tap_id=tap, ip=network["ipv6"], ) all_networks = shared.etcd_client.get_prefix( - shared.settings["etcd"]["network_prefix"], + settings["etcd"]["network_prefix"], value_in_json=True, ) @@ -228,7 +229,7 @@ class VM: def resolve_network(network_name, network_owner): network = shared.etcd_client.get( join_path( - shared.settings["etcd"]["network_prefix"], + settings["etcd"]["network_prefix"], network_owner, network_name, ), diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index ee9da2e..91f100e 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -4,6 +4,7 @@ import argparse import subprocess as sp from os.path import join as join_path +from uncloud.common.settings import settings from uncloud.common.shared import shared from uncloud.imagescanner import logger @@ -29,10 +30,10 @@ def qemu_img_type(path): return qemu_img_info["format"] -def main(arguments): +def main(debug=False): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( - shared.settings["etcd"]["image_prefix"], value_in_json=True + settings["etcd"]["image_prefix"], value_in_json=True ) images_to_be_created = list( filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) @@ -45,13 +46,13 @@ def main(arguments): image_filename = image.value["filename"] image_store_name = image.value["store_name"] image_full_path = join_path( - shared.settings["storage"]["file_dir"], + settings["storage"]["file_dir"], image_owner, image_filename, ) image_stores = shared.etcd_client.get_prefix( - shared.settings["etcd"]["image_store_prefix"], + settings["etcd"]["image_store_prefix"], value_in_json=True, ) user_image_store = next( diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index 374260e..73d59cd 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -5,6 +5,7 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException +from uncloud.common.settings import settings from uncloud.common.shared import shared app = Flask(__name__) @@ -12,10 +13,8 @@ api = Api(app) app.logger.handlers.clear() -DEFAULT_PORT=1234 - arg_parser = argparse.ArgumentParser('metadata', add_help=False) -arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT)) +arg_parser.add_argument('--port', '-p', default=80, help='By default bind to port 80') @app.errorhandler(Exception) @@ -73,7 +72,7 @@ class Root(Resource): ) else: etcd_key = os.path.join( - shared.settings["etcd"]["user_prefix"], + settings["etcd"]["user_prefix"], data.value["owner_realm"], data.value["owner"], "key", @@ -89,7 +88,9 @@ class Root(Resource): api.add_resource(Root, "/") -def main(arguments): - port = arguments['port'] - debug = arguments['debug'] +def main(port=None, debug=False): app.run(debug=debug, host="::", port=port) + + +if __name__ == "__main__": + main() diff --git a/uncloud/oneshot/__init__.py b/uncloud/oneshot/__init__.py deleted file mode 100644 index eea436a..0000000 --- a/uncloud/oneshot/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import logging - -logger = logging.getLogger(__name__) diff --git a/uncloud/oneshot/main.py b/uncloud/oneshot/main.py deleted file mode 100644 index 5b9b61c..0000000 --- a/uncloud/oneshot/main.py +++ /dev/null @@ -1,123 +0,0 @@ -import argparse -import os - - -from pathlib import Path -from uncloud.vmm import VMM -from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap - -from . import virtualmachine, logger - -### -# Argument parser loaded by scripts/uncloud. -arg_parser = argparse.ArgumentParser('oneshot', add_help=False) - -# Actions. -arg_parser.add_argument('--list', action='store_true', - help='list UUID and name of running VMs') -arg_parser.add_argument('--start', nargs=3, - metavar=('IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'), - help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix') -arg_parser.add_argument('--stop', metavar='UUID', - help='stop a VM') -arg_parser.add_argument('--get-status', metavar='UUID', - help='return the status of the VM') -arg_parser.add_argument('--get-vnc', metavar='UUID', - help='return the path of the VNC socket of the VM') -arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK', - help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix') - -# Arguments. -arg_parser.add_argument('--workdir', default=Path.home(), - help='Working directory, defaulting to $HOME') -arg_parser.add_argument('--mac', - help='MAC address of the VM to create (--start)') -arg_parser.add_argument('--memory', type=int, - help='Memory (MB) to allocate (--start)') -arg_parser.add_argument('--cores', type=int, - help='Number of cores to allocate (--start)') -arg_parser.add_argument('--threads', type=int, - help='Number of threads to allocate (--start)') -arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], - help='Format of OS image (--start)') -arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm', - help='QEMU acceleration to use (--start)') -arg_parser.add_argument('--upstream-interface', default='eth0', - help='Name of upstream interface (--start)') - -### -# Helpers. - -# XXX: check if it is possible to use the type returned by ETCD queries. -class UncloudEntryWrapper: - def __init__(self, value): - self.value = value - - def value(self): - return self.value - -def status_line(vm): - return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status()) - -### -# Entrypoint. - -def main(arguments): - # Initialize VMM. - workdir = arguments['workdir'] - vmm = VMM(vmm_backend=workdir) - - # Harcoded debug values. - net_id = 0 - - # Build VM configuration. - vm_config = {} - vm_options = [ - 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', - '--upstream_interface', 'upstream_interface', 'network', 'accel' - ] - for option in vm_options: - if arguments.get(option): - vm_config[option] = arguments[option] - - vm_config['net_id'] = net_id - - # Execute requested VM action. - if arguments['reconfigure_radvd']: - # TODO: check that RADVD is available. - prefix = arguments['reconfigure_radvd'] - network = UncloudEntryWrapper({ - 'id': net_id, - 'ipv6': prefix - }) - - # Make use of uncloud.host.virtualmachine for network configuration. - update_radvd_conf([network]) - elif arguments['start']: - # Extract from --start positional arguments. Quite fragile. - vm_config['image'] = arguments['start'][0] - vm_config['network'] = arguments['start'][1] - vm_config['upstream_interface'] = arguments['start'][2] - - vm_config['tap_interface'] = "uc{}".format(len(vmm.discover())) - vm = virtualmachine.VM(vmm, vm_config) - vm.start() - elif arguments['stop']: - vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']}) - vm = virtualmachine.VM(vmm, vm_config) - vm.stop() - elif arguments['get_status']: - vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']}) - print(status_line(vm)) - elif arguments['get_vnc']: - vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']}) - print(vm.get_vnc_addr()) - elif arguments['list']: - vms = vmm.discover() - print("Found {} VMs.".format(len(vms))) - for uuid in vms: - vm = virtualmachine.VM(vmm, {'uuid': uuid}) - print(status_line(vm)) - else: - print('Please specify an action: --start, --stop, --list,\ ---get-status, --get-vnc, --reconfigure-radvd') diff --git a/uncloud/oneshot/virtualmachine.py b/uncloud/oneshot/virtualmachine.py deleted file mode 100644 index c8c2909..0000000 --- a/uncloud/oneshot/virtualmachine.py +++ /dev/null @@ -1,81 +0,0 @@ -import uuid -import os - -from uncloud.host.virtualmachine import create_vxlan_br_tap -from uncloud.oneshot import logger - -class VM(object): - def __init__(self, vmm, config): - self.config = config - self.vmm = vmm - - # Extract VM specs/metadata from configuration. - self.name = config.get('name', 'no-name') - self.memory = config.get('memory', 1024) - self.cores = config.get('cores', 1) - self.threads = config.get('threads', 1) - self.image_format = config.get('image_format', 'qcow2') - self.image = config.get('image') - self.uuid = config.get('uuid', str(uuid.uuid4())) - self.mac = config.get('mac') - self.accel = config.get('accel', 'kvm') - - self.net_id = config.get('net_id', 0) - self.upstream_interface = config.get('upstream_interface', 'eth0') - self.tap_interface = config.get('tap_interface', 'uc0') - self.network = config.get('network') - - def get_qemu_args(self): - command = ( - "-uuid {uuid} -name {name} -machine pc,accel={accel}" - " -drive file={image},format={image_format},if=virtio" - " -device virtio-rng-pci" - " -m {memory} -smp cores={cores},threads={threads}" - " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" - " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" - ).format( - uuid=self.uuid, name=self.name, accel=self.accel, - image=self.image, image_format=self.image_format, - memory=self.memory, cores=self.cores, threads=self.threads, - net_id=self.net_id, tap=self.tap_interface, mac=self.mac - ) - - return command.split(" ") - - def start(self): - # Check that VM image is available. - if not os.path.isfile(self.image): - logger.error("Image {} does not exist. Aborting.".format(self.image)) - - # Create Bridge, VXLAN and tap interface for VM. - create_vxlan_br_tap( - self.net_id, self.upstream_interface, self.tap_interface, self.network - ) - - # Generate config for and run QEMU. - qemu_args = self.get_qemu_args() - logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) - self.vmm.start( - uuid=self.uuid, - migration=False, - *qemu_args - ) - - def stop(self): - self.vmm.stop(self.uuid) - - def get_status(self): - return self.vmm.get_status(self.uuid) - - def get_vnc_addr(self): - return self.vmm.get_vnc(self.uuid) - - def get_uuid(self): - return self.uuid - - def get_name(self): - success, json = self.vmm.execute_command(uuid, 'query-name') - if success: - return json['return']['name'] - - return None diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index 79db322..108d126 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -7,6 +7,7 @@ from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus from uncloud.common.shared import shared +from uncloud.common.settings import settings def accumulated_specs(vms_specs): @@ -129,7 +130,7 @@ def assign_host(vm): type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=shared.settings["etcd"]["request_prefix"], + request_prefix=settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 38c07bf..20a52cb 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -6,6 +6,7 @@ import argparse +from uncloud.common.settings import settings from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared from uncloud.scheduler import logger @@ -15,7 +16,7 @@ from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection, arg_parser = argparse.ArgumentParser('scheduler', add_help=False) -def main(arguments): +def main(debug=False): # The below while True is neccessary for gracefully handling leadership transfer and temporary # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) @@ -23,9 +24,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for request_iterator in [ - shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), ]: for request_event in request_iterator: @@ -49,3 +50,7 @@ def main(arguments): shared.vm_pool.put(vm_entry) logger.info('No Resource Left. Emailing admin....') + + +if __name__ == '__main__': + main() diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 6db61eb..4c893f6 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -100,9 +100,9 @@ class TransferVM(Process): class VMM: # Virtual Machine Manager def __init__( - self, - qemu_path="/usr/bin/qemu-system-x86_64", - vmm_backend=os.path.expanduser("~/uncloud/vmm/"), + self, + qemu_path="/usr/bin/qemu-system-x86_64", + vmm_backend=os.path.expanduser("~/uncloud/vmm/"), ): self.qemu_path = qemu_path self.vmm_backend = vmm_backend @@ -125,7 +125,7 @@ class VMM: os.makedirs(self.socket_dir, exist_ok=True) def is_running(self, uuid): - sock_path = os.path.join(self.socket_dir, uuid) + sock_path = os.path.join(self.vmm_backend, uuid) try: sock = socket.socket(socket.AF_UNIX) sock.connect(sock_path) @@ -163,7 +163,7 @@ class VMM: qmp_arg = ( "-qmp", "unix:{},server,nowait".format( - join_path(self.socket_dir, uuid) + join_path(self.vmm_backend, uuid) ), ) vnc_arg = ( @@ -212,7 +212,7 @@ class VMM: def execute_command(self, uuid, command, **kwargs): # execute_command -> sucess?, output try: - with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as ( + with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as ( sock_handle, file_handle, ): @@ -255,8 +255,8 @@ class VMM: def discover(self): vms = [ uuid - for uuid in os.listdir(self.socket_dir) - if not isdir(join_path(self.socket_dir, uuid)) + for uuid in os.listdir(self.vmm_backend) + if not isdir(join_path(self.vmm_backend, uuid)) ] return vms