From 6d715e83483a09decb05c2420ca8cf85815e4386 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 7 Dec 2019 13:51:50 +0100 Subject: [PATCH 001/163] [config] setup default values to remove startup failures --- ucloud/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ucloud/config.py b/ucloud/config.py index 7c141a3..a2f523f 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -25,13 +25,13 @@ etcd_wrapper_kwargs = { etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) -host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX')) -vm_pool = VmPool(etcd_client, env_vars.get('VM_PREFIX')) -request_pool = RequestPool(etcd_client, env_vars.get('REQUEST_PREFIX')) +host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX', "hosts")) +vm_pool = VmPool(etcd_client, env_vars.get('VM_PREFIX', "vms")) +request_pool = RequestPool(etcd_client, env_vars.get('REQUEST_PREFIX', "requests")) running_vms = [] -__storage_backend = env_vars.get("STORAGE_BACKEND") +__storage_backend = env_vars.get("STORAGE_BACKEND", "filesystem") if __storage_backend == "filesystem": image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=env_vars.get("VM_DIR"), image_base=env_vars.get("IMAGE_DIR")) From 9ae75f20e8280efbc6cd0a328e2590b82d7b2020 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 7 Dec 2019 14:01:44 +0100 Subject: [PATCH 002/163] Generate version from git Fixes #3 --- bin/gen-version | 29 +++++++++++++++++++++++++++++ bin/ucloud | 2 +- setup.py | 11 ++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100755 bin/gen-version diff --git a/bin/gen-version b/bin/gen-version new file mode 100755 index 0000000..8f622b8 --- /dev/null +++ b/bin/gen-version @@ -0,0 +1,29 @@ +#!/bin/sh +# -*- coding: utf-8 -*- +# +# 2019 Nico Schottelius (nico-ucloud at schottelius.org) +# +# This file is part of ucloud. +# +# ucloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ucloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ucloud. If not, see . +# +# + + +# Wrapper for real script to allow execution from checkout +dir=${0%/*} + +# Ensure version is present - the bundled/shipped version contains a static version, +# the git version contains a dynamic version +printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../ucloud/version.py diff --git a/bin/ucloud b/bin/ucloud index e178413..ba337fd 100755 --- a/bin/ucloud +++ b/bin/ucloud @@ -25,7 +25,7 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../ucloud/version.py +${dir}/gen-version libdir=$(cd "${dir}/../" && pwd -P) export PYTHONPATH="${libdir}" diff --git a/setup.py b/setup.py index 9a35f27..14dffb7 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,17 @@ from setuptools import setup, find_packages with open("README.md", "r") as fh: long_description = fh.read() +try: + import ucloud.version + version = ucloud.version.VERSION +except: + import subprocess + c = subprocess.run(["git", "describe"], capture_output=True) + version = c.stdout.decode("utf-8") + + setup(name='ucloud', - version='0.0.1', + version=version, description='All ucloud server components.', url='https://code.ungleich.ch/ucloud/ucloud', long_description=long_description, From 2244b94fd80b089605aa6d3fea634ebaa2740c6f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 7 Dec 2019 14:10:16 +0100 Subject: [PATCH 003/163] Fix another UndefinedValueError: VM_DIR decouple.UndefinedValueError: VM_DIR not found. Declare it as envvar or define a default value. --- ucloud/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ucloud/config.py b/ucloud/config.py index a2f523f..a5b8f00 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -33,8 +33,8 @@ running_vms = [] __storage_backend = env_vars.get("STORAGE_BACKEND", "filesystem") if __storage_backend == "filesystem": - image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=env_vars.get("VM_DIR"), - image_base=env_vars.get("IMAGE_DIR")) + image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=env_vars.get("VM_DIR", "/tmp/ucloud-vms"), + image_base=env_vars.get("IMAGE_DIR", "/tmp/ucloud-images")) elif __storage_backend == "ceph": image_storage_handler = CEPHBasedImageStorageHandler(vm_base="ssd", image_base="ssd") else: From f9dbdc730a1ab16813de47b0c48b324bdea5960e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 7 Dec 2019 14:15:48 +0100 Subject: [PATCH 004/163] Remove logging configuration Leave it to the OS/env to set this up. Fixes #6 --- scripts/ucloud | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/ucloud b/scripts/ucloud index 7be6b24..d700d79 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -17,13 +17,6 @@ if __name__ == "__main__": arg_parser.add_argument('component_args', nargs='*') args = arg_parser.parse_args() - logging.basicConfig( - level=logging.DEBUG, - filename=join_path("/", "etc", "ucloud", "log.txt"), - filemode="a", - format="%(name)s %(asctime)s: %(levelname)s - %(message)s", - datefmt="%d-%b-%y %H:%M:%S", - ) try: check() @@ -56,4 +49,4 @@ if __name__ == "__main__": except Exception as e: logging.exception(e) - print(e) \ No newline at end of file + print(e) From 9517e73233415c6c76438c024db8ea718166455f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 7 Dec 2019 14:25:21 +0100 Subject: [PATCH 005/163] Migrate sanity_check.py into the respective daemons --- scripts/ucloud | 2 -- ucloud/host/main.py | 9 +++++++++ ucloud/imagescanner/main.py | 13 +++++++++++++ ucloud/sanity_checks.py | 33 --------------------------------- 4 files changed, 22 insertions(+), 35 deletions(-) delete mode 100644 ucloud/sanity_checks.py diff --git a/scripts/ucloud b/scripts/ucloud index d700d79..1bb752b 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -18,8 +18,6 @@ if __name__ == "__main__": args = arg_parser.parse_args() try: - check() - if args.component == 'api': from ucloud.api.main import main diff --git a/ucloud/host/main.py b/ucloud/host/main.py index ccf0a8d..ca68351 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -71,8 +71,17 @@ def maintenance(host): if _vm: running_vms.remove(_vm) +def check(): + if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('VM_DIR')): + print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" + " in .env file must exists. But, it don't.") + sys.exit(1) + + def main(hostname): + check() + heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX')) diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index 20ce9d5..6ff01f8 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -18,6 +18,19 @@ def qemu_img_type(path): qemu_img_info = json.loads(qemu_img_info.decode("utf-8")) return qemu_img_info["format"] +def check(): + """ check whether settings are sane, refuse to start if they aren't """ + if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('IMAGE_DIR')): + print("You have set STORAGE_BACKEND to filesystem, but " + "{} does not exist. Refusing to start".format(env_vars.get('IMAGE_DIR'))) + sys.exit(1) + + try: + subprocess.check_output(['which', 'qemu-img']) + except Exception: + print("qemu-img missing") + sys.exit(1) + def main(): # We want to get images entries that requests images to be created diff --git a/ucloud/sanity_checks.py b/ucloud/sanity_checks.py deleted file mode 100644 index 143f767..0000000 --- a/ucloud/sanity_checks.py +++ /dev/null @@ -1,33 +0,0 @@ -import sys -import subprocess as sp - -from os.path import isdir -from ucloud.config import env_vars - - -def check(): - ######################### - # ucloud-image-scanner # - ######################### - if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('IMAGE_DIR')): - print("You have set STORAGE_BACKEND to filesystem. So," - "the {} must exists. But, it don't".format(env_vars.get('IMAGE_DIR'))) - sys.exit(1) - - try: - sp.check_output(['which', 'qemu-img']) - except Exception: - print("qemu-img missing") - sys.exit(1) - - ############### - # ucloud-host # - ############### - - if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('VM_DIR')): - print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" - " in .env file must exists. But, it don't.") - sys.exit(1) - -if __name__ == "__main__": - check() \ No newline at end of file From cfb09c29de86f3cf7f9a5d442092d54c7aa9797e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 12:28:25 +0100 Subject: [PATCH 006/163] simplify main script --- scripts/ucloud | 47 ++++++++++++++++------------------------------- 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/scripts/ucloud b/scripts/ucloud index 1bb752b..277bf2e 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -1,49 +1,34 @@ #!/usr/bin/env python3 import argparse -import multiprocessing as mp import logging +import importlib -from os.path import join as join_path -from ucloud.sanity_checks import check +# For the exception +import decouple +import sys + +COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata'] if __name__ == "__main__": + log = logging.getLogger("ucloud") + arg_parser = argparse.ArgumentParser(prog='ucloud', description='Open Source Cloud Management Software') - arg_parser.add_argument('component', - choices=['api', 'scheduler', 'host', - 'filescanner', 'imagescanner', - 'metadata']) + arg_parser.add_argument('component', choices=COMMANDS) arg_parser.add_argument('component_args', nargs='*') args = arg_parser.parse_args() try: - if args.component == 'api': - from ucloud.api.main import main + name = args.component - main() - elif args.component == 'host': - from ucloud.host.main import main + mod = importlib.import_module("ucloud.{}.main".format(name)) + main = getattr(mod, "main") + main() - hostname = args.component_args - mp.set_start_method('spawn') - main(*hostname) - elif args.component == 'scheduler': - from ucloud.scheduler.main import main - - main() - elif args.component == 'filescanner': - from ucloud.filescanner.main import main - - main() - elif args.component == 'imagescanner': - from ucloud.imagescanner.main import main - - main() - elif args.component == 'metadata': - from ucloud.metadata.main import main - - main() + except decouple.UndefinedValueError as e: + print(e) + sys.exit(1) except Exception as e: logging.exception(e) From e459434b9177105a223409c3cd60ec37f1cd594a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 12:58:26 +0100 Subject: [PATCH 007/163] add sample ucloud.conf --- conf/ucloud.conf | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 conf/ucloud.conf diff --git a/conf/ucloud.conf b/conf/ucloud.conf new file mode 100644 index 0000000..7f83d80 --- /dev/null +++ b/conf/ucloud.conf @@ -0,0 +1,12 @@ +# This section contains default values for all other sections +[DEFAULT] + +NETWORK_PREFIX = moo + +[api] +NETWORK_PREFIX = foo + +[woo] +NETWORK_PREFIX = foo + +[noval] \ No newline at end of file From 6d0ce65f5c3124d0f038b767df70d08c37540fec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 12:59:18 +0100 Subject: [PATCH 008/163] begin to switch to configparser To not have unwanted environment influence --- scripts/ucloud | 17 ++++++++++------- ucloud/config.py | 40 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 11 deletions(-) diff --git a/scripts/ucloud b/scripts/ucloud index 277bf2e..b8ef32d 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -3,10 +3,8 @@ import argparse import logging import importlib - -# For the exception -import decouple import sys +import os COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata'] @@ -15,20 +13,25 @@ if __name__ == "__main__": arg_parser = argparse.ArgumentParser(prog='ucloud', description='Open Source Cloud Management Software') + arg_parser.add_argument('-c', '--conf-dir', help="Configuration directory") arg_parser.add_argument('component', choices=COMMANDS) arg_parser.add_argument('component_args', nargs='*') args = arg_parser.parse_args() try: name = args.component - mod = importlib.import_module("ucloud.{}.main".format(name)) main = getattr(mod, "main") + + if args.conf_dir: + print("setting conf") + os.environ['UCLOUD_CONF_DIR'] = args.conf_dir + main() - except decouple.UndefinedValueError as e: - print(e) - sys.exit(1) + # except decouple.UndefinedValueError as e: + # print(e) + # sys.exit(1) except Exception as e: logging.exception(e) diff --git a/ucloud/config.py b/ucloud/config.py index a5b8f00..b508f56 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -4,14 +4,46 @@ from ucloud.common.host import HostPool from ucloud.common.request import RequestPool from ucloud.common.vm import VmPool from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler + from decouple import Config, RepositoryEnv, RepositoryEmpty +# Replacing decouple inline +import configparser +import os +import os.path + +import logging + +log = logging.getLogger("ucloud.config") + + +conf_name = "ucloud.conf" + +try: + conf_dir = os.environ["UCLOUD_CONF_DIR"] +except KeyError: + conf_dir = "/etc/ucloud" + +config_file = os.path.join(conf_dir, conf_name) + +config = configparser.ConfigParser() + +try: + with open(config_file, "r") as conf_fd: + conf.read(conf_fd) +except FileNotFoundError: + log.warn("Configuration file not found - using defaults") + + +# Compatibility to old code +env_vars = config + # Try importing config, but don't fail if it does not exist -try: - env_vars = Config(RepositoryEnv('/etc/ucloud/ucloud.conf')) -except FileNotFoundError: - env_vars = Config(RepositoryEmpty()) +# try: +# env_vars = Config(RepositoryEnv('/etc/ucloud/ucloud.conf')) +# except FileNotFoundError: +# env_vars = Config(RepositoryEmpty()) etcd_wrapper_args = () From a4bedb01f60346b9abde13552bfdaef5fbbc4638 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:00:42 +0100 Subject: [PATCH 009/163] [api] begin to move to configparser --- ucloud/api/create_image_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ucloud/api/create_image_store.py b/ucloud/api/create_image_store.py index 17fa63c..259b9c8 100755 --- a/ucloud/api/create_image_store.py +++ b/ucloud/api/create_image_store.py @@ -3,7 +3,7 @@ import os from uuid import uuid4 -from ucloud.config import etcd_client, env_vars +from ucloud.config import etcd_client, config data = { "is_public": True, @@ -13,4 +13,4 @@ data = { "attributes": {"list": [], "key": [], "pool": "images"}, } -etcd_client.put(os.path.join(env_vars.get('IMAGE_STORE_PREFIX'), uuid4().hex), json.dumps(data)) +etcd_client.put(os.path.join(config['api']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data)) From cdbfb96e712f51970729bdd999aeb531ba072336 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:09:52 +0100 Subject: [PATCH 010/163] [api] config updates and add default values --- conf/ucloud.conf | 6 ++++++ ucloud/api/common_fields.py | 5 ++--- ucloud/api/helper.py | 26 +++++++++++++------------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 7f83d80..0ebb705 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -1,8 +1,14 @@ # This section contains default values for all other sections [DEFAULT] +AUTH_NAME = "replace me" +AUTH_SEED = "replace me" +AUTH_REALM = "replace me" + NETWORK_PREFIX = moo +OTP_VERIFY_ENDPOINT = verify/ + [api] NETWORK_PREFIX = foo diff --git a/ucloud/api/common_fields.py b/ucloud/api/common_fields.py index e9903ac..1ceb1b0 100755 --- a/ucloud/api/common_fields.py +++ b/ucloud/api/common_fields.py @@ -1,7 +1,6 @@ import os -from ucloud.config import etcd_client, env_vars - +from ucloud.config import etcd_client, config class Optional: pass @@ -48,6 +47,6 @@ class VmUUIDField(Field): self.validation = self.vm_uuid_validation def vm_uuid_validation(self): - r = etcd_client.get(os.path.join(env_vars.get('VM_PREFIX'), self.uuid)) + r = etcd_client.get(os.path.join(config['api']['VM_PREFIX'], self.uuid)) if not r: self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index 63d2f90..6735f05 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -7,15 +7,15 @@ import requests from pyotp import TOTP -from ucloud.config import vm_pool, env_vars +from ucloud.config import vm_pool, config def check_otp(name, realm, token): try: data = { - "auth_name": env_vars.get("AUTH_NAME"), - "auth_token": TOTP(env_vars.get("AUTH_SEED")).now(), - "auth_realm": env_vars.get("AUTH_REALM"), + "auth_name": config['api']["AUTH_NAME"], + "auth_token": TOTP(config['api']["AUTH_SEED"]).now(), + "auth_realm": config['api']["AUTH_REALM"], "name": name, "realm": realm, "token": token, @@ -25,8 +25,8 @@ def check_otp(name, realm, token): response = requests.post( "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( - OTP_SERVER=env_vars.get("OTP_SERVER", ""), - OTP_VERIFY_ENDPOINT=env_vars.get("OTP_VERIFY_ENDPOINT", "verify/"), + OTP_SERVER=config['api']["OTP_SERVER"], + OTP_VERIFY_ENDPOINT=config['api']["OTP_VERIFY_ENDPOINT"] ), json=data, ) @@ -35,7 +35,7 @@ def check_otp(name, realm, token): def resolve_vm_name(name, owner): """Return UUID of Virtual Machine of name == name and owner == owner - + Input: name of vm, owner of vm. Output: uuid of vm if found otherwise None """ @@ -54,7 +54,7 @@ def resolve_vm_name(name, owner): def resolve_image_name(name, etcd_client): """Return image uuid given its name and its store - + * If the provided name is not in correct format i.e {store_name}:{image_name} return ValueError * If no such image found then return KeyError @@ -70,9 +70,9 @@ def resolve_image_name(name, etcd_client): """ Examples, where it would work and where it would raise exception "images:alpine" --> ["images", "alpine"] - + "images" --> ["images"] it would raise Exception as non enough value to unpack - + "images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception as too many values to unpack """ @@ -80,7 +80,7 @@ def resolve_image_name(name, etcd_client): except Exception: raise ValueError("Image name not in correct format i.e {store_name}:{image_name}") - images = etcd_client.get_prefix(env_vars.get('IMAGE_PREFIX'), value_in_json=True) + images = etcd_client.get_prefix(config['api']['IMAGE_PREFIX'], value_in_json=True) # Try to find image with name == image_name and store_name == store_name try: @@ -119,14 +119,14 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt=' def get_ip_addr(mac_address, device): """Return IP address of a device provided its mac address / link local address and the device with which it is connected. - + For Example, if we call get_ip_addr(mac_address="52:54:00:12:34:56", device="br0") the following two scenarios can happen 1. It would return None if we can't be able to find device whose mac_address is equal to the arg:mac_address or the mentioned arg:device does not exists or the ip address we found is local. 2. It would return ip_address of device whose mac_address is equal to arg:mac_address - and is connected/neighbor of arg:device + and is connected/neighbor of arg:device """ try: output = sp.check_output(['ip', '-6', 'neigh', 'show', 'dev', device], stderr=sp.PIPE) From 7486fafbaafc7d7d0db5f9d2d21b2e1d386fe105 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 13:23:26 +0100 Subject: [PATCH 011/163] [scheduler] refactored from env_vars to config --- ucloud/scheduler/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index e2c975a..9ebc4c8 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -6,7 +6,7 @@ from ucloud.common.request import RequestEntry, RequestType from ucloud.config import etcd_client -from ucloud.config import host_pool, request_pool, vm_pool, env_vars +from ucloud.config import host_pool, request_pool, vm_pool, config from .helper import (get_suitable_host, dead_host_mitigation, dead_host_detection, assign_host, NoSuitableHostFound) from . import logger @@ -18,8 +18,8 @@ def main(): pending_vms = [] for request_iterator in [ - etcd_client.get_prefix(env_vars.get('REQUEST_PREFIX'), value_in_json=True), - etcd_client.watch_prefix(env_vars.get('REQUEST_PREFIX'), timeout=5, value_in_json=True), + etcd_client.get_prefix(config['etcd']['REQUEST_PREFIX'], value_in_json=True), + etcd_client.watch_prefix(config['etcd']['REQUEST_PREFIX'], timeout=5, value_in_json=True), ]: for request_event in request_iterator: request_entry = RequestEntry(request_event) @@ -46,7 +46,7 @@ def main(): r = RequestEntry.from_scratch(type="ScheduleVM", uuid=pending_vm_entry.uuid, hostname=pending_vm_entry.hostname, - request_prefix=env_vars.get("REQUEST_PREFIX")) + request_prefix=config['etcd']["REQUEST_PREFIX"]) request_pool.put(r) elif request_entry.type == RequestType.ScheduleVM: @@ -72,7 +72,7 @@ def main(): r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, uuid=request_entry.uuid, destination=request_entry.destination, - request_prefix=env_vars.get("REQUEST_PREFIX")) + request_prefix=config['etcd']["REQUEST_PREFIX"]) request_pool.put(r) # If the Request is about a VM that just want to get started/created From 787b2363058e97696b4ce6d8bc514115b979e4f4 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 13:25:03 +0100 Subject: [PATCH 012/163] fixed brackets --- ucloud/scheduler/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index 9ebc4c8..33e94f2 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -46,7 +46,7 @@ def main(): r = RequestEntry.from_scratch(type="ScheduleVM", uuid=pending_vm_entry.uuid, hostname=pending_vm_entry.hostname, - request_prefix=config['etcd']["REQUEST_PREFIX"]) + request_prefix=config['etcd']['REQUEST_PREFIX']) request_pool.put(r) elif request_entry.type == RequestType.ScheduleVM: @@ -72,7 +72,7 @@ def main(): r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, uuid=request_entry.uuid, destination=request_entry.destination, - request_prefix=config['etcd']["REQUEST_PREFIX"]) + request_prefix=config['etcd']['REQUEST_PREFIX']) request_pool.put(r) # If the Request is about a VM that just want to get started/created From 76f63633ca84fff61147e0537430f998cce09c68 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:29:24 +0100 Subject: [PATCH 013/163] [api] done -> configparser --- conf/ucloud.conf | 21 +++++++++++++--- ucloud/api/main.py | 56 +++++++++++++++++++++---------------------- ucloud/api/schemas.py | 12 +++++----- 3 files changed, 52 insertions(+), 37 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 0ebb705..60d5042 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -12,7 +12,22 @@ OTP_VERIFY_ENDPOINT = verify/ [api] NETWORK_PREFIX = foo -[woo] -NETWORK_PREFIX = foo +[network] +PREFIX_LENGTH = 64 +PREFIX = 2001:db8::/48 -[noval] \ No newline at end of file +[netbox] +NETBOX_URL = https://replace-me.example.com +NETBOX_TOKEN = replace me + +[etcd] + +FILE_PREFIX = file/ +HOST_PREFIx = host/ +IMAGE_PREFIX = image/ +IMAGE_STORE_PREFIX = imagestore/ + +NETWORK_PREFIX = network/ +REQUEST_PREFIX = request/ +USER_PREFIX = user/ +VM_PREFIX = vm/ diff --git a/ucloud/api/main.py b/ucloud/api/main.py index 1475fb0..d3ddd5d 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -10,7 +10,7 @@ from flask_restful import Resource, Api from ucloud.common import counters from ucloud.common.vm import VMStatus from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, env_vars, image_storage_handler) +from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, config, image_storage_handler) from . import schemas from .helper import generate_mac, mac2ipv6 from . import logger @@ -28,7 +28,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(env_vars.get("VM_PREFIX"), vm_uuid) + vm_key = join_path(config['api']["VM_PREFIX"), vm_uuid) specs = { "cpu": validator.specs["cpu"], "ram": validator.specs["ram"], @@ -56,7 +56,7 @@ class CreateVM(Resource): # Create ScheduleVM Request r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=env_vars.get("REQUEST_PREFIX") + request_prefix=config['api']["REQUEST_PREFIX") ) request_pool.put(r) @@ -71,7 +71,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = vm_pool.get( - join_path(env_vars.get("VM_PREFIX"), data["uuid"]) + join_path(config['api']["VM_PREFIX"), data["uuid"]) ) vm_value = vm.value.copy() vm_value["ip"] = [] @@ -79,7 +79,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = etcd_client.get( join_path( - env_vars.get("NETWORK_PREFIX"), + config['api']["NETWORK_PREFIX"), data["name"], network_name, ), @@ -100,7 +100,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = etcd_client.get( - join_path(env_vars.get("FILE_PREFIX"), data["uuid"]) + join_path(config['api']["FILE_PREFIX"), data["uuid"]) ) file_entry_value = json.loads(file_entry.value) @@ -113,7 +113,7 @@ class CreateImage(Resource): "visibility": "public", } etcd_client.put( - join_path(env_vars.get("IMAGE_PREFIX"), data["uuid"]), + join_path(config['etcd']["IMAGE_PREFIX"), data["uuid"]), json.dumps(image_entry_json), ) @@ -125,7 +125,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = etcd_client.get_prefix( - env_vars.get("IMAGE_PREFIX"), value_in_json=True + config['etcd']["IMAGE_PREFIX"), value_in_json=True ) r = { "images": [] @@ -148,7 +148,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = vm_pool.get( - join_path(env_vars.get("VM_PREFIX"), data["uuid"]) + join_path(config['etcd']["VM_PREFIX"), data["uuid"]) ) action = data["action"] @@ -172,7 +172,7 @@ class VMAction(Resource): type="{}VM".format(action.title()), uuid=data["uuid"], hostname=vm_entry.hostname, - request_prefix=env_vars.get("REQUEST_PREFIX") + request_prefix=config['etcd']["REQUEST_PREFIX"] ) request_pool.put(r) return {"message": "VM {} Queued".format(action.title())}, 200 @@ -193,10 +193,10 @@ class VMMigration(Resource): type=RequestType.ScheduleVM, uuid=vm.uuid, destination=join_path( - env_vars.get("HOST_PREFIX"), validator.destination.value + config['etcd']["HOST_PREFIX"], validator.destination.value ), migration=True, - request_prefix=env_vars.get("REQUEST_PREFIX") + request_prefix=config['etcd']["REQUEST_PREFIX"] ) request_pool.put(r) return {"message": "VM Migration Initialization Queued"}, 200 @@ -212,7 +212,7 @@ class ListUserVM(Resource): if validator.is_valid(): vms = etcd_client.get_prefix( - env_vars.get("VM_PREFIX"), value_in_json=True + config['etcd']["VM_PREFIX"], value_in_json=True ) return_vms = [] user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) @@ -246,7 +246,7 @@ class ListUserFiles(Resource): if validator.is_valid(): files = etcd_client.get_prefix( - env_vars.get("FILE_PREFIX"), value_in_json=True + config['etcd']["FILE_PREFIX"], value_in_json=True ) return_files = [] user_files = list( @@ -270,7 +270,7 @@ class CreateHost(Resource): data = request.json validator = schemas.CreateHostSchema(data) if validator.is_valid(): - host_key = join_path(env_vars.get("HOST_PREFIX"), uuid4().hex) + host_key = join_path(config['etcd']["HOST_PREFIX"], uuid4().hex) host_entry = { "specs": data["specs"], "hostname": data["hostname"], @@ -309,7 +309,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - env_vars.get('USER_PREFIX'), + config['etcd']['USER_PREFIX'], data["realm"], data["name"], "key", @@ -326,7 +326,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - env_vars.get('USER_PREFIX'), + config['etcd']['USER_PREFIX'), data["realm"], data["name"], "key", @@ -355,7 +355,7 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - env_vars.get("USER_PREFIX"), + config['etcd']["USER_PREFIX"], data["realm"], data["name"], "key", @@ -385,7 +385,7 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - env_vars.get("USER_PREFIX"), + config['etcd']["USER_PREFIX"], data["realm"], data["name"], "key", @@ -421,17 +421,17 @@ class CreateNetwork(Resource): } if validator.user.value: nb = pynetbox.api( - url=env_vars.get("NETBOX_URL"), - token=env_vars.get("NETBOX_TOKEN"), + url=config['netbox']["NETBOX_URL"], + token=config['netbox']["NETBOX_TOKEN"], ) nb_prefix = nb.ipam.prefixes.get( - prefix=env_vars.get("PREFIX") + prefix=config['network']["PREFIX"] ) prefix = nb_prefix.available_prefixes.create( data={ - "prefix_length": env_vars.get( - "PREFIX_LENGTH", cast=int + "prefix_length": config['network'][ + "PREFIX_LENGTH"] ), "description": '{}\'s network "{}"'.format( data["name"], data["network_name"] @@ -444,7 +444,7 @@ class CreateNetwork(Resource): network_entry["ipv6"] = "fd00::/64" network_key = join_path( - env_vars.get("NETWORK_PREFIX"), + config['network']["NETWORK_PREFIX"], data["name"], data["network_name"], ) @@ -462,7 +462,7 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - env_vars.get("NETWORK_PREFIX"), data["name"] + config['network']["NETWORK_PREFIX"], data["name"] ) networks = etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] @@ -498,7 +498,7 @@ api.add_resource(CreateNetwork, "/network/create") def main(): - image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'), value_in_json=True)) + image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True)) if len(image_stores) == 0: data = { "is_public": True, @@ -508,7 +508,7 @@ def main(): "attributes": {"list": [], "key": [], "pool": "images"}, } - etcd_client.put(join_path(env_vars.get('IMAGE_STORE_PREFIX'), uuid4().hex), json.dumps(data)) + etcd_client.put(join_path(config['etcd']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data)) app.run(host="::", debug=True) diff --git a/ucloud/api/schemas.py b/ucloud/api/schemas.py index c4f60ca..23db184 100755 --- a/ucloud/api/schemas.py +++ b/ucloud/api/schemas.py @@ -21,7 +21,7 @@ import bitmath from ucloud.common.host import HostStatus from ucloud.common.vm import VMStatus -from ucloud.config import etcd_client, env_vars, vm_pool, host_pool +from ucloud.config import etcd_client, config, vm_pool, host_pool from . import helper from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -102,14 +102,14 @@ class CreateImageSchema(BaseSchema): super().__init__(data, fields) def file_uuid_validation(self): - file_entry = etcd_client.get(os.path.join(env_vars.get('FILE_PREFIX'), self.uuid.value)) + file_entry = etcd_client.get(os.path.join(config['etcd']['FILE_PREFIX'], self.uuid.value)) if file_entry is None: self.add_error( "Image File with uuid '{}' Not Found".format(self.uuid.value) ) def image_store_name_validation(self): - image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'))) + image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'])) image_store = next( filter( @@ -235,7 +235,7 @@ class CreateVMSchema(OTPSchema): if _network: for net in _network: - network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'), + network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'], self.name.value, net), value_in_json=True) if not network: @@ -400,7 +400,7 @@ class VmMigrationSchema(OTPSchema): if vm.status != VMStatus.running: self.add_error("Can't migrate non-running VM") - if vm.hostname == os.path.join(env_vars.get('HOST_PREFIX'), self.destination.value): + if vm.hostname == os.path.join(config['etcd']['HOST_PREFIX'], self.destination.value): self.add_error("Destination host couldn't be same as Source Host") @@ -442,7 +442,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'), + network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'], self.name.value, self.network_name.value), value_in_json=True) From dd33b89941e447ea3dc9738c10aeb95a81c26dea Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 13:31:56 +0100 Subject: [PATCH 014/163] [scheduler] helper.py refactored from env_vars to config --- ucloud/scheduler/helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ucloud/scheduler/helper.py b/ucloud/scheduler/helper.py index ba577d6..1754045 100755 --- a/ucloud/scheduler/helper.py +++ b/ucloud/scheduler/helper.py @@ -6,7 +6,7 @@ import bitmath from ucloud.common.host import HostStatus from ucloud.common.request import RequestEntry, RequestType from ucloud.common.vm import VMStatus -from ucloud.config import vm_pool, host_pool, request_pool, env_vars +from ucloud.config import vm_pool, host_pool, request_pool, config def accumulated_specs(vms_specs): @@ -106,7 +106,7 @@ def assign_host(vm): r = RequestEntry.from_scratch(type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=env_vars.get("REQUEST_PREFIX")) + request_prefix=config['etcd']['REQUEST_PREFIX']) request_pool.put(r) vm.log.append("VM scheduled for starting") From 431a6f6d9bc15ccb4bd479d21ec07391ea05fe57 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:32:06 +0100 Subject: [PATCH 015/163] [metadata] -> configparser --- ucloud/metadata/main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ucloud/metadata/main.py b/ucloud/metadata/main.py index e7cb33b..9281d7c 100644 --- a/ucloud/metadata/main.py +++ b/ucloud/metadata/main.py @@ -3,7 +3,7 @@ import os from flask import Flask, request from flask_restful import Resource, Api -from ucloud.config import etcd_client, env_vars, vm_pool +from ucloud.config import etcd_client, config, vm_pool app = Flask(__name__) api = Api(app) @@ -43,9 +43,7 @@ class Root(Resource): if not data: return {'message': 'Metadata for such VM does not exists.'}, 404 else: - - # {env_vars.get('USER_PREFIX')}/{realm}/{name}/key - etcd_key = os.path.join(env_vars.get('USER_PREFIX'), data.value['owner_realm'], + etcd_key = os.path.join(config['etcd']['USER_PREFIX'], data.value['owner_realm'], data.value['owner'], 'key') etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True) user_personal_ssh_keys = [key.value for key in etcd_entry] From c37bf19f9215929409cb1d2387120d3515644b41 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:36:45 +0100 Subject: [PATCH 016/163] ++conf --- conf/ucloud.conf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 60d5042..f2b681a 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -1,4 +1,5 @@ # This section contains default values for all other sections +# [DEFAULT] AUTH_NAME = "replace me" @@ -21,9 +22,16 @@ NETBOX_URL = https://replace-me.example.com NETBOX_TOKEN = replace me [etcd] +ETCD_URL = localhost +ETCD_PORT = 2379 + +CA_CERT = changeme +CERT_CERT = changeme +CERT_KEY = changeme + FILE_PREFIX = file/ -HOST_PREFIx = host/ +HOST_PREFIx = hosts IMAGE_PREFIX = image/ IMAGE_STORE_PREFIX = imagestore/ From 608d1eb28006ba5cf70786275ec3caa026a38394 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 13:41:32 +0100 Subject: [PATCH 017/163] [host] main.py refactored from env_vars to config --- ucloud/host/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ucloud/host/main.py b/ucloud/host/main.py index ca68351..0ca345b 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -8,7 +8,7 @@ from ucloud.common.request import RequestEntry, RequestType from ucloud.config import (vm_pool, request_pool, etcd_client, running_vms, etcd_wrapper_args, etcd_wrapper_kwargs, - HostPool, env_vars) + HostPool, config) from .helper import find_free_port from . import virtualmachine @@ -18,7 +18,7 @@ from ucloud.host import logger def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) - host_pool = HostPool(client, env_vars.get('HOST_PREFIX')) + host_pool = HostPool(client, config['etcd']['HOST_PREFIX']) this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) while True: @@ -72,7 +72,7 @@ def maintenance(host): running_vms.remove(_vm) def check(): - if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('VM_DIR')): + if config['etcd']['STORAGE_BACKEND'] == 'filesystem' and not isdir(config['etcd']['VM_DIR']): print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" " in .env file must exists. But, it don't.") sys.exit(1) @@ -84,7 +84,7 @@ def main(hostname): heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) - host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX')) + host_pool = HostPool(etcd_client, config['etcd']['HOST_PREFIX']) host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) assert host is not None, "No such host with name = {}".format(hostname) @@ -106,8 +106,8 @@ def main(hostname): # beat updating mechanism in separated thread for events_iterator in [ - etcd_client.get_prefix(env_vars.get('REQUEST_PREFIX'), value_in_json=True), - etcd_client.watch_prefix(env_vars.get('REQUEST_PREFIX'), timeout=10, value_in_json=True), + etcd_client.get_prefix(config['etcd']['REQUEST_PREFIX'], value_in_json=True), + etcd_client.watch_prefix(config['etcd']['REQUEST_PREFIX'], timeout=10, value_in_json=True), ]: for request_event in events_iterator: request_event = RequestEntry(request_event) From 72af426b3a3687e66597bad823d06310b403d907 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:41:42 +0100 Subject: [PATCH 018/163] update config x2 --- conf/ucloud.conf | 8 ++++---- ucloud/config.py | 29 +++++++++++++---------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index f2b681a..7cd9725 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -35,7 +35,7 @@ HOST_PREFIx = hosts IMAGE_PREFIX = image/ IMAGE_STORE_PREFIX = imagestore/ -NETWORK_PREFIX = network/ -REQUEST_PREFIX = request/ -USER_PREFIX = user/ -VM_PREFIX = vm/ +NETWORK_PREFIX = networks +REQUEST_PREFIX = requests +USER_PREFIX = users +VM_PREFIX = vms diff --git a/ucloud/config.py b/ucloud/config.py index b508f56..b8b4dcf 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -35,10 +35,6 @@ except FileNotFoundError: log.warn("Configuration file not found - using defaults") -# Compatibility to old code -env_vars = config - - # Try importing config, but don't fail if it does not exist # try: # env_vars = Config(RepositoryEnv('/etc/ucloud/ucloud.conf')) @@ -48,26 +44,27 @@ env_vars = config etcd_wrapper_args = () etcd_wrapper_kwargs = { - 'host': env_vars.get('ETCD_URL', 'localhost'), - 'port': env_vars.get('ETCD_PORT', 2379), - 'ca_cert': env_vars.get('CA_CERT', None), - 'cert_cert': env_vars.get('CERT_CERT', None), - 'cert_key': env_vars.get('CERT_KEY', None) + 'host': config['etcd']['ETCD_URL'], + 'port': config['etcd']['ETCD_PORT'], + 'ca_cert': config['etcd']['CA_CERT'], + 'cert_cert': config['etcd']['CERT_CERT'], + 'cert_key': config['etcd']['CERT_KEY'] } etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) -host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX', "hosts")) -vm_pool = VmPool(etcd_client, env_vars.get('VM_PREFIX', "vms")) -request_pool = RequestPool(etcd_client, env_vars.get('REQUEST_PREFIX', "requests")) +host_pool = HostPool(etcd_client, config['etcd']['HOST_PREFIX']) +vm_pool = VmPool(etcd_client, config['etcd']['VM_PREFIX']) +request_pool = RequestPool(etcd_client, config['etcd']['REQUEST_PREFIX']) running_vms = [] -__storage_backend = env_vars.get("STORAGE_BACKEND", "filesystem") +__storage_backend = config['storage']["STORAGE_BACKEND"] if __storage_backend == "filesystem": - image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=env_vars.get("VM_DIR", "/tmp/ucloud-vms"), - image_base=env_vars.get("IMAGE_DIR", "/tmp/ucloud-images")) + image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=config['storage']["VM_DIR"], + image_base=config['storage']["IMAGE_DIR"]) elif __storage_backend == "ceph": - image_storage_handler = CEPHBasedImageStorageHandler(vm_base="ssd", image_base="ssd") + image_storage_handler = CEPHBasedImageStorageHandler(vm_base=config['storage']["CEPH_VM_POOL"], + image_base=config['storage']["CEPH_IMAGE_POOL"]) else: raise Exception("Unknown Image Storage Handler") From 0d38a66a347a1550e9a8604e2df2877250833dd6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:46:24 +0100 Subject: [PATCH 019/163] add a wrapper to re install ucloud and then run it --- bin/ucloud-run-reinstall | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 bin/ucloud-run-reinstall diff --git a/bin/ucloud-run-reinstall b/bin/ucloud-run-reinstall new file mode 100644 index 0000000..b189bbc --- /dev/null +++ b/bin/ucloud-run-reinstall @@ -0,0 +1,29 @@ +#!/bin/sh +# -*- coding: utf-8 -*- +# +# 2012-2019 Nico Schottelius (nico-ucloud at schottelius.org) +# +# This file is part of ucloud. +# +# ucloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ucloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ucloud. If not, see . +# +# + +# Wrapper for real script to allow execution from checkout +dir=${0%/*} + +${dir}/gen-version; +pip uninstall -y ucloud +python setup.py install +${dir}/ucloud "$@" From 012f3cb3b56a4dcf33f14492edda2f246c1be1c8 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 13:46:33 +0100 Subject: [PATCH 020/163] [conf] added storage dictionary --- conf/ucloud.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 7cd9725..e0e3c27 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -39,3 +39,10 @@ NETWORK_PREFIX = networks REQUEST_PREFIX = requests USER_PREFIX = users VM_PREFIX = vms + +[storage] +STORAGE_BACKEND = #values = filesystem, +VM_DIR = +IMG_DIR = +CEPH_VM_POOL = +CEPH_IMG_POOL = From 00563c7dc2e1f7fc4cd3c9f859afb36adef9daa0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:51:40 +0100 Subject: [PATCH 021/163] [filescanner] use configparser --- conf/ucloud.conf | 11 ++++++++++- ucloud/filescanner/main.py | 9 ++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index e0e3c27..2a06c81 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -41,8 +41,17 @@ USER_PREFIX = users VM_PREFIX = vms [storage] -STORAGE_BACKEND = #values = filesystem, +#values = filesystem, ceph +STORAGE_BACKEND = + +# if STORAGE_BACKEND = filesystem VM_DIR = IMG_DIR = + +# if STORAGE_BACKEND = ceph CEPH_VM_POOL = CEPH_IMG_POOL = + +# Importing uploaded files +FILE_DIR = /var/lib/ucloud/files +FILE_PREFIX = noclue-ahmed \ No newline at end of file diff --git a/ucloud/filescanner/main.py b/ucloud/filescanner/main.py index b70cb5b..ef6fee6 100755 --- a/ucloud/filescanner/main.py +++ b/ucloud/filescanner/main.py @@ -6,7 +6,7 @@ import time from uuid import uuid4 from . import logger -from ucloud.config import env_vars, etcd_client +from ucloud.config import config, etcd_client def getxattr(file, attr): @@ -37,7 +37,7 @@ def setxattr(file, attr, value): def sha512sum(file: str): """Use sha512sum utility to compute sha512 sum of arg:file - + IF arg:file does not exists: raise FileNotFoundError exception ELSE IF sum successfully computer: @@ -69,9 +69,8 @@ except Exception as e: def main(): - BASE_DIR = env_vars.get("BASE_DIR") - - FILE_PREFIX = env_vars.get("FILE_PREFIX") + BASE_DIR = config['storage']["FILE_DIR"] + FILE_PREFIX = config['storage']["FILE_PREFIX"] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob("{}/**".format(BASE_DIR), recursive=True) From b2de2772447d8c8793fa0b3455503457f45b86af Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:51:52 +0100 Subject: [PATCH 022/163] [conf] add values for filescanner --- ucloud/config.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ucloud/config.py b/ucloud/config.py index b8b4dcf..834d51e 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -5,18 +5,14 @@ from ucloud.common.request import RequestPool from ucloud.common.vm import VmPool from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler -from decouple import Config, RepositoryEnv, RepositoryEmpty - # Replacing decouple inline import configparser import os import os.path import logging - log = logging.getLogger("ucloud.config") - conf_name = "ucloud.conf" try: @@ -34,14 +30,6 @@ try: except FileNotFoundError: log.warn("Configuration file not found - using defaults") - -# Try importing config, but don't fail if it does not exist -# try: -# env_vars = Config(RepositoryEnv('/etc/ucloud/ucloud.conf')) -# except FileNotFoundError: -# env_vars = Config(RepositoryEmpty()) - - etcd_wrapper_args = () etcd_wrapper_kwargs = { 'host': config['etcd']['ETCD_URL'], From fee1cfd4ff8376a0456ac64e94e8fb47d408e17e Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 14:01:41 +0100 Subject: [PATCH 023/163] [conf] added ssh dictionary --- conf/ucloud.conf | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index e0e3c27..e6b92cb 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -41,8 +41,21 @@ USER_PREFIX = users VM_PREFIX = vms [storage] -STORAGE_BACKEND = #values = filesystem, +#values = filesystem, ceph +STORAGE_BACKEND = + +# if STORAGE_BACKEND = filesystem VM_DIR = IMG_DIR = + +# if STORAGE_BACKEND = ceph CEPH_VM_POOL = CEPH_IMG_POOL = + +# Importing uploaded files +FILE_DIR = /var/lib/ucloud/files +FILE_PREFIX = noclue-ahmed + +[ssh] +SSH_USERNAME = +SSH_PRIVATEKEY = From 1e70d0183d1ba4308409ddd34bcb3eb575a55377 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 14:05:38 +0100 Subject: [PATCH 024/163] [conf] added ssh dictionary --- conf/ucloud.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 2a06c81..574874e 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -54,4 +54,8 @@ CEPH_IMG_POOL = # Importing uploaded files FILE_DIR = /var/lib/ucloud/files -FILE_PREFIX = noclue-ahmed \ No newline at end of file +FILE_PREFIX = noclue-ahmed + +[ssh] +SSH_USERNAME = +SSH_PRIVATEKEY = From c6fe2cb1c4274136d1d36f60b5aa885a15dd7e88 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 14:06:15 +0100 Subject: [PATCH 025/163] [host] virtualmachine.py refactored from env_vars to config --- ucloud/host/virtualmachine.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index 7524083..33e4433 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -22,7 +22,7 @@ from ucloud.common.helpers import get_ipv6_address from ucloud.common.request import RequestEntry, RequestType from ucloud.common.vm import VMEntry, VMStatus from ucloud.config import (etcd_client, request_pool, - running_vms, vm_pool, env_vars, + running_vms, vm_pool, config, image_storage_handler) from . import qmp from ucloud.host import logger @@ -46,7 +46,7 @@ def delete_network_interface(iface): def resolve_network(network_name, network_owner): - network = etcd_client.get(join_path(env_vars.get("NETWORK_PREFIX"), + network = etcd_client.get(join_path(config['etcd']["NETWORK_PREFIX"], network_owner, network_name), value_in_json=True) @@ -179,7 +179,7 @@ def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, mi for network_mac_and_tap in vm_networks: network_name, mac, tap = network_mac_and_tap - _key = os.path.join(env_vars.get('NETWORK_PREFIX'), vm_entry.owner, network_name) + _key = os.path.join(config['etcd']['NETWORK_PREFIX'], vm_entry.owner, network_name) network = etcd_client.get(_key, value_in_json=True) network_type = network.value["type"] network_id = str(network.value["id"]) @@ -187,7 +187,7 @@ def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, mi if network_type == "vxlan": tap = create_vxlan_br_tap(_id=network_id, - _dev=env_vars.get("VXLAN_PHY_DEV"), + _dev=config['etcd']["VXLAN_PHY_DEV"], tap_id=tap, ip=network_ipv6) update_radvd_conf(etcd_client) @@ -303,13 +303,13 @@ def transfer(request_event): _host, _port = request_event.parameters["host"], request_event.parameters["port"] _uuid = request_event.uuid _destination = request_event.destination_host_key - vm = get_vm(running_vms, join_path(env_vars.get('VM_PREFIX'), _uuid)) + vm = get_vm(running_vms, join_path(config['etcd']['VM_PREFIX'], _uuid)) if vm: tunnel = sshtunnel.SSHTunnelForwarder( _host, - ssh_username=env_vars.get("ssh_username"), - ssh_pkey=env_vars.get("ssh_pkey"), + ssh_username=config['ssh']["ssh_username"], + ssh_pkey=config['ssh']["SSH_PRIVATEKEY"], remote_bind_address=("127.0.0.1", _port), ssh_proxy_enabled=True, ssh_proxy=(_host, 22) @@ -373,7 +373,7 @@ def launch_vm(vm_entry, migration=False, migration_port=None, destination_host_k parameters={"host": get_ipv6_address(), "port": migration_port}, uuid=vm_entry.uuid, destination_host_key=destination_host_key, - request_prefix=env_vars.get("REQUEST_PREFIX") + request_prefix=config['etcd']["REQUEST_PREFIX"] ) request_pool.put(r) else: From 9ec9083c57f7d0005ce8cc513625b7798b3020c4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 14:08:40 +0100 Subject: [PATCH 026/163] conf update Signed-off-by: Nico Schottelius --- conf/ucloud.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 2a06c81..cebe58e 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -30,9 +30,9 @@ CERT_CERT = changeme CERT_KEY = changeme -FILE_PREFIX = file/ +FILE_PREFIX = files HOST_PREFIx = hosts -IMAGE_PREFIX = image/ +IMAGE_PREFIX = images IMAGE_STORE_PREFIX = imagestore/ NETWORK_PREFIX = networks From 2a1e80dbc57cd83a64fecc1988b4f6786d14d4c4 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 14:11:19 +0100 Subject: [PATCH 027/163] [imagescanner] main.py refactored from env_vars to config --- ucloud/imagescanner/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index 6ff01f8..df4dfad 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -3,7 +3,7 @@ import os import subprocess from os.path import join as join_path -from ucloud.config import etcd_client, env_vars, image_storage_handler +from ucloud.config import etcd_client, config, image_storage_handler from ucloud.imagescanner import logger @@ -20,9 +20,9 @@ def qemu_img_type(path): def check(): """ check whether settings are sane, refuse to start if they aren't """ - if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('IMAGE_DIR')): + if config['etcd']['STORAGE_BACKEND'] == 'filesystem' and not isdir(config['etcd']['IMAGE_DIR']): print("You have set STORAGE_BACKEND to filesystem, but " - "{} does not exist. Refusing to start".format(env_vars.get('IMAGE_DIR'))) + "{} does not exist. Refusing to start".format(config['etcd']['IMAGE_DIR'])) sys.exit(1) try: @@ -34,7 +34,7 @@ def check(): def main(): # We want to get images entries that requests images to be created - images = etcd_client.get_prefix(env_vars.get('IMAGE_PREFIX'), value_in_json=True) + images = etcd_client.get_prefix(config['etcd']['IMAGE_PREFIX'], value_in_json=True) images_to_be_created = list(filter(lambda im: im.value['status'] == 'TO_BE_CREATED', images)) for image in images_to_be_created: @@ -43,9 +43,9 @@ def main(): image_owner = image.value['owner'] image_filename = image.value['filename'] image_store_name = image.value['store_name'] - image_full_path = join_path(env_vars.get('BASE_DIR'), image_owner, image_filename) + image_full_path = join_path(config['etcd']['BASE_DIR'], image_owner, image_filename) - image_stores = etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'), value_in_json=True) + image_stores = etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True) user_image_store = next(filter( lambda s, store_name=image_store_name: s.value["name"] == store_name, image_stores From bfbf08c7cddf54bd49ac601fd154e60b96623307 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 14:11:44 +0100 Subject: [PATCH 028/163] [conf] added unkown values --- conf/ucloud.conf | 5 ++ ucloud/host/main.py.old | 152 +++++++++++++++++++++++++++++++++++ ucloud/scheduler/main.py.old | 93 +++++++++++++++++++++ 3 files changed, 250 insertions(+) create mode 100755 ucloud/host/main.py.old create mode 100755 ucloud/scheduler/main.py.old diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 574874e..2c3178a 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -59,3 +59,8 @@ FILE_PREFIX = noclue-ahmed [ssh] SSH_USERNAME = SSH_PRIVATEKEY = + +# unkown vars: +IMAGE_DIR = +BASE_DIR = +IMAGE_STORE_PREFIX = diff --git a/ucloud/host/main.py.old b/ucloud/host/main.py.old new file mode 100755 index 0000000..ae4c069 --- /dev/null +++ b/ucloud/host/main.py.old @@ -0,0 +1,152 @@ +import argparse +import multiprocessing as mp +import time + +from etcd3_wrapper import Etcd3Wrapper + +from ucloud.common.request import RequestEntry, RequestType +from ucloud.config import (vm_pool, request_pool, + etcd_client, running_vms, + etcd_wrapper_args, etcd_wrapper_kwargs, + HostPool, config) + +from .helper import find_free_port +from . import virtualmachine +from ucloud.host import logger + + +def update_heartbeat(hostname): + """Update Last HeartBeat Time for :param hostname: in etcd""" + client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) + host_pool = HostPool(client, env_vars.get('HOST_PREFIX')) + this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + + while True: + this_host.update_heartbeat() + host_pool.put(this_host) + time.sleep(10) + + +def maintenance(host): + # To capture vm running according to running_vms list + + # This is to capture successful migration of a VM. + # Suppose, this host is running "vm1" and user initiated + # request to migrate this "vm1" to some other host. On, + # successful migration the destination host would set + # the vm hostname to itself. Thus, we are checking + # whether this host vm is successfully migrated. If yes + # then we shutdown "vm1" on this host. + + to_be_removed = [] + for running_vm in running_vms: + with vm_pool.get_put(running_vm.key) as vm_entry: + if vm_entry.hostname != host.key and not vm_entry.in_migration: + running_vm.handle.shutdown() + logger.info("VM migration not completed successfully.") + to_be_removed.append(running_vm) + + for r in to_be_removed: + running_vms.remove(r) + + # To check vm running according to etcd entries + alleged_running_vms = vm_pool.by_status("RUNNING", vm_pool.by_host(host.key)) + + for vm_entry in alleged_running_vms: + _vm = virtualmachine.get_vm(running_vms, vm_entry.key) + # Whether, the allegedly running vm is in our + # running_vms list or not if it is said to be + # running on this host but it is not then we + # need to shut it down + + # This is to capture poweroff/shutdown of a VM + # initiated by user inside VM. OR crash of VM by some + # user running process + if (_vm and not _vm.handle.is_running()) or not _vm: + logger.debug("_vm = %s, is_running() = %s" % (_vm, _vm.handle.is_running())) + vm_entry.add_log("""{} is not running but is said to be running. + So, shutting it down and declare it killed""".format(vm_entry.key)) + vm_entry.declare_killed() + vm_pool.put(vm_entry) + if _vm: + running_vms.remove(_vm) + +def check(): + if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('VM_DIR')): + print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" + " in .env file must exists. But, it don't.") + sys.exit(1) + + + +def main(hostname): + check() + + heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) + + host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX')) + host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + assert host is not None, "No such host with name = {}".format(hostname) + + try: + heartbeat_updating_process.start() + except Exception as e: + logger.info("No Need To Go Further. Our heartbeat updating mechanism is not working") + logger.exception(e) + exit(-1) + + logger.info("%s Session Started %s", '*' * 5, '*' * 5) + + # It is seen that under heavy load, timeout event doesn't come + # in a predictive manner (which is intentional because we give + # higher priority to customer's requests) which delays heart + # beat update which in turn misunderstood by scheduler that the + # host is dead when it is actually alive. So, to ensure that we + # update the heart beat in a predictive manner we start Heart + # beat updating mechanism in separated thread + + for events_iterator in [ + etcd_client.get_prefix(env_vars.get('REQUEST_PREFIX'), value_in_json=True), + etcd_client.watch_prefix(env_vars.get('REQUEST_PREFIX'), timeout=10, value_in_json=True), + ]: + for request_event in events_iterator: + request_event = RequestEntry(request_event) + + if request_event.type == "TIMEOUT": + maintenance(host) + continue + + # If the event is directed toward me OR I am destination of a InitVMMigration + if request_event.hostname == host.key or request_event.destination == host.key: + logger.debug("VM Request: %s", request_event) + + request_pool.client.client.delete(request_event.key) + vm_entry = vm_pool.get(request_event.uuid) + + if vm_entry: + if request_event.type == RequestType.StartVM: + virtualmachine.start(vm_entry) + + elif request_event.type == RequestType.StopVM: + virtualmachine.stop(vm_entry) + + elif request_event.type == RequestType.DeleteVM: + virtualmachine.delete(vm_entry) + + elif request_event.type == RequestType.InitVMMigration: + virtualmachine.start(vm_entry, host.key, find_free_port()) + + elif request_event.type == RequestType.TransferVM: + virtualmachine.transfer(request_event) + else: + logger.info("VM Entry missing") + + logger.info("Running VMs %s", running_vms) + + +if __name__ == "__main__": + argparser = argparse.ArgumentParser() + argparser.add_argument("hostname", help="Name of this host. e.g /v1/host/1") + args = argparser.parse_args() + mp.set_start_method('spawn') + main(args.hostname) diff --git a/ucloud/scheduler/main.py.old b/ucloud/scheduler/main.py.old new file mode 100755 index 0000000..e2c975a --- /dev/null +++ b/ucloud/scheduler/main.py.old @@ -0,0 +1,93 @@ +# TODO +# 1. send an email to an email address defined by env['admin-email'] +# if resources are finished +# 2. Introduce a status endpoint of the scheduler - +# maybe expose a prometheus compatible output + +from ucloud.common.request import RequestEntry, RequestType +from ucloud.config import etcd_client +from ucloud.config import host_pool, request_pool, vm_pool, env_vars +from .helper import (get_suitable_host, dead_host_mitigation, dead_host_detection, + assign_host, NoSuitableHostFound) +from . import logger + + +def main(): + logger.info("%s SESSION STARTED %s", '*' * 5, '*' * 5) + + pending_vms = [] + + for request_iterator in [ + etcd_client.get_prefix(env_vars.get('REQUEST_PREFIX'), value_in_json=True), + etcd_client.watch_prefix(env_vars.get('REQUEST_PREFIX'), timeout=5, value_in_json=True), + ]: + for request_event in request_iterator: + request_entry = RequestEntry(request_event) + # Never Run time critical mechanism inside timeout + # mechanism because timeout mechanism only comes + # when no other event is happening. It means under + # heavy load there would not be a timeout event. + if request_entry.type == "TIMEOUT": + + # Detect hosts that are dead and set their status + # to "DEAD", and their VMs' status to "KILLED" + dead_hosts = dead_host_detection() + if dead_hosts: + logger.debug("Dead hosts: %s", dead_hosts) + dead_host_mitigation(dead_hosts) + + # If there are VMs that weren't assigned a host + # because there wasn't a host available which + # meets requirement of that VM then we would + # create a new ScheduleVM request for that VM + # on our behalf. + while pending_vms: + pending_vm_entry = pending_vms.pop() + r = RequestEntry.from_scratch(type="ScheduleVM", + uuid=pending_vm_entry.uuid, + hostname=pending_vm_entry.hostname, + request_prefix=env_vars.get("REQUEST_PREFIX")) + request_pool.put(r) + + elif request_entry.type == RequestType.ScheduleVM: + logger.debug("%s, %s", request_entry.key, request_entry.value) + + vm_entry = vm_pool.get(request_entry.uuid) + if vm_entry is None: + logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) + continue + etcd_client.client.delete(request_entry.key) # consume Request + + # If the Request is about a VM which is labelled as "migration" + # and has a destination + if hasattr(request_entry, "migration") and request_entry.migration \ + and hasattr(request_entry, "destination") and request_entry.destination: + try: + get_suitable_host(vm_specs=vm_entry.specs, + hosts=[host_pool.get(request_entry.destination)]) + except NoSuitableHostFound: + logger.info("Requested destination host doesn't have enough capacity" + "to hold %s" % vm_entry.uuid) + else: + r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, + uuid=request_entry.uuid, + destination=request_entry.destination, + request_prefix=env_vars.get("REQUEST_PREFIX")) + request_pool.put(r) + + # If the Request is about a VM that just want to get started/created + else: + # assign_host only returns None when we couldn't be able to assign + # a host to a VM because of resource constraints + try: + assign_host(vm_entry) + except NoSuitableHostFound: + vm_entry.add_log("Can't schedule VM. No Resource Left.") + vm_pool.put(vm_entry) + + pending_vms.append(vm_entry) + logger.info("No Resource Left. Emailing admin....") + + +if __name__ == "__main__": + main() From 8b90755015ef098935cd2b4e0eb878402ea0fee6 Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Dec 2019 14:14:32 +0100 Subject: [PATCH 029/163] removed unwanted file --- ucloud/host/main.py.old | 152 ---------------------------------------- 1 file changed, 152 deletions(-) delete mode 100755 ucloud/host/main.py.old diff --git a/ucloud/host/main.py.old b/ucloud/host/main.py.old deleted file mode 100755 index ae4c069..0000000 --- a/ucloud/host/main.py.old +++ /dev/null @@ -1,152 +0,0 @@ -import argparse -import multiprocessing as mp -import time - -from etcd3_wrapper import Etcd3Wrapper - -from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import (vm_pool, request_pool, - etcd_client, running_vms, - etcd_wrapper_args, etcd_wrapper_kwargs, - HostPool, config) - -from .helper import find_free_port -from . import virtualmachine -from ucloud.host import logger - - -def update_heartbeat(hostname): - """Update Last HeartBeat Time for :param hostname: in etcd""" - client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) - host_pool = HostPool(client, env_vars.get('HOST_PREFIX')) - this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) - - while True: - this_host.update_heartbeat() - host_pool.put(this_host) - time.sleep(10) - - -def maintenance(host): - # To capture vm running according to running_vms list - - # This is to capture successful migration of a VM. - # Suppose, this host is running "vm1" and user initiated - # request to migrate this "vm1" to some other host. On, - # successful migration the destination host would set - # the vm hostname to itself. Thus, we are checking - # whether this host vm is successfully migrated. If yes - # then we shutdown "vm1" on this host. - - to_be_removed = [] - for running_vm in running_vms: - with vm_pool.get_put(running_vm.key) as vm_entry: - if vm_entry.hostname != host.key and not vm_entry.in_migration: - running_vm.handle.shutdown() - logger.info("VM migration not completed successfully.") - to_be_removed.append(running_vm) - - for r in to_be_removed: - running_vms.remove(r) - - # To check vm running according to etcd entries - alleged_running_vms = vm_pool.by_status("RUNNING", vm_pool.by_host(host.key)) - - for vm_entry in alleged_running_vms: - _vm = virtualmachine.get_vm(running_vms, vm_entry.key) - # Whether, the allegedly running vm is in our - # running_vms list or not if it is said to be - # running on this host but it is not then we - # need to shut it down - - # This is to capture poweroff/shutdown of a VM - # initiated by user inside VM. OR crash of VM by some - # user running process - if (_vm and not _vm.handle.is_running()) or not _vm: - logger.debug("_vm = %s, is_running() = %s" % (_vm, _vm.handle.is_running())) - vm_entry.add_log("""{} is not running but is said to be running. - So, shutting it down and declare it killed""".format(vm_entry.key)) - vm_entry.declare_killed() - vm_pool.put(vm_entry) - if _vm: - running_vms.remove(_vm) - -def check(): - if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('VM_DIR')): - print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" - " in .env file must exists. But, it don't.") - sys.exit(1) - - - -def main(hostname): - check() - - heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) - - host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX')) - host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) - assert host is not None, "No such host with name = {}".format(hostname) - - try: - heartbeat_updating_process.start() - except Exception as e: - logger.info("No Need To Go Further. Our heartbeat updating mechanism is not working") - logger.exception(e) - exit(-1) - - logger.info("%s Session Started %s", '*' * 5, '*' * 5) - - # It is seen that under heavy load, timeout event doesn't come - # in a predictive manner (which is intentional because we give - # higher priority to customer's requests) which delays heart - # beat update which in turn misunderstood by scheduler that the - # host is dead when it is actually alive. So, to ensure that we - # update the heart beat in a predictive manner we start Heart - # beat updating mechanism in separated thread - - for events_iterator in [ - etcd_client.get_prefix(env_vars.get('REQUEST_PREFIX'), value_in_json=True), - etcd_client.watch_prefix(env_vars.get('REQUEST_PREFIX'), timeout=10, value_in_json=True), - ]: - for request_event in events_iterator: - request_event = RequestEntry(request_event) - - if request_event.type == "TIMEOUT": - maintenance(host) - continue - - # If the event is directed toward me OR I am destination of a InitVMMigration - if request_event.hostname == host.key or request_event.destination == host.key: - logger.debug("VM Request: %s", request_event) - - request_pool.client.client.delete(request_event.key) - vm_entry = vm_pool.get(request_event.uuid) - - if vm_entry: - if request_event.type == RequestType.StartVM: - virtualmachine.start(vm_entry) - - elif request_event.type == RequestType.StopVM: - virtualmachine.stop(vm_entry) - - elif request_event.type == RequestType.DeleteVM: - virtualmachine.delete(vm_entry) - - elif request_event.type == RequestType.InitVMMigration: - virtualmachine.start(vm_entry, host.key, find_free_port()) - - elif request_event.type == RequestType.TransferVM: - virtualmachine.transfer(request_event) - else: - logger.info("VM Entry missing") - - logger.info("Running VMs %s", running_vms) - - -if __name__ == "__main__": - argparser = argparse.ArgumentParser() - argparser.add_argument("hostname", help="Name of this host. e.g /v1/host/1") - args = argparser.parse_args() - mp.set_start_method('spawn') - main(args.hostname) From 5b44034602faa719939f268d0163461bdb2d0943 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 14:15:36 +0100 Subject: [PATCH 030/163] cleanup --- bin/ucloud-run-reinstall | 0 scripts/ucloud | 11 +++-------- ucloud/api/main.py | 22 ++++++++++------------ ucloud/config.py | 3 +-- 4 files changed, 14 insertions(+), 22 deletions(-) mode change 100644 => 100755 bin/ucloud-run-reinstall diff --git a/bin/ucloud-run-reinstall b/bin/ucloud-run-reinstall old mode 100644 new mode 100755 diff --git a/scripts/ucloud b/scripts/ucloud index b8ef32d..0780fb4 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -18,21 +18,16 @@ if __name__ == "__main__": arg_parser.add_argument('component_args', nargs='*') args = arg_parser.parse_args() + if args.conf_dir: + os.environ['UCLOUD_CONF_DIR'] = args.conf_dir + try: name = args.component mod = importlib.import_module("ucloud.{}.main".format(name)) main = getattr(mod, "main") - if args.conf_dir: - print("setting conf") - os.environ['UCLOUD_CONF_DIR'] = args.conf_dir - main() - # except decouple.UndefinedValueError as e: - # print(e) - # sys.exit(1) - except Exception as e: logging.exception(e) print(e) diff --git a/ucloud/api/main.py b/ucloud/api/main.py index d3ddd5d..bbda7e9 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -28,7 +28,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(config['api']["VM_PREFIX"), vm_uuid) + vm_key = join_path(config['etcd']["VM_PREFIX"], vm_uuid) specs = { "cpu": validator.specs["cpu"], "ram": validator.specs["ram"], @@ -56,7 +56,7 @@ class CreateVM(Resource): # Create ScheduleVM Request r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=config['api']["REQUEST_PREFIX") + request_prefix=config['etcd']["REQUEST_PREFIX"] ) request_pool.put(r) @@ -71,7 +71,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = vm_pool.get( - join_path(config['api']["VM_PREFIX"), data["uuid"]) + join_path(config['etcd']["VM_PREFIX"], data["uuid"]) ) vm_value = vm.value.copy() vm_value["ip"] = [] @@ -79,7 +79,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = etcd_client.get( join_path( - config['api']["NETWORK_PREFIX"), + config['etcd']["NETWORK_PREFIX"], data["name"], network_name, ), @@ -100,7 +100,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = etcd_client.get( - join_path(config['api']["FILE_PREFIX"), data["uuid"]) + join_path(config['etcd']["FILE_PREFIX"], data["uuid"]) ) file_entry_value = json.loads(file_entry.value) @@ -113,7 +113,7 @@ class CreateImage(Resource): "visibility": "public", } etcd_client.put( - join_path(config['etcd']["IMAGE_PREFIX"), data["uuid"]), + join_path(config['etcd']["IMAGE_PREFIX"], data["uuid"]), json.dumps(image_entry_json), ) @@ -125,7 +125,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = etcd_client.get_prefix( - config['etcd']["IMAGE_PREFIX"), value_in_json=True + config['etcd']["IMAGE_PREFIX"], value_in_json=True ) r = { "images": [] @@ -148,7 +148,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = vm_pool.get( - join_path(config['etcd']["VM_PREFIX"), data["uuid"]) + join_path(config['etcd']["VM_PREFIX"], data["uuid"]) ) action = data["action"] @@ -326,7 +326,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']['USER_PREFIX'), + config['etcd']['USER_PREFIX'], data["realm"], data["name"], "key", @@ -430,9 +430,7 @@ class CreateNetwork(Resource): prefix = nb_prefix.available_prefixes.create( data={ - "prefix_length": config['network'][ - "PREFIX_LENGTH"] - ), + "prefix_length": config['network']["PREFIX_LENGTH"], "description": '{}\'s network "{}"'.format( data["name"], data["network_name"] ), diff --git a/ucloud/config.py b/ucloud/config.py index 834d51e..4a067cb 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -25,8 +25,7 @@ config_file = os.path.join(conf_dir, conf_name) config = configparser.ConfigParser() try: - with open(config_file, "r") as conf_fd: - conf.read(conf_fd) + config.read(config_file) except FileNotFoundError: log.warn("Configuration file not found - using defaults") From 0283894ba2eafd4dbb8eace1188d979334695069 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 14:16:22 +0100 Subject: [PATCH 031/163] remove non-unknown vars --- conf/ucloud.conf | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index cf7ff7e..ae33cf5 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -57,10 +57,5 @@ FILE_DIR = /var/lib/ucloud/files FILE_PREFIX = noclue-ahmed [ssh] -SSH_USERNAME = -SSH_PRIVATEKEY = - -# unkown vars: -IMAGE_DIR = -BASE_DIR = -IMAGE_STORE_PREFIX = +SSH_USERNAME = +SSH_PRIVATEKEY = \ No newline at end of file From 8e159c8be134393d08c85b7b0594269877721cc7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 14:20:26 +0100 Subject: [PATCH 032/163] add hacking template --- ucloud/docs/source/hacking.rst | 16 ++++++++++++++++ ucloud/docs/source/index.rst | 1 + 2 files changed, 17 insertions(+) create mode 100644 ucloud/docs/source/hacking.rst diff --git a/ucloud/docs/source/hacking.rst b/ucloud/docs/source/hacking.rst new file mode 100644 index 0000000..976970b --- /dev/null +++ b/ucloud/docs/source/hacking.rst @@ -0,0 +1,16 @@ +Hacking +======= +How to hack on the code. + +[ to be done by Balazs: + +* make nice +* indent with shell script mode + +] + +* git clone the repo +* cd to the repo +* Setup your venv: python -m venv venv +* Run ./bin/ucloud-run-reinstall - it should print you an error + message on how to use ucloud diff --git a/ucloud/docs/source/index.rst b/ucloud/docs/source/index.rst index 879ac32..b31cff3 100644 --- a/ucloud/docs/source/index.rst +++ b/ucloud/docs/source/index.rst @@ -16,6 +16,7 @@ Welcome to ucloud's documentation! admin-guide user-guide/how-to-create-an-os-image-for-ucloud troubleshooting + hacking Indices and tables From e79f1b4de9a82876bdb8736f5828a01b3ce8339e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 14:22:56 +0100 Subject: [PATCH 033/163] ++notes --- ucloud/docs/source/hacking.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/ucloud/docs/source/hacking.rst b/ucloud/docs/source/hacking.rst index 976970b..2df42a7 100644 --- a/ucloud/docs/source/hacking.rst +++ b/ucloud/docs/source/hacking.rst @@ -12,5 +12,6 @@ How to hack on the code. * git clone the repo * cd to the repo * Setup your venv: python -m venv venv +* . ./venv/bin/activate # you need the leading dot for sourcing! * Run ./bin/ucloud-run-reinstall - it should print you an error message on how to use ucloud From 8afd524c55329a32b4d75b352153ce04d1addf4c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 14:55:26 +0100 Subject: [PATCH 034/163] [config] inline etcd3 to get things moving faster - cleanup later --- ucloud/config.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/ucloud/config.py b/ucloud/config.py index 4a067cb..5662e64 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -1,5 +1,3 @@ -from etcd3_wrapper import Etcd3Wrapper - from ucloud.common.host import HostPool from ucloud.common.request import RequestPool from ucloud.common.vm import VmPool @@ -29,6 +27,84 @@ try: except FileNotFoundError: log.warn("Configuration file not found - using defaults") + +################################################################################ +# ETCD3 support +import etcd3 +import json +import queue +import copy +from collections import namedtuple + +PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) + +class EtcdEntry: + # key: str + # value: str + + def __init__(self, meta, value, value_in_json=False): + self.key = meta.key.decode("utf-8") + self.value = value.decode("utf-8") + + if value_in_json: + self.value = json.loads(self.value) + +class Etcd3Wrapper: + def __init__(self, *args, **kwargs): + self.client = etcd3.client(*args, **kwargs) + + def get(self, *args, value_in_json=False, **kwargs): + _value, _key = self.client.get(*args, **kwargs) + if _key is None or _value is None: + return None + return EtcdEntry(_key, _value, value_in_json=value_in_json) + + def put(self, *args, value_in_json=False, **kwargs): + _key, _value = args + if value_in_json: + _value = json.dumps(_value) + + if not isinstance(_key, str): + _key = _key.decode("utf-8") + + return self.client.put(_key, _value, **kwargs) + + def get_prefix(self, *args, value_in_json=False, **kwargs): + r = self.client.get_prefix(*args, **kwargs) + for entry in r: + e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) + if e.value: + yield e + + def watch_prefix(self, key, timeout=0, value_in_json=False): + timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"), + value=str.encode(json.dumps({"status": "TIMEOUT", + "type": "TIMEOUT"})), + value_in_json=value_in_json) + + event_queue = queue.Queue() + + def add_event_to_queue(event): + for e in event.events: + if e.value: + event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json)) + + self.client.add_watch_prefix_callback(key, add_event_to_queue) + + while True: + try: + while True: + v = event_queue.get(timeout=timeout) + yield v + except queue.Empty: + event_queue.put(copy.deepcopy(timeout_event)) + + +class PsuedoEtcdEntry(EtcdEntry): + def __init__(self, key, value, value_in_json=False): + super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json) + + etcd_wrapper_args = () etcd_wrapper_kwargs = { 'host': config['etcd']['ETCD_URL'], From f919719b1e2686c85c01b9f7b4fee4aec5baea5c Mon Sep 17 00:00:00 2001 From: Ahmed Bilal <49-ahmedbilal@users.noreply.code.ungleich.ch> Date: Sun, 8 Dec 2019 15:03:49 +0100 Subject: [PATCH 035/163] Update ucloud.conf --- conf/ucloud.conf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index ae33cf5..a98250a 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -25,15 +25,15 @@ NETBOX_TOKEN = replace me ETCD_URL = localhost ETCD_PORT = 2379 -CA_CERT = changeme -CERT_CERT = changeme -CERT_KEY = changeme +CA_CERT +CERT_CERT +CERT_KEY FILE_PREFIX = files HOST_PREFIx = hosts IMAGE_PREFIX = images -IMAGE_STORE_PREFIX = imagestore/ +IMAGE_STORE_PREFIX = imagestore NETWORK_PREFIX = networks REQUEST_PREFIX = requests @@ -54,7 +54,6 @@ CEPH_IMG_POOL = # Importing uploaded files FILE_DIR = /var/lib/ucloud/files -FILE_PREFIX = noclue-ahmed [ssh] SSH_USERNAME = From 71279a968f649f203f6b5577f495594176ca818d Mon Sep 17 00:00:00 2001 From: meow Date: Sat, 14 Dec 2019 20:23:31 +0500 Subject: [PATCH 036/163] Fix issues in naming and few other things --- conf/ucloud.conf | 75 ++++++++-------- scripts/ucloud | 14 ++- setup.py | 9 +- ucloud/api/common_fields.py | 2 +- ucloud/api/create_image_store.py | 2 +- ucloud/api/helper.py | 21 +++-- ucloud/api/main.py | 93 +++++++++++--------- ucloud/api/schemas.py | 13 +-- ucloud/common/classes.py | 2 +- ucloud/common/counters.py | 2 +- ucloud/common/etcd_wrapper.py | 74 ++++++++++++++++ ucloud/common/helpers.py | 20 +---- ucloud/common/request.py | 3 +- ucloud/config.py | 144 +++++++------------------------ ucloud/filescanner/main.py | 24 +++--- ucloud/host/main.py | 19 ++-- ucloud/host/virtualmachine.py | 14 +-- ucloud/imagescanner/main.py | 11 +-- ucloud/metadata/main.py | 3 +- ucloud/scheduler/helper.py | 2 +- ucloud/scheduler/main.py | 8 +- 21 files changed, 274 insertions(+), 281 deletions(-) create mode 100644 ucloud/common/etcd_wrapper.py diff --git a/conf/ucloud.conf b/conf/ucloud.conf index a98250a..222cc4f 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -1,60 +1,53 @@ -# This section contains default values for all other sections -# -[DEFAULT] - -AUTH_NAME = "replace me" -AUTH_SEED = "replace me" -AUTH_REALM = "replace me" - -NETWORK_PREFIX = moo - -OTP_VERIFY_ENDPOINT = verify/ - -[api] -NETWORK_PREFIX = foo +[otp] +server = https://otp.ungleich.ch/ungleichotp/ +verify_endpoint = verify/ +auth_name = replace_me +auth_realm = replace_me +auth_seed = replace_me [network] -PREFIX_LENGTH = 64 -PREFIX = 2001:db8::/48 +prefix_length = 64 +prefix = 2001:db8::/48 +vxlan_phy_dev = eno1 [netbox] -NETBOX_URL = https://replace-me.example.com -NETBOX_TOKEN = replace me +url = https://replace-me.example.com +token = replace_me [etcd] -ETCD_URL = localhost -ETCD_PORT = 2379 +url = localhost +port = 2379 -CA_CERT -CERT_CERT -CERT_KEY +ca_cert +cert_cert +cert_key - -FILE_PREFIX = files -HOST_PREFIx = hosts -IMAGE_PREFIX = images -IMAGE_STORE_PREFIX = imagestore - -NETWORK_PREFIX = networks -REQUEST_PREFIX = requests -USER_PREFIX = users -VM_PREFIX = vms +file_prefix = /files/ +host_prefix = /hosts/ +image_prefix = /images/ +image_store_prefix = /imagestore/ +network_prefix = /networks/ +request_prefix = /requests/ +user_prefix = /users/ +vm_prefix = /vms/ [storage] + #values = filesystem, ceph -STORAGE_BACKEND = +backend = filesystem # if STORAGE_BACKEND = filesystem -VM_DIR = -IMG_DIR = +vm_dir = /var/lib/ucloud/vms +image_dir = /var/lib/ucloud/images # if STORAGE_BACKEND = ceph -CEPH_VM_POOL = -CEPH_IMG_POOL = +ceph_vm_pool = ssd +ceph_image_pool = ssd # Importing uploaded files -FILE_DIR = /var/lib/ucloud/files +file_dir = /var/lib/ucloud/files +# For Migrating VMs over ssh/tcp [ssh] -SSH_USERNAME = -SSH_PRIVATEKEY = \ No newline at end of file +username +private_key_path \ No newline at end of file diff --git a/scripts/ucloud b/scripts/ucloud index 0780fb4..2da1094 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -5,11 +5,17 @@ import logging import importlib import sys import os +import multiprocessing as mp + COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata'] if __name__ == "__main__": - log = logging.getLogger("ucloud") + logging.basicConfig(level=logging.DEBUG, + format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s', + filename='/var/log/ucloud.log', filemode='a') + + logger = logging.getLogger("ucloud") arg_parser = argparse.ArgumentParser(prog='ucloud', description='Open Source Cloud Management Software') @@ -22,12 +28,12 @@ if __name__ == "__main__": os.environ['UCLOUD_CONF_DIR'] = args.conf_dir try: + mp.set_start_method('spawn') name = args.component mod = importlib.import_module("ucloud.{}.main".format(name)) main = getattr(mod, "main") - - main() + main(*args.component_args) except Exception as e: - logging.exception(e) + logger.exception(e) print(e) diff --git a/setup.py b/setup.py index 14dffb7..a7ddbe6 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ try: version = ucloud.version.VERSION except: import subprocess - c = subprocess.run(["git", "describe"], capture_output=True) - version = c.stdout.decode("utf-8") + c = subprocess.check_output(['git', 'describe']) + version = c.decode("utf-8").strip() setup(name='ucloud', @@ -28,8 +28,7 @@ setup(name='ucloud', packages=find_packages(), install_requires=[ 'requests', - 'python-decouple', - 'flask', + 'Flask', 'flask-restful', 'bitmath', 'pyotp', @@ -37,8 +36,8 @@ setup(name='ucloud', 'sphinx', 'pynetbox', 'sphinx-rtd-theme', - 'etcd3_wrapper @ https://code.ungleich.ch/ungleich-public/etcd3_wrapper/repository/master/archive.tar.gz#egg=etcd3_wrapper', 'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3', ], scripts=['scripts/ucloud'], + data_files=[('/etc/ucloud/', ['conf/ucloud.conf'])], zip_safe=False) diff --git a/ucloud/api/common_fields.py b/ucloud/api/common_fields.py index 1ceb1b0..cf86283 100755 --- a/ucloud/api/common_fields.py +++ b/ucloud/api/common_fields.py @@ -47,6 +47,6 @@ class VmUUIDField(Field): self.validation = self.vm_uuid_validation def vm_uuid_validation(self): - r = etcd_client.get(os.path.join(config['api']['VM_PREFIX'], self.uuid)) + r = etcd_client.get(os.path.join(config['etcd']['vm_prefix'], self.uuid)) if not r: self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/ucloud/api/create_image_store.py b/ucloud/api/create_image_store.py index 259b9c8..9023bd6 100755 --- a/ucloud/api/create_image_store.py +++ b/ucloud/api/create_image_store.py @@ -13,4 +13,4 @@ data = { "attributes": {"list": [], "key": [], "pool": "images"}, } -etcd_client.put(os.path.join(config['api']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data)) +etcd_client.put(os.path.join(config['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index 6735f05..2dfb7de 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -2,7 +2,7 @@ import binascii import ipaddress import random import subprocess as sp - +import logging import requests from pyotp import TOTP @@ -10,23 +10,28 @@ from pyotp import TOTP from ucloud.config import vm_pool, config +logger = logging.getLogger("ucloud.api.helper") + def check_otp(name, realm, token): try: data = { - "auth_name": config['api']["AUTH_NAME"], - "auth_token": TOTP(config['api']["AUTH_SEED"]).now(), - "auth_realm": config['api']["AUTH_REALM"], + "auth_name": config['otp']['auth_name'], + "auth_token": TOTP(config['otp']['auth_seed']).now(), + "auth_realm": config['otp']['auth_realm'], "name": name, "realm": realm, "token": token, } - except binascii.Error: + except binascii.Error as err: + logger.error( + "Cannot compute OTP for seed: {}".format(config['otp']['auth_seed']) + ) return 400 response = requests.post( "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( - OTP_SERVER=config['api']["OTP_SERVER"], - OTP_VERIFY_ENDPOINT=config['api']["OTP_VERIFY_ENDPOINT"] + OTP_SERVER=config['otp']['server'], + OTP_VERIFY_ENDPOINT=config['otp']['verify_endpoint'] ), json=data, ) @@ -80,7 +85,7 @@ def resolve_image_name(name, etcd_client): except Exception: raise ValueError("Image name not in correct format i.e {store_name}:{image_name}") - images = etcd_client.get_prefix(config['api']['IMAGE_PREFIX'], value_in_json=True) + images = etcd_client.get_prefix(config['etcd']['image_prefix'], value_in_json=True) # Try to find image with name == image_name and store_name == store_name try: diff --git a/ucloud/api/main.py b/ucloud/api/main.py index bbda7e9..0e202d8 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -10,7 +10,10 @@ from flask_restful import Resource, Api from ucloud.common import counters from ucloud.common.vm import VMStatus from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, config, image_storage_handler) +from ucloud.config import ( + etcd_client, request_pool, vm_pool, + host_pool, config, image_storage_handler +) from . import schemas from .helper import generate_mac, mac2ipv6 from . import logger @@ -28,7 +31,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(config['etcd']["VM_PREFIX"], vm_uuid) + vm_key = join_path(config['etcd']['vm_prefix'], vm_uuid) specs = { "cpu": validator.specs["cpu"], "ram": validator.specs["ram"], @@ -56,7 +59,7 @@ class CreateVM(Resource): # Create ScheduleVM Request r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=config['etcd']["REQUEST_PREFIX"] + request_prefix=config['etcd']['request_prefix'] ) request_pool.put(r) @@ -71,7 +74,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = vm_pool.get( - join_path(config['etcd']["VM_PREFIX"], data["uuid"]) + join_path(config['etcd']['vm_prefix'], data["uuid"]) ) vm_value = vm.value.copy() vm_value["ip"] = [] @@ -79,7 +82,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = etcd_client.get( join_path( - config['etcd']["NETWORK_PREFIX"], + config['etcd']['network_prefix'], data["name"], network_name, ), @@ -100,7 +103,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = etcd_client.get( - join_path(config['etcd']["FILE_PREFIX"], data["uuid"]) + join_path(config['etcd']['file_prefix'], data["uuid"]) ) file_entry_value = json.loads(file_entry.value) @@ -113,7 +116,7 @@ class CreateImage(Resource): "visibility": "public", } etcd_client.put( - join_path(config['etcd']["IMAGE_PREFIX"], data["uuid"]), + join_path(config['etcd']['image_prefix'], data["uuid"]), json.dumps(image_entry_json), ) @@ -125,7 +128,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = etcd_client.get_prefix( - config['etcd']["IMAGE_PREFIX"], value_in_json=True + config['etcd']['image_prefix'], value_in_json=True ) r = { "images": [] @@ -148,7 +151,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = vm_pool.get( - join_path(config['etcd']["VM_PREFIX"], data["uuid"]) + join_path(config['etcd']['vm_prefix'], data["uuid"]) ) action = data["action"] @@ -172,7 +175,7 @@ class VMAction(Resource): type="{}VM".format(action.title()), uuid=data["uuid"], hostname=vm_entry.hostname, - request_prefix=config['etcd']["REQUEST_PREFIX"] + request_prefix=config['etcd']['request_prefix'] ) request_pool.put(r) return {"message": "VM {} Queued".format(action.title())}, 200 @@ -193,10 +196,10 @@ class VMMigration(Resource): type=RequestType.ScheduleVM, uuid=vm.uuid, destination=join_path( - config['etcd']["HOST_PREFIX"], validator.destination.value + config['etcd']['host_prefix'], validator.destination.value ), migration=True, - request_prefix=config['etcd']["REQUEST_PREFIX"] + request_prefix=config['etcd']['request_prefix'] ) request_pool.put(r) return {"message": "VM Migration Initialization Queued"}, 200 @@ -212,7 +215,7 @@ class ListUserVM(Resource): if validator.is_valid(): vms = etcd_client.get_prefix( - config['etcd']["VM_PREFIX"], value_in_json=True + config['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) @@ -246,7 +249,7 @@ class ListUserFiles(Resource): if validator.is_valid(): files = etcd_client.get_prefix( - config['etcd']["FILE_PREFIX"], value_in_json=True + config['etcd']['file_prefix'], value_in_json=True ) return_files = [] user_files = list( @@ -270,7 +273,7 @@ class CreateHost(Resource): data = request.json validator = schemas.CreateHostSchema(data) if validator.is_valid(): - host_key = join_path(config['etcd']["HOST_PREFIX"], uuid4().hex) + host_key = join_path(config['etcd']['host_prefix'], uuid4().hex) host_entry = { "specs": data["specs"], "hostname": data["hostname"], @@ -309,7 +312,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - config['etcd']['USER_PREFIX'], + config['etcd']['user_prefix'], data["realm"], data["name"], "key", @@ -326,7 +329,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']['USER_PREFIX'], + config['etcd']['user_prefix'], data["realm"], data["name"], "key", @@ -355,7 +358,7 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']["USER_PREFIX"], + config['etcd']['user_prefix'], data["realm"], data["name"], "key", @@ -385,7 +388,7 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']["USER_PREFIX"], + config['etcd']['user_prefix'], data["realm"], data["name"], "key", @@ -420,31 +423,35 @@ class CreateNetwork(Resource): "type": data["type"], } if validator.user.value: - nb = pynetbox.api( - url=config['netbox']["NETBOX_URL"], - token=config['netbox']["NETBOX_TOKEN"], - ) - nb_prefix = nb.ipam.prefixes.get( - prefix=config['network']["PREFIX"] - ) - - prefix = nb_prefix.available_prefixes.create( - data={ - "prefix_length": config['network']["PREFIX_LENGTH"], - "description": '{}\'s network "{}"'.format( - data["name"], data["network_name"] - ), - "is_pool": True, - } - ) - network_entry["ipv6"] = prefix["prefix"] + try: + nb = pynetbox.api( + url=config['netbox']['url'], + token=config['netbox']['token'], + ) + nb_prefix = nb.ipam.prefixes.get( + prefix=config['network']['prefix'] + ) + prefix = nb_prefix.available_prefixes.create( + data={ + "prefix_length": int(config['network']['prefix_length']), + "description": '{}\'s network "{}"'.format( + data["name"], data["network_name"] + ), + "is_pool": True, + } + ) + except Exception: + logger.exception("Exception occur while contacting netbox") + return {"message": "Error occured while creating network."} + else: + network_entry["ipv6"] = prefix["prefix"] else: network_entry["ipv6"] = "fd00::/64" network_key = join_path( - config['network']["NETWORK_PREFIX"], - data["name"], - data["network_name"], + config['etcd']['network_prefix'], + data['name'], + data['network_name'], ) etcd_client.put(network_key, network_entry, value_in_json=True) return {"message": "Network successfully added."} @@ -460,7 +467,7 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - config['network']["NETWORK_PREFIX"], data["name"] + config['etcd']['network_prefix'], data["name"] ) networks = etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] @@ -496,7 +503,7 @@ api.add_resource(CreateNetwork, "/network/create") def main(): - image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True)) + image_stores = list(etcd_client.get_prefix(config['etcd']['image_store_prefix'], value_in_json=True)) if len(image_stores) == 0: data = { "is_public": True, @@ -506,7 +513,7 @@ def main(): "attributes": {"list": [], "key": [], "pool": "images"}, } - etcd_client.put(join_path(config['etcd']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data)) + etcd_client.put(join_path(config['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) app.run(host="::", debug=True) diff --git a/ucloud/api/schemas.py b/ucloud/api/schemas.py index 23db184..a3e0aa8 100755 --- a/ucloud/api/schemas.py +++ b/ucloud/api/schemas.py @@ -22,7 +22,7 @@ import bitmath from ucloud.common.host import HostStatus from ucloud.common.vm import VMStatus from ucloud.config import etcd_client, config, vm_pool, host_pool -from . import helper +from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -102,14 +102,14 @@ class CreateImageSchema(BaseSchema): super().__init__(data, fields) def file_uuid_validation(self): - file_entry = etcd_client.get(os.path.join(config['etcd']['FILE_PREFIX'], self.uuid.value)) + file_entry = etcd_client.get(os.path.join(config['etcd']['file_prefix'], self.uuid.value)) if file_entry is None: self.add_error( "Image File with uuid '{}' Not Found".format(self.uuid.value) ) def image_store_name_validation(self): - image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'])) + image_stores = list(etcd_client.get_prefix(config['etcd']['image_store_prefix'])) image_store = next( filter( @@ -220,6 +220,7 @@ class CreateVMSchema(OTPSchema): try: image_uuid = helper.resolve_image_name(self.image.value, etcd_client) except Exception as e: + logger.exception("Cannot resolve image name = %s", self.image.value) self.add_error(str(e)) else: self.image_uuid = image_uuid @@ -235,7 +236,7 @@ class CreateVMSchema(OTPSchema): if _network: for net in _network: - network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'], + network = etcd_client.get(os.path.join(config['etcd']['network_prefix'], self.name.value, net), value_in_json=True) if not network: @@ -400,7 +401,7 @@ class VmMigrationSchema(OTPSchema): if vm.status != VMStatus.running: self.add_error("Can't migrate non-running VM") - if vm.hostname == os.path.join(config['etcd']['HOST_PREFIX'], self.destination.value): + if vm.hostname == os.path.join(config['etcd']['host_prefix'], self.destination.value): self.add_error("Destination host couldn't be same as Source Host") @@ -442,7 +443,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'], + network = etcd_client.get(os.path.join(config['etcd']['network_prefix'], self.name.value, self.network_name.value), value_in_json=True) diff --git a/ucloud/common/classes.py b/ucloud/common/classes.py index 2eae809..29dffd4 100644 --- a/ucloud/common/classes.py +++ b/ucloud/common/classes.py @@ -1,4 +1,4 @@ -from etcd3_wrapper import EtcdEntry +from .etcd_wrapper import EtcdEntry class SpecificEtcdEntryBase: diff --git a/ucloud/common/counters.py b/ucloud/common/counters.py index 066a870..2d4a8e9 100644 --- a/ucloud/common/counters.py +++ b/ucloud/common/counters.py @@ -1,4 +1,4 @@ -from etcd3_wrapper import Etcd3Wrapper +from .etcd_wrapper import Etcd3Wrapper def increment_etcd_counter(etcd_client: Etcd3Wrapper, key): diff --git a/ucloud/common/etcd_wrapper.py b/ucloud/common/etcd_wrapper.py new file mode 100644 index 0000000..a3fb83f --- /dev/null +++ b/ucloud/common/etcd_wrapper.py @@ -0,0 +1,74 @@ +import etcd3 +import json +import queue +import copy + +from collections import namedtuple + +PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) + +class EtcdEntry: + # key: str + # value: str + + def __init__(self, meta, value, value_in_json=False): + self.key = meta.key.decode("utf-8") + self.value = value.decode("utf-8") + + if value_in_json: + self.value = json.loads(self.value) + +class Etcd3Wrapper: + def __init__(self, *args, **kwargs): + self.client = etcd3.client(*args, **kwargs) + + def get(self, *args, value_in_json=False, **kwargs): + _value, _key = self.client.get(*args, **kwargs) + if _key is None or _value is None: + return None + return EtcdEntry(_key, _value, value_in_json=value_in_json) + + def put(self, *args, value_in_json=False, **kwargs): + _key, _value = args + if value_in_json: + _value = json.dumps(_value) + + if not isinstance(_key, str): + _key = _key.decode("utf-8") + + return self.client.put(_key, _value, **kwargs) + + def get_prefix(self, *args, value_in_json=False, **kwargs): + r = self.client.get_prefix(*args, **kwargs) + for entry in r: + e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) + if e.value: + yield e + + def watch_prefix(self, key, timeout=0, value_in_json=False): + timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"), + value=str.encode(json.dumps({"status": "TIMEOUT", + "type": "TIMEOUT"})), + value_in_json=value_in_json) + + event_queue = queue.Queue() + + def add_event_to_queue(event): + for e in event.events: + if e.value: + event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json)) + + self.client.add_watch_prefix_callback(key, add_event_to_queue) + + while True: + try: + while True: + v = event_queue.get(timeout=timeout) + yield v + except queue.Empty: + event_queue.put(copy.deepcopy(timeout_event)) + + +class PsuedoEtcdEntry(EtcdEntry): + def __init__(self, key, value, value_in_json=False): + super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json) diff --git a/ucloud/common/helpers.py b/ucloud/common/helpers.py index 1bdf0b4..501aa90 100644 --- a/ucloud/common/helpers.py +++ b/ucloud/common/helpers.py @@ -6,21 +6,7 @@ import json from ipaddress import ip_address from os.path import join as join_path - - -def create_package_loggers(packages, base_path, mode="a"): - loggers = {} - for pkg in packages: - logger = logging.getLogger(pkg) - logger_handler = logging.FileHandler( - join_path(base_path, "{}.txt".format(pkg)), - mode=mode - ) - logger.setLevel(logging.DEBUG) - logger_handler.setFormatter(logging.Formatter(fmt="%(asctime)s: %(levelname)s - %(message)s", - datefmt="%d-%b-%y %H:%M:%S")) - logger.addHandler(logger_handler) - loggers[pkg] = logger +from . import logger # TODO: Should be removed as soon as migration @@ -35,7 +21,7 @@ def get_ipv4_address(): except socket.timeout: address = "127.0.0.1" except Exception as e: - logging.getLogger().exception(e) + logger.exception(e) address = "127.0.0.1" else: address = s.getsockname()[0] @@ -49,6 +35,6 @@ def get_ipv6_address(): content = json.loads(r.content.decode("utf-8")) ip = ip_address(content["ip"]).exploded except Exception as e: - logging.exception(e) + logger.exception(e) else: return ip diff --git a/ucloud/common/request.py b/ucloud/common/request.py index cadac80..1e4594d 100644 --- a/ucloud/common/request.py +++ b/ucloud/common/request.py @@ -2,8 +2,7 @@ import json from os.path import join from uuid import uuid4 -from etcd3_wrapper.etcd3_wrapper import PsuedoEtcdEntry - +from .etcd_wrapper import PsuedoEtcdEntry from .classes import SpecificEtcdEntryBase diff --git a/ucloud/config.py b/ucloud/config.py index 5662e64..3eee897 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -1,133 +1,53 @@ +import configparser +import os +import logging + from ucloud.common.host import HostPool from ucloud.common.request import RequestPool from ucloud.common.vm import VmPool from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler +from ucloud.common.etcd_wrapper import Etcd3Wrapper -# Replacing decouple inline -import configparser -import os -import os.path - -import logging -log = logging.getLogger("ucloud.config") - -conf_name = "ucloud.conf" - -try: - conf_dir = os.environ["UCLOUD_CONF_DIR"] -except KeyError: - conf_dir = "/etc/ucloud" +log = logging.getLogger('ucloud.config') +conf_name = 'ucloud.conf' +conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud') config_file = os.path.join(conf_dir, conf_name) -config = configparser.ConfigParser() +config = configparser.ConfigParser(allow_no_value=True) -try: +if os.access(config_file, os.R_OK): config.read(config_file) -except FileNotFoundError: - log.warn("Configuration file not found - using defaults") - - -################################################################################ -# ETCD3 support -import etcd3 -import json -import queue -import copy -from collections import namedtuple - -PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) - -class EtcdEntry: - # key: str - # value: str - - def __init__(self, meta, value, value_in_json=False): - self.key = meta.key.decode("utf-8") - self.value = value.decode("utf-8") - - if value_in_json: - self.value = json.loads(self.value) - -class Etcd3Wrapper: - def __init__(self, *args, **kwargs): - self.client = etcd3.client(*args, **kwargs) - - def get(self, *args, value_in_json=False, **kwargs): - _value, _key = self.client.get(*args, **kwargs) - if _key is None or _value is None: - return None - return EtcdEntry(_key, _value, value_in_json=value_in_json) - - def put(self, *args, value_in_json=False, **kwargs): - _key, _value = args - if value_in_json: - _value = json.dumps(_value) - - if not isinstance(_key, str): - _key = _key.decode("utf-8") - - return self.client.put(_key, _value, **kwargs) - - def get_prefix(self, *args, value_in_json=False, **kwargs): - r = self.client.get_prefix(*args, **kwargs) - for entry in r: - e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) - if e.value: - yield e - - def watch_prefix(self, key, timeout=0, value_in_json=False): - timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"), - value=str.encode(json.dumps({"status": "TIMEOUT", - "type": "TIMEOUT"})), - value_in_json=value_in_json) - - event_queue = queue.Queue() - - def add_event_to_queue(event): - for e in event.events: - if e.value: - event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json)) - - self.client.add_watch_prefix_callback(key, add_event_to_queue) - - while True: - try: - while True: - v = event_queue.get(timeout=timeout) - yield v - except queue.Empty: - event_queue.put(copy.deepcopy(timeout_event)) - - -class PsuedoEtcdEntry(EtcdEntry): - def __init__(self, key, value, value_in_json=False): - super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json) - +else: + log.warning('Configuration file not found - using defaults') etcd_wrapper_args = () etcd_wrapper_kwargs = { - 'host': config['etcd']['ETCD_URL'], - 'port': config['etcd']['ETCD_PORT'], - 'ca_cert': config['etcd']['CA_CERT'], - 'cert_cert': config['etcd']['CERT_CERT'], - 'cert_key': config['etcd']['CERT_KEY'] + 'host': config['etcd']['url'], + 'port': config['etcd']['port'], + 'ca_cert': config['etcd']['ca_cert'], + 'cert_cert': config['etcd']['cert_cert'], + 'cert_key': config['etcd']['cert_key'] } etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) -host_pool = HostPool(etcd_client, config['etcd']['HOST_PREFIX']) -vm_pool = VmPool(etcd_client, config['etcd']['VM_PREFIX']) -request_pool = RequestPool(etcd_client, config['etcd']['REQUEST_PREFIX']) +host_pool = HostPool(etcd_client, config['etcd']['host_prefix']) +vm_pool = VmPool(etcd_client, config['etcd']['vm_prefix']) +request_pool = RequestPool(etcd_client, config['etcd']['request_prefix']) running_vms = [] -__storage_backend = config['storage']["STORAGE_BACKEND"] -if __storage_backend == "filesystem": - image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=config['storage']["VM_DIR"], - image_base=config['storage']["IMAGE_DIR"]) -elif __storage_backend == "ceph": - image_storage_handler = CEPHBasedImageStorageHandler(vm_base=config['storage']["CEPH_VM_POOL"], - image_base=config['storage']["CEPH_IMAGE_POOL"]) +__storage_backend = config['storage']['backend'] +if __storage_backend == 'filesystem': + image_storage_handler = FileSystemBasedImageStorageHandler( + vm_base=config['storage']['vm_dir'], + image_base=config['storage']['image_dir'] + ) +elif __storage_backend == 'ceph': + image_storage_handler = CEPHBasedImageStorageHandler( + vm_base=config['storage']['ceph_vm_pool'], + image_base=config['storage']['ceph_image_pool'] + ) else: - raise Exception("Unknown Image Storage Handler") + raise Exception('Unknown Image Storage Handler') diff --git a/ucloud/filescanner/main.py b/ucloud/filescanner/main.py index ef6fee6..265f9d9 100755 --- a/ucloud/filescanner/main.py +++ b/ucloud/filescanner/main.py @@ -3,6 +3,7 @@ import os import pathlib import subprocess as sp import time +import sys from uuid import uuid4 from . import logger @@ -19,7 +20,6 @@ def getxattr(file, attr): '--absolute-names'], stderr=sp.DEVNULL) value = value.decode("utf-8") except sp.CalledProcessError as e: - logger.exception(e) value = None return value @@ -63,14 +63,13 @@ try: sp.check_output(['which', 'getfattr']) sp.check_output(['which', 'setfattr']) except Exception as e: - logger.exception(e) - print('Make sure you have getfattr and setfattr available') - exit(1) + logger.error("You don't seems to have both getfattr and setfattr") + sys.exit(1) def main(): - BASE_DIR = config['storage']["FILE_DIR"] - FILE_PREFIX = config['storage']["FILE_PREFIX"] + BASE_DIR = config['storage']['file_dir'] + FILE_PREFIX = config['etcd']['file_prefix'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob("{}/**".format(BASE_DIR), recursive=True) @@ -79,7 +78,7 @@ def main(): files = list(filter(os.path.isfile, files)) untracked_files = list( - filter(lambda f: not bool(getxattr(f, "user.utracked")), files) + filter(lambda f: not bool(getxattr(f, "utracked")), files) ) tracked_files = list( @@ -89,7 +88,8 @@ def main(): file_id = uuid4() # Get Username - owner = pathlib.Path(file).parts[3] + owner = pathlib.Path(file).parts[len(pathlib.Path(BASE_DIR).parts)] + # Get Creation Date of File # Here, we are assuming that ctime is creation time # which is mostly not true. @@ -101,9 +101,7 @@ def main(): # Compute sha512 sum sha_sum = sha512sum(file) - # File Path excluding base and username - file_path = pathlib.Path(file).parts[4:] - file_path = os.path.join(*file_path) + file_path = pathlib.Path(file).parts[-1] # Create Entry entry_key = os.path.join(FILE_PREFIX, str(file_id)) @@ -115,10 +113,10 @@ def main(): "size": size } - print("Tracking {}".format(file)) + logger.info("Tracking %s", file) # Insert Entry etcd_client.put(entry_key, entry_value, value_in_json=True) - setxattr(file, "user.utracked", True) + setxattr(file, "utracked", True) if __name__ == "__main__": diff --git a/ucloud/host/main.py b/ucloud/host/main.py index 0ca345b..bd03a08 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -1,9 +1,10 @@ import argparse import multiprocessing as mp import time +import sys -from etcd3_wrapper import Etcd3Wrapper - +from os.path import isdir +from ucloud.common.etcd_wrapper import Etcd3Wrapper from ucloud.common.request import RequestEntry, RequestType from ucloud.config import (vm_pool, request_pool, etcd_client, running_vms, @@ -18,7 +19,7 @@ from ucloud.host import logger def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) - host_pool = HostPool(client, config['etcd']['HOST_PREFIX']) + host_pool = HostPool(client, config['etcd']['host_prefix']) this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) while True: @@ -72,9 +73,11 @@ def maintenance(host): running_vms.remove(_vm) def check(): - if config['etcd']['STORAGE_BACKEND'] == 'filesystem' and not isdir(config['etcd']['VM_DIR']): + if config['storage']['backend'] == 'filesystem' and \ + not isdir(config['storage']['vm_dir']): + print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" - " in .env file must exists. But, it don't.") + " in /etc/ucloud/ucloud.conf file must exists. But, it don't.") sys.exit(1) @@ -84,7 +87,7 @@ def main(hostname): heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) - host_pool = HostPool(etcd_client, config['etcd']['HOST_PREFIX']) + host_pool = HostPool(etcd_client, config['etcd']['host_prefix']) host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) assert host is not None, "No such host with name = {}".format(hostname) @@ -106,8 +109,8 @@ def main(hostname): # beat updating mechanism in separated thread for events_iterator in [ - etcd_client.get_prefix(config['etcd']['REQUEST_PREFIX'], value_in_json=True), - etcd_client.watch_prefix(config['etcd']['REQUEST_PREFIX'], timeout=10, value_in_json=True), + etcd_client.get_prefix(config['etcd']['request_prefix'], value_in_json=True), + etcd_client.watch_prefix(config['etcd']['request_prefix'], timeout=10, value_in_json=True), ]: for request_event in events_iterator: request_event = RequestEntry(request_event) diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index 33e4433..4a7584a 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -46,7 +46,7 @@ def delete_network_interface(iface): def resolve_network(network_name, network_owner): - network = etcd_client.get(join_path(config['etcd']["NETWORK_PREFIX"], + network = etcd_client.get(join_path(config['etcd']['network_prefix'], network_owner, network_name), value_in_json=True) @@ -179,7 +179,7 @@ def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, mi for network_mac_and_tap in vm_networks: network_name, mac, tap = network_mac_and_tap - _key = os.path.join(config['etcd']['NETWORK_PREFIX'], vm_entry.owner, network_name) + _key = os.path.join(config['etcd']['network_prefix'], vm_entry.owner, network_name) network = etcd_client.get(_key, value_in_json=True) network_type = network.value["type"] network_id = str(network.value["id"]) @@ -187,7 +187,7 @@ def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, mi if network_type == "vxlan": tap = create_vxlan_br_tap(_id=network_id, - _dev=config['etcd']["VXLAN_PHY_DEV"], + _dev=config['network']['vxlan_phy_dev'], tap_id=tap, ip=network_ipv6) update_radvd_conf(etcd_client) @@ -303,13 +303,13 @@ def transfer(request_event): _host, _port = request_event.parameters["host"], request_event.parameters["port"] _uuid = request_event.uuid _destination = request_event.destination_host_key - vm = get_vm(running_vms, join_path(config['etcd']['VM_PREFIX'], _uuid)) + vm = get_vm(running_vms, join_path(config['etcd']['vm_prefix'], _uuid)) if vm: tunnel = sshtunnel.SSHTunnelForwarder( _host, - ssh_username=config['ssh']["ssh_username"], - ssh_pkey=config['ssh']["SSH_PRIVATEKEY"], + ssh_username=config['ssh']['username'], + ssh_pkey=config['ssh']['private_key_path'], remote_bind_address=("127.0.0.1", _port), ssh_proxy_enabled=True, ssh_proxy=(_host, 22) @@ -373,7 +373,7 @@ def launch_vm(vm_entry, migration=False, migration_port=None, destination_host_k parameters={"host": get_ipv6_address(), "port": migration_port}, uuid=vm_entry.uuid, destination_host_key=destination_host_key, - request_prefix=config['etcd']["REQUEST_PREFIX"] + request_prefix=config['etcd']['request_prefix'] ) request_pool.put(r) else: diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index df4dfad..2658641 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -20,9 +20,9 @@ def qemu_img_type(path): def check(): """ check whether settings are sane, refuse to start if they aren't """ - if config['etcd']['STORAGE_BACKEND'] == 'filesystem' and not isdir(config['etcd']['IMAGE_DIR']): + if config['storage']['backend'] == 'filesystem' and not isdir(config['storage']['image_dir']): print("You have set STORAGE_BACKEND to filesystem, but " - "{} does not exist. Refusing to start".format(config['etcd']['IMAGE_DIR'])) + "{} does not exist. Refusing to start".format(config['storage']['image_dir'])) sys.exit(1) try: @@ -34,7 +34,7 @@ def check(): def main(): # We want to get images entries that requests images to be created - images = etcd_client.get_prefix(config['etcd']['IMAGE_PREFIX'], value_in_json=True) + images = etcd_client.get_prefix(config['etcd']['image_prefix'], value_in_json=True) images_to_be_created = list(filter(lambda im: im.value['status'] == 'TO_BE_CREATED', images)) for image in images_to_be_created: @@ -43,9 +43,10 @@ def main(): image_owner = image.value['owner'] image_filename = image.value['filename'] image_store_name = image.value['store_name'] - image_full_path = join_path(config['etcd']['BASE_DIR'], image_owner, image_filename) + image_full_path = join_path(config['storage']['file_dir'], image_owner, image_filename) - image_stores = etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True) + image_stores = etcd_client.get_prefix(config['etcd']['image_store_prefix'], + value_in_json=True) user_image_store = next(filter( lambda s, store_name=image_store_name: s.value["name"] == store_name, image_stores diff --git a/ucloud/metadata/main.py b/ucloud/metadata/main.py index 9281d7c..16b7c6d 100644 --- a/ucloud/metadata/main.py +++ b/ucloud/metadata/main.py @@ -43,7 +43,8 @@ class Root(Resource): if not data: return {'message': 'Metadata for such VM does not exists.'}, 404 else: - etcd_key = os.path.join(config['etcd']['USER_PREFIX'], data.value['owner_realm'], + etcd_key = os.path.join(config['etcd']['user_prefix'], + data.value['owner_realm'], data.value['owner'], 'key') etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True) user_personal_ssh_keys = [key.value for key in etcd_entry] diff --git a/ucloud/scheduler/helper.py b/ucloud/scheduler/helper.py index 1754045..560bdbc 100755 --- a/ucloud/scheduler/helper.py +++ b/ucloud/scheduler/helper.py @@ -106,7 +106,7 @@ def assign_host(vm): r = RequestEntry.from_scratch(type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=config['etcd']['REQUEST_PREFIX']) + request_prefix=config['etcd']['request_prefix']) request_pool.put(r) vm.log.append("VM scheduled for starting") diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index 33e94f2..91a333e 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -18,8 +18,8 @@ def main(): pending_vms = [] for request_iterator in [ - etcd_client.get_prefix(config['etcd']['REQUEST_PREFIX'], value_in_json=True), - etcd_client.watch_prefix(config['etcd']['REQUEST_PREFIX'], timeout=5, value_in_json=True), + etcd_client.get_prefix(config['etcd']['request_prefix'], value_in_json=True), + etcd_client.watch_prefix(config['etcd']['request_prefix'], timeout=5, value_in_json=True), ]: for request_event in request_iterator: request_entry = RequestEntry(request_event) @@ -46,7 +46,7 @@ def main(): r = RequestEntry.from_scratch(type="ScheduleVM", uuid=pending_vm_entry.uuid, hostname=pending_vm_entry.hostname, - request_prefix=config['etcd']['REQUEST_PREFIX']) + request_prefix=config['etcd']['request_prefix']) request_pool.put(r) elif request_entry.type == RequestType.ScheduleVM: @@ -72,7 +72,7 @@ def main(): r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, uuid=request_entry.uuid, destination=request_entry.destination, - request_prefix=config['etcd']['REQUEST_PREFIX']) + request_prefix=config['etcd']['request_prefix']) request_pool.put(r) # If the Request is about a VM that just want to get started/created From bc58a6ed9cceb60099cbdef9ac7ae8394e792c3f Mon Sep 17 00:00:00 2001 From: meow Date: Sat, 21 Dec 2019 14:36:55 +0500 Subject: [PATCH 037/163] Configuration/Setting module added --- conf/ucloud.conf | 46 ------------------- scripts/ucloud | 65 ++++++++++++++++++--------- ucloud/api/helper.py | 6 +-- ucloud/config.py | 30 ++++--------- ucloud/configure/__init__.py | 0 ucloud/configure/main.py | 67 ++++++++++++++++++++++++++++ ucloud/filescanner/main.py | 1 + ucloud/host/main.py | 10 ++--- ucloud/imagescanner/main.py | 10 +++-- ucloud/scheduler/main.py | 2 - ucloud/settings/__init__.py | 86 ++++++++++++++++++++++++++++++++++++ 11 files changed, 217 insertions(+), 106 deletions(-) create mode 100644 ucloud/configure/__init__.py create mode 100644 ucloud/configure/main.py create mode 100644 ucloud/settings/__init__.py diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 222cc4f..334bbeb 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -1,19 +1,3 @@ -[otp] -server = https://otp.ungleich.ch/ungleichotp/ -verify_endpoint = verify/ -auth_name = replace_me -auth_realm = replace_me -auth_seed = replace_me - -[network] -prefix_length = 64 -prefix = 2001:db8::/48 -vxlan_phy_dev = eno1 - -[netbox] -url = https://replace-me.example.com -token = replace_me - [etcd] url = localhost port = 2379 @@ -21,33 +5,3 @@ port = 2379 ca_cert cert_cert cert_key - -file_prefix = /files/ -host_prefix = /hosts/ -image_prefix = /images/ -image_store_prefix = /imagestore/ -network_prefix = /networks/ -request_prefix = /requests/ -user_prefix = /users/ -vm_prefix = /vms/ - -[storage] - -#values = filesystem, ceph -backend = filesystem - -# if STORAGE_BACKEND = filesystem -vm_dir = /var/lib/ucloud/vms -image_dir = /var/lib/ucloud/images - -# if STORAGE_BACKEND = ceph -ceph_vm_pool = ssd -ceph_image_pool = ssd - -# Importing uploaded files -file_dir = /var/lib/ucloud/files - -# For Migrating VMs over ssh/tcp -[ssh] -username -private_key_path \ No newline at end of file diff --git a/scripts/ucloud b/scripts/ucloud index 2da1094..8ea6027 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -3,37 +3,60 @@ import argparse import logging import importlib -import sys import os import multiprocessing as mp +import sys + +from ucloud.configure.main import update_config, configure_parser -COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata'] +def exception_hook(exc_type, exc_value, exc_traceback): + logger.error( + "Uncaught exception", + exc_info=(exc_type, exc_value, exc_traceback) + ) + print(exc_type, exc_value) -if __name__ == "__main__": + +if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG, format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s', filename='/var/log/ucloud.log', filemode='a') - logger = logging.getLogger("ucloud") - arg_parser = argparse.ArgumentParser(prog='ucloud', - description='Open Source Cloud Management Software') - arg_parser.add_argument('-c', '--conf-dir', help="Configuration directory") - arg_parser.add_argument('component', choices=COMMANDS) - arg_parser.add_argument('component_args', nargs='*') + sys.excepthook = exception_hook + mp.set_start_method('spawn') + + arg_parser = argparse.ArgumentParser() + subparsers = arg_parser.add_subparsers(dest="command") + + api_parser = subparsers.add_parser("api") + + host_parser = subparsers.add_parser("host") + host_parser.add_argument("--hostname", required=True) + + scheduler_parser = subparsers.add_parser("scheduler") + + filescanner_parser = subparsers.add_parser("filescanner") + + imagescanner_parser = subparsers.add_parser("imagescanner") + + metadata_parser = subparsers.add_parser("metadata") + + config_parser = subparsers.add_parser("configure") + configure_parser(config_parser) args = arg_parser.parse_args() - if args.conf_dir: - os.environ['UCLOUD_CONF_DIR'] = args.conf_dir + if not args.command: + arg_parser.print_help() + else: + arguments = vars(args) + try: + name = arguments.pop('command') + mod = importlib.import_module("ucloud.{}.main".format(name)) + main = getattr(mod, "main") + main(**arguments) - try: - mp.set_start_method('spawn') - name = args.component - mod = importlib.import_module("ucloud.{}.main".format(name)) - main = getattr(mod, "main") - main(*args.component_args) - - except Exception as e: - logger.exception(e) - print(e) + except Exception as e: + logger.exception(e) + print(e) \ No newline at end of file diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index 2dfb7de..1448e02 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -29,11 +29,7 @@ def check_otp(name, realm, token): return 400 response = requests.post( - "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( - OTP_SERVER=config['otp']['server'], - OTP_VERIFY_ENDPOINT=config['otp']['verify_endpoint'] - ), - json=data, + config['otp']['verification_controller_url'], json=data ) return response.status_code diff --git a/ucloud/config.py b/ucloud/config.py index 3eee897..97d1561 100644 --- a/ucloud/config.py +++ b/ucloud/config.py @@ -5,32 +5,18 @@ import logging from ucloud.common.host import HostPool from ucloud.common.request import RequestPool from ucloud.common.vm import VmPool -from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler +from ucloud.common.storage_handlers import (FileSystemBasedImageStorageHandler, + CEPHBasedImageStorageHandler) from ucloud.common.etcd_wrapper import Etcd3Wrapper +from ucloud.settings import Settings +from os.path import join as join_path -log = logging.getLogger('ucloud.config') +logger = logging.getLogger('ucloud.config') -conf_name = 'ucloud.conf' -conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud') -config_file = os.path.join(conf_dir, conf_name) -config = configparser.ConfigParser(allow_no_value=True) -if os.access(config_file, os.R_OK): - config.read(config_file) -else: - log.warning('Configuration file not found - using defaults') - -etcd_wrapper_args = () -etcd_wrapper_kwargs = { - 'host': config['etcd']['url'], - 'port': config['etcd']['port'], - 'ca_cert': config['etcd']['ca_cert'], - 'cert_cert': config['etcd']['cert_cert'], - 'cert_key': config['etcd']['cert_key'] -} - -etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) +config = Settings() +etcd_client = config.get_etcd_client() host_pool = HostPool(etcd_client, config['etcd']['host_prefix']) vm_pool = VmPool(etcd_client, config['etcd']['vm_prefix']) @@ -38,7 +24,7 @@ request_pool = RequestPool(etcd_client, config['etcd']['request_prefix']) running_vms = [] -__storage_backend = config['storage']['backend'] +__storage_backend = config['storage']['storage_backend'] if __storage_backend == 'filesystem': image_storage_handler = FileSystemBasedImageStorageHandler( vm_base=config['storage']['vm_dir'], diff --git a/ucloud/configure/__init__.py b/ucloud/configure/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ucloud/configure/main.py b/ucloud/configure/main.py new file mode 100644 index 0000000..0baa8eb --- /dev/null +++ b/ucloud/configure/main.py @@ -0,0 +1,67 @@ +import argparse +import sys +import os + +from ucloud.settings import Settings + +config = Settings() +etcd_client = config.get_etcd_client() + +def update_config(section, kwargs): + uncloud_config = etcd_client.get(config.config_key, + value_in_json=True) + if not uncloud_config: + uncloud_config = {} + else: + uncloud_config = uncloud_config.value + + uncloud_config[section] = kwargs + etcd_client.put(config.config_key, uncloud_config, value_in_json=True) + + +def configure_parser(parser): + configure_subparsers = parser.add_subparsers(dest="subcommand") + + otp_parser = configure_subparsers.add_parser("otp") + otp_parser.add_argument("--verification-controller-url", + required=True, metavar="URL") + otp_parser.add_argument("--auth-name", required=True, + metavar="OTP-NAME") + otp_parser.add_argument("--auth-realm", required=True, + metavar="OTP-REALM") + otp_parser.add_argument("--auth-seed", required=True, + metavar="OTP-SEED") + + network_parser = configure_subparsers.add_parser("network") + network_parser.add_argument("--prefix-length", required=True, type=int) + network_parser.add_argument("--prefix", required=True) + network_parser.add_argument("--vxlan-phy-dev", required=True) + + netbox_parser = configure_subparsers.add_parser("netbox") + netbox_parser.add_argument("--url", required=True) + netbox_parser.add_argument("--token", required=True) + + ssh_parser = configure_subparsers.add_parser("ssh") + ssh_parser.add_argument('--username', default="root") + ssh_parser.add_argument('--private-key-path', + default=os.path.expanduser("~/.ssh/id_rsa")) + + storage_parser = configure_subparsers.add_parser("storage") + storage_parser.add_argument('--file-dir', required=True) + storage_parser_subparsers = storage_parser.add_subparsers(dest="storage_backend") + + filesystem_storage_parser = storage_parser_subparsers.add_parser("filesystem") + filesystem_storage_parser.add_argument('--vm-dir', required=True) + filesystem_storage_parser.add_argument('--image-dir', required=True) + + ceph_storage_parser = storage_parser_subparsers.add_parser("ceph") + ceph_storage_parser.add_argument('--ceph-vm-pool', required=True) + ceph_storage_parser.add_argument('--ceph-image-pool', required=True) + + +def main(**kwargs): + subcommand = kwargs.pop('subcommand') + if not subcommand: + pass + else: + update_config(subcommand, kwargs) diff --git a/ucloud/filescanner/main.py b/ucloud/filescanner/main.py index 265f9d9..14a77cf 100755 --- a/ucloud/filescanner/main.py +++ b/ucloud/filescanner/main.py @@ -4,6 +4,7 @@ import pathlib import subprocess as sp import time import sys + from uuid import uuid4 from . import logger diff --git a/ucloud/host/main.py b/ucloud/host/main.py index bd03a08..ddc52c7 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -8,17 +8,16 @@ from ucloud.common.etcd_wrapper import Etcd3Wrapper from ucloud.common.request import RequestEntry, RequestType from ucloud.config import (vm_pool, request_pool, etcd_client, running_vms, - etcd_wrapper_args, etcd_wrapper_kwargs, HostPool, config) from .helper import find_free_port -from . import virtualmachine -from ucloud.host import logger +from . import virtualmachine, logger def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" - client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) + + client = config.get_etcd_client() host_pool = HostPool(client, config['etcd']['host_prefix']) this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) @@ -73,7 +72,7 @@ def maintenance(host): running_vms.remove(_vm) def check(): - if config['storage']['backend'] == 'filesystem' and \ + if config['storage']['storage_backend'] == 'filesystem' and \ not isdir(config['storage']['vm_dir']): print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" @@ -81,7 +80,6 @@ def check(): sys.exit(1) - def main(hostname): check() diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index 2658641..135f8cb 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -1,7 +1,9 @@ import json import os import subprocess +import sys +from os.path import isdir from os.path import join as join_path from ucloud.config import etcd_client, config, image_storage_handler from ucloud.imagescanner import logger @@ -20,10 +22,10 @@ def qemu_img_type(path): def check(): """ check whether settings are sane, refuse to start if they aren't """ - if config['storage']['backend'] == 'filesystem' and not isdir(config['storage']['image_dir']): - print("You have set STORAGE_BACKEND to filesystem, but " - "{} does not exist. Refusing to start".format(config['storage']['image_dir'])) - sys.exit(1) + if config['storage']['storage_backend'] == 'filesystem' and not isdir(config['storage']['image_dir']): + sys.exit("You have set STORAGE_BACKEND to filesystem, but " + "{} does not exist. Refusing to start".format(config['storage']['image_dir']) + ) try: subprocess.check_output(['which', 'qemu-img']) diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index 91a333e..49d6291 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -13,8 +13,6 @@ from . import logger def main(): - logger.info("%s SESSION STARTED %s", '*' * 5, '*' * 5) - pending_vms = [] for request_iterator in [ diff --git a/ucloud/settings/__init__.py b/ucloud/settings/__init__.py new file mode 100644 index 0000000..5f29c41 --- /dev/null +++ b/ucloud/settings/__init__.py @@ -0,0 +1,86 @@ +import configparser +import logging +import sys +import os + +from ucloud.common.etcd_wrapper import Etcd3Wrapper + + +logger = logging.getLogger(__name__) + + +class CustomConfigParser(configparser.RawConfigParser): + def __getitem__(self, key): + try: + result = super().__getitem__(key) + except KeyError as err: + raise KeyError("Key '{}' not found in config file"\ + .format(key)) from err + else: + return result + + +class Settings(object): + def __init__(self, config_key='/uncloud/config/'): + conf_name = 'ucloud.conf' + conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud') + config_file = os.path.join(conf_dir, conf_name) + + self.config_parser = CustomConfigParser(allow_no_value=True) + self.config_key = config_key + + self.read_internal_values() + self.read_config_file_values(config_file) + + self.etcd_wrapper_args = tuple() + self.etcd_wrapper_kwargs = { + 'host': self.config_parser['etcd']['url'], + 'port': self.config_parser['etcd']['port'], + 'ca_cert': self.config_parser['etcd']['ca_cert'], + 'cert_cert': self.config_parser['etcd']['cert_cert'], + 'cert_key': self.config_parser['etcd']['cert_key'] + } + + + def get_etcd_client(self): + args = self.etcd_wrapper_args + kwargs = self.etcd_wrapper_kwargs + return Etcd3Wrapper(*args, **kwargs) + + def read_internal_values(self): + self.config_parser.read_dict({ + 'etcd': { + 'file_prefix': '/files/', + 'host_prefix': '/hosts/', + 'image_prefix': '/images/', + 'image_store_prefix': '/imagestore/', + 'network_prefix': '/networks/', + 'request_prefix': '/requests/', + 'user_prefix': '/users/', + 'vm_prefix': '/vms/', + } + }) + + def read_config_file_values(self, config_file): + try: + # Trying to read configuration file + with open(config_file, "r") as config_file_handle: + self.config_parser.read_file(config_file_handle) + except FileNotFoundError: + sys.exit('Configuration file {} not found!'.format(config_file)) + except Exception as err: + logger.exception(err) + sys.exit("Error occurred while reading configuration file") + + def read_values_from_etcd(self): + etcd_client = self.get_etcd_client() + config_from_etcd = etcd_client.get(self.config_key, value_in_json=True) + if config_from_etcd: + self.config_parser.read_dict(config_from_etcd.value) + else: + return + sys.exit("No settings found in etcd at key {}".format(self.config_key)) + + def __getitem__(self, key): + self.read_values_from_etcd() + return self.config_parser[key] From 04993e41067edccd1fef3b6b48b4e8062f884c9f Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 22 Dec 2019 12:26:48 +0500 Subject: [PATCH 038/163] Refactoring, Removal of most global vars, config default path is ~/ucloud/ --- scripts/ucloud | 31 +- setup.py | 4 +- ucloud/api/common_fields.py | 6 +- ucloud/api/create_image_store.py | 5 +- ucloud/api/helper.py | 18 +- ucloud/api/main.py | 118 +++--- ucloud/api/schemas.py | 23 +- ucloud/common/etcd_wrapper.py | 40 +- ucloud/common/network.py | 60 +++ ucloud/common/storage_handlers.py | 18 + ucloud/config.py | 39 -- ucloud/configure/main.py | 10 +- ucloud/filescanner/main.py | 17 +- ucloud/host/helper.py | 13 - ucloud/host/main.py | 110 +---- ucloud/host/virtualmachine.py | 648 +++++++++++++++--------------- ucloud/imagescanner/main.py | 19 +- ucloud/metadata/main.py | 10 +- ucloud/scheduler/helper.py | 25 +- ucloud/scheduler/main.py | 24 +- ucloud/scheduler/main.py.old | 93 ----- ucloud/settings/__init__.py | 38 +- ucloud/shared/__init__.py | 30 ++ 23 files changed, 673 insertions(+), 726 deletions(-) create mode 100644 ucloud/common/network.py delete mode 100644 ucloud/config.py delete mode 100644 ucloud/host/helper.py delete mode 100755 ucloud/scheduler/main.py.old create mode 100644 ucloud/shared/__init__.py diff --git a/scripts/ucloud b/scripts/ucloud index 8ea6027..7f3ef3a 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -3,29 +3,23 @@ import argparse import logging import importlib -import os import multiprocessing as mp import sys -from ucloud.configure.main import update_config, configure_parser +from ucloud.configure.main import configure_parser def exception_hook(exc_type, exc_value, exc_traceback): - logger.error( - "Uncaught exception", - exc_info=(exc_type, exc_value, exc_traceback) - ) - print(exc_type, exc_value) + logger.error( + 'Uncaught exception', + exc_info=(exc_type, exc_value, exc_traceback) + ) + print('Error: ', end='') + print(exc_type, exc_value, exc_traceback) if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG, - format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s', - filename='/var/log/ucloud.log', filemode='a') - logger = logging.getLogger("ucloud") - sys.excepthook = exception_hook - mp.set_start_method('spawn') arg_parser = argparse.ArgumentParser() subparsers = arg_parser.add_subparsers(dest="command") @@ -50,13 +44,20 @@ if __name__ == '__main__': if not args.command: arg_parser.print_help() else: + logging.basicConfig( + level=logging.DEBUG, + format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s', + handlers=[logging.handlers.SysLogHandler(address = '/dev/log')] + ) + logger = logging.getLogger("ucloud") + mp.set_start_method('spawn') + arguments = vars(args) try: name = arguments.pop('command') mod = importlib.import_module("ucloud.{}.main".format(name)) main = getattr(mod, "main") main(**arguments) - except Exception as e: - logger.exception(e) + logger.exception('Error') print(e) \ No newline at end of file diff --git a/setup.py b/setup.py index a7ddbe6..e273d68 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,5 @@ +import os + from setuptools import setup, find_packages with open("README.md", "r") as fh: @@ -39,5 +41,5 @@ setup(name='ucloud', 'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3', ], scripts=['scripts/ucloud'], - data_files=[('/etc/ucloud/', ['conf/ucloud.conf'])], + data_files=[(os.path.expanduser('~/ucloud/'), ['conf/ucloud.conf'])], zip_safe=False) diff --git a/ucloud/api/common_fields.py b/ucloud/api/common_fields.py index cf86283..a793d26 100755 --- a/ucloud/api/common_fields.py +++ b/ucloud/api/common_fields.py @@ -1,6 +1,8 @@ import os -from ucloud.config import etcd_client, config +from ucloud.shared import shared +from ucloud.settings import settings + class Optional: pass @@ -47,6 +49,6 @@ class VmUUIDField(Field): self.validation = self.vm_uuid_validation def vm_uuid_validation(self): - r = etcd_client.get(os.path.join(config['etcd']['vm_prefix'], self.uuid)) + r = shared.etcd_client.get(os.path.join(settings['etcd']['vm_prefix'], self.uuid)) if not r: self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/ucloud/api/create_image_store.py b/ucloud/api/create_image_store.py index 9023bd6..978a182 100755 --- a/ucloud/api/create_image_store.py +++ b/ucloud/api/create_image_store.py @@ -3,7 +3,8 @@ import os from uuid import uuid4 -from ucloud.config import etcd_client, config +from ucloud.shared import shared +from ucloud.settings import settings data = { "is_public": True, @@ -13,4 +14,4 @@ data = { "attributes": {"list": [], "key": [], "pool": "images"}, } -etcd_client.put(os.path.join(config['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) +shared.etcd_client.put(os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index 1448e02..9cda36e 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -7,29 +7,29 @@ import requests from pyotp import TOTP -from ucloud.config import vm_pool, config - +from ucloud.shared import shared +from ucloud.settings import settings logger = logging.getLogger("ucloud.api.helper") def check_otp(name, realm, token): try: data = { - "auth_name": config['otp']['auth_name'], - "auth_token": TOTP(config['otp']['auth_seed']).now(), - "auth_realm": config['otp']['auth_realm'], + "auth_name": settings['otp']['auth_name'], + "auth_token": TOTP(settings['otp']['auth_seed']).now(), + "auth_realm": settings['otp']['auth_realm'], "name": name, "realm": realm, "token": token, } except binascii.Error as err: logger.error( - "Cannot compute OTP for seed: {}".format(config['otp']['auth_seed']) + "Cannot compute OTP for seed: {}".format(settings['otp']['auth_seed']) ) return 400 response = requests.post( - config['otp']['verification_controller_url'], json=data + settings['otp']['verification_controller_url'], json=data ) return response.status_code @@ -43,7 +43,7 @@ def resolve_vm_name(name, owner): result = next( filter( lambda vm: vm.value["owner"] == owner and vm.value["name"] == name, - vm_pool.vms, + shared.vm_pool.vms, ), None, ) @@ -81,7 +81,7 @@ def resolve_image_name(name, etcd_client): except Exception: raise ValueError("Image name not in correct format i.e {store_name}:{image_name}") - images = etcd_client.get_prefix(config['etcd']['image_prefix'], value_in_json=True) + images = etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True) # Try to find image with name == image_name and store_name == store_name try: diff --git a/ucloud/api/main.py b/ucloud/api/main.py index 0e202d8..05972ff 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -10,10 +10,9 @@ from flask_restful import Resource, Api from ucloud.common import counters from ucloud.common.vm import VMStatus from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import ( - etcd_client, request_pool, vm_pool, - host_pool, config, image_storage_handler -) +from ucloud.settings import settings +from ucloud.shared import shared + from . import schemas from .helper import generate_mac, mac2ipv6 from . import logger @@ -31,7 +30,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(config['etcd']['vm_prefix'], vm_uuid) + vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) specs = { "cpu": validator.specs["cpu"], "ram": validator.specs["ram"], @@ -39,7 +38,7 @@ class CreateVM(Resource): "hdd": validator.specs["hdd"], } macs = [generate_mac() for _ in range(len(data["network"]))] - tap_ids = [counters.increment_etcd_counter(etcd_client, "/v1/counter/tap") + tap_ids = [counters.increment_etcd_counter(shared.etcd_client, "/v1/counter/tap") for _ in range(len(data["network"]))] vm_entry = { "name": data["vm_name"], @@ -54,14 +53,14 @@ class CreateVM(Resource): "network": list(zip(data["network"], macs, tap_ids)), "metadata": {"ssh-keys": []}, } - etcd_client.put(vm_key, vm_entry, value_in_json=True) + shared.etcd_client.put(vm_key, vm_entry, value_in_json=True) # Create ScheduleVM Request r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=config['etcd']['request_prefix'] + request_prefix=settings['etcd']['request_prefix'] ) - request_pool.put(r) + shared.request_pool.put(r) return {"message": "VM Creation Queued"}, 200 return validator.get_errors(), 400 @@ -73,16 +72,16 @@ class VmStatus(Resource): data = request.json validator = schemas.VMStatusSchema(data) if validator.is_valid(): - vm = vm_pool.get( - join_path(config['etcd']['vm_prefix'], data["uuid"]) + vm = shared.vm_pool.get( + join_path(settings['etcd']['vm_prefix'], data["uuid"]) ) vm_value = vm.value.copy() vm_value["ip"] = [] for network_mac_and_tap in vm.network: network_name, mac, tap = network_mac_and_tap - network = etcd_client.get( + network = shared.etcd_client.get( join_path( - config['etcd']['network_prefix'], + settings['etcd']['network_prefix'], data["name"], network_name, ), @@ -102,8 +101,8 @@ class CreateImage(Resource): data = request.json validator = schemas.CreateImageSchema(data) if validator.is_valid(): - file_entry = etcd_client.get( - join_path(config['etcd']['file_prefix'], data["uuid"]) + file_entry = shared.etcd_client.get( + join_path(settings['etcd']['file_prefix'], data["uuid"]) ) file_entry_value = json.loads(file_entry.value) @@ -115,8 +114,8 @@ class CreateImage(Resource): "store_name": data["image_store"], "visibility": "public", } - etcd_client.put( - join_path(config['etcd']['image_prefix'], data["uuid"]), + shared.etcd_client.put( + join_path(settings['etcd']['image_prefix'], data["uuid"]), json.dumps(image_entry_json), ) @@ -127,8 +126,8 @@ class CreateImage(Resource): class ListPublicImages(Resource): @staticmethod def get(): - images = etcd_client.get_prefix( - config['etcd']['image_prefix'], value_in_json=True + images = shared.etcd_client.get_prefix( + settings['etcd']['image_prefix'], value_in_json=True ) r = { "images": [] @@ -150,8 +149,8 @@ class VMAction(Resource): validator = schemas.VmActionSchema(data) if validator.is_valid(): - vm_entry = vm_pool.get( - join_path(config['etcd']['vm_prefix'], data["uuid"]) + vm_entry = shared.vm_pool.get( + join_path(settings['etcd']['vm_prefix'], data["uuid"]) ) action = data["action"] @@ -159,25 +158,25 @@ class VMAction(Resource): action = "schedule" if action == "delete" and vm_entry.hostname == "": - if image_storage_handler.is_vm_image_exists(vm_entry.uuid): - r_status = image_storage_handler.delete_vm_image(vm_entry.uuid) + if shared.storage_handler.is_vm_image_exists(vm_entry.uuid): + r_status = shared.storage_handler.delete_vm_image(vm_entry.uuid) if r_status: - etcd_client.client.delete(vm_entry.key) + shared.etcd_client.client.delete(vm_entry.key) return {"message": "VM successfully deleted"} else: logger.error("Some Error Occurred while deleting VM") return {"message": "VM deletion unsuccessfull"} else: - etcd_client.client.delete(vm_entry.key) + shared.etcd_client.client.delete(vm_entry.key) return {"message": "VM successfully deleted"} r = RequestEntry.from_scratch( type="{}VM".format(action.title()), uuid=data["uuid"], hostname=vm_entry.hostname, - request_prefix=config['etcd']['request_prefix'] + request_prefix=settings['etcd']['request_prefix'] ) - request_pool.put(r) + shared.request_pool.put(r) return {"message": "VM {} Queued".format(action.title())}, 200 else: return validator.get_errors(), 400 @@ -190,18 +189,18 @@ class VMMigration(Resource): validator = schemas.VmMigrationSchema(data) if validator.is_valid(): - vm = vm_pool.get(data["uuid"]) + vm = shared.vm_pool.get(data["uuid"]) r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm.uuid, destination=join_path( - config['etcd']['host_prefix'], validator.destination.value + settings['etcd']['host_prefix'], validator.destination.value ), migration=True, - request_prefix=config['etcd']['request_prefix'] + request_prefix=settings['etcd']['request_prefix'] ) - request_pool.put(r) + shared.request_pool.put(r) return {"message": "VM Migration Initialization Queued"}, 200 else: return validator.get_errors(), 400 @@ -214,8 +213,8 @@ class ListUserVM(Resource): validator = schemas.OTPSchema(data) if validator.is_valid(): - vms = etcd_client.get_prefix( - config['etcd']['vm_prefix'], value_in_json=True + vms = shared.etcd_client.get_prefix( + settings['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) @@ -227,7 +226,6 @@ class ListUserVM(Resource): "specs": vm.value["specs"], "status": vm.value["status"], "hostname": vm.value["hostname"], - # "mac": vm.value["mac"], "vnc_socket": None if vm.value.get("vnc_socket", None) is None else vm.value["vnc_socket"], @@ -248,8 +246,8 @@ class ListUserFiles(Resource): validator = schemas.OTPSchema(data) if validator.is_valid(): - files = etcd_client.get_prefix( - config['etcd']['file_prefix'], value_in_json=True + files = shared.etcd_client.get_prefix( + settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] user_files = list( @@ -273,14 +271,14 @@ class CreateHost(Resource): data = request.json validator = schemas.CreateHostSchema(data) if validator.is_valid(): - host_key = join_path(config['etcd']['host_prefix'], uuid4().hex) + host_key = join_path(settings['etcd']['host_prefix'], uuid4().hex) host_entry = { "specs": data["specs"], "hostname": data["hostname"], "status": "DEAD", "last_heartbeat": "", } - etcd_client.put(host_key, host_entry, value_in_json=True) + shared.etcd_client.put(host_key, host_entry, value_in_json=True) return {"message": "Host Created"}, 200 @@ -290,7 +288,7 @@ class CreateHost(Resource): class ListHost(Resource): @staticmethod def get(): - hosts = host_pool.hosts + hosts = shared.host_pool.hosts r = { host.key: { "status": host.status, @@ -312,12 +310,12 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - config['etcd']['user_prefix'], + settings['etcd']['user_prefix'], data["realm"], data["name"], "key", ) - etcd_entry = etcd_client.get_prefix( + etcd_entry = shared.etcd_client.get_prefix( etcd_key, value_in_json=True ) @@ -329,13 +327,13 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']['user_prefix'], + settings['etcd']['user_prefix'], data["realm"], data["name"], "key", data["key_name"], ) - etcd_entry = etcd_client.get(etcd_key, value_in_json=True) + etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) if etcd_entry: return { @@ -358,13 +356,13 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']['user_prefix'], + settings['etcd']['user_prefix'], data["realm"], data["name"], "key", data["key_name"], ) - etcd_entry = etcd_client.get(etcd_key, value_in_json=True) + etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) if etcd_entry: return { "message": "Key with name '{}' already exists".format( @@ -373,7 +371,7 @@ class AddSSHKey(Resource): } else: # Key Not Found. It implies user' haven't added any key yet. - etcd_client.put(etcd_key, data["key"], value_in_json=True) + shared.etcd_client.put(etcd_key, data["key"], value_in_json=True) return {"message": "Key added successfully"} else: return validator.get_errors(), 400 @@ -388,15 +386,15 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - config['etcd']['user_prefix'], + settings['etcd']['user_prefix'], data["realm"], data["name"], "key", data["key_name"], ) - etcd_entry = etcd_client.get(etcd_key, value_in_json=True) + etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) if etcd_entry: - etcd_client.client.delete(etcd_key) + shared.etcd_client.client.delete(etcd_key) return {"message": "Key successfully removed."} else: return { @@ -418,22 +416,22 @@ class CreateNetwork(Resource): network_entry = { "id": counters.increment_etcd_counter( - etcd_client, "/v1/counter/vxlan" + shared.etcd_client, "/v1/counter/vxlan" ), "type": data["type"], } if validator.user.value: try: nb = pynetbox.api( - url=config['netbox']['url'], - token=config['netbox']['token'], + url=settings['netbox']['url'], + token=settings['netbox']['token'], ) nb_prefix = nb.ipam.prefixes.get( - prefix=config['network']['prefix'] + prefix=settings['network']['prefix'] ) prefix = nb_prefix.available_prefixes.create( data={ - "prefix_length": int(config['network']['prefix_length']), + "prefix_length": int(settings['network']['prefix_length']), "description": '{}\'s network "{}"'.format( data["name"], data["network_name"] ), @@ -449,11 +447,11 @@ class CreateNetwork(Resource): network_entry["ipv6"] = "fd00::/64" network_key = join_path( - config['etcd']['network_prefix'], + settings['etcd']['network_prefix'], data['name'], data['network_name'], ) - etcd_client.put(network_key, network_entry, value_in_json=True) + shared.etcd_client.put(network_key, network_entry, value_in_json=True) return {"message": "Network successfully added."} else: return validator.get_errors(), 400 @@ -467,9 +465,9 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - config['etcd']['network_prefix'], data["name"] + settings['etcd']['network_prefix'], data["name"] ) - networks = etcd_client.get_prefix(prefix, value_in_json=True) + networks = shared.etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] for net in networks: net.value["name"] = net.key.split("/")[-1] @@ -503,7 +501,7 @@ api.add_resource(CreateNetwork, "/network/create") def main(): - image_stores = list(etcd_client.get_prefix(config['etcd']['image_store_prefix'], value_in_json=True)) + image_stores = list(shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix'], value_in_json=True)) if len(image_stores) == 0: data = { "is_public": True, @@ -513,7 +511,7 @@ def main(): "attributes": {"list": [], "key": [], "pool": "images"}, } - etcd_client.put(join_path(config['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) + shared.etcd_client.put(join_path(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) app.run(host="::", debug=True) diff --git a/ucloud/api/schemas.py b/ucloud/api/schemas.py index a3e0aa8..d639be4 100755 --- a/ucloud/api/schemas.py +++ b/ucloud/api/schemas.py @@ -21,7 +21,8 @@ import bitmath from ucloud.common.host import HostStatus from ucloud.common.vm import VMStatus -from ucloud.config import etcd_client, config, vm_pool, host_pool +from ucloud.shared import shared +from ucloud.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -102,14 +103,14 @@ class CreateImageSchema(BaseSchema): super().__init__(data, fields) def file_uuid_validation(self): - file_entry = etcd_client.get(os.path.join(config['etcd']['file_prefix'], self.uuid.value)) + file_entry = shared.etcd_client.get(os.path.join(settings['etcd']['file_prefix'], self.uuid.value)) if file_entry is None: self.add_error( "Image File with uuid '{}' Not Found".format(self.uuid.value) ) def image_store_name_validation(self): - image_stores = list(etcd_client.get_prefix(config['etcd']['image_store_prefix'])) + image_stores = list(shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix'])) image_store = next( filter( @@ -218,7 +219,7 @@ class CreateVMSchema(OTPSchema): def image_validation(self): try: - image_uuid = helper.resolve_image_name(self.image.value, etcd_client) + image_uuid = helper.resolve_image_name(self.image.value, shared.etcd_client) except Exception as e: logger.exception("Cannot resolve image name = %s", self.image.value) self.add_error(str(e)) @@ -236,7 +237,7 @@ class CreateVMSchema(OTPSchema): if _network: for net in _network: - network = etcd_client.get(os.path.join(config['etcd']['network_prefix'], + network = shared.etcd_client.get(os.path.join(settings['etcd']['network_prefix'], self.name.value, net), value_in_json=True) if not network: @@ -310,7 +311,7 @@ class VMStatusSchema(OTPSchema): super().__init__(data, fields) def validation(self): - vm = vm_pool.get(self.uuid.value) + vm = shared.vm_pool.get(self.uuid.value) if not ( vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" ): @@ -343,7 +344,7 @@ class VmActionSchema(OTPSchema): ) def validation(self): - vm = vm_pool.get(self.uuid.value) + vm = shared.vm_pool.get(self.uuid.value) if not ( vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" ): @@ -383,7 +384,7 @@ class VmMigrationSchema(OTPSchema): def destination_validation(self): hostname = self.destination.value - host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + host = next(filter(lambda h: h.hostname == hostname, shared.host_pool.hosts), None) if not host: self.add_error("No Such Host ({}) exists".format(self.destination.value)) elif host.status != HostStatus.alive: @@ -392,7 +393,7 @@ class VmMigrationSchema(OTPSchema): self.destination.value = host.key def validation(self): - vm = vm_pool.get(self.uuid.value) + vm = shared.vm_pool.get(self.uuid.value) if not ( vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" ): @@ -401,7 +402,7 @@ class VmMigrationSchema(OTPSchema): if vm.status != VMStatus.running: self.add_error("Can't migrate non-running VM") - if vm.hostname == os.path.join(config['etcd']['host_prefix'], self.destination.value): + if vm.hostname == os.path.join(settings['etcd']['host_prefix'], self.destination.value): self.add_error("Destination host couldn't be same as Source Host") @@ -443,7 +444,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = etcd_client.get(os.path.join(config['etcd']['network_prefix'], + network = shared.etcd_client.get(os.path.join(settings['etcd']['network_prefix'], self.name.value, self.network_name.value), value_in_json=True) diff --git a/ucloud/common/etcd_wrapper.py b/ucloud/common/etcd_wrapper.py index a3fb83f..e249e6c 100644 --- a/ucloud/common/etcd_wrapper.py +++ b/ucloud/common/etcd_wrapper.py @@ -4,40 +4,63 @@ import queue import copy from collections import namedtuple +from functools import wraps + +from . import logger + +PseudoEtcdMeta = namedtuple('PseudoEtcdMeta', ['key']) -PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) class EtcdEntry: # key: str # value: str def __init__(self, meta, value, value_in_json=False): - self.key = meta.key.decode("utf-8") - self.value = value.decode("utf-8") + self.key = meta.key.decode('utf-8') + self.value = value.decode('utf-8') if value_in_json: self.value = json.loads(self.value) + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as err: + raise etcd3.exceptions.ConnectionFailedError('etcd connection failed') from err + except etcd3.exceptions.ConnectionTimeoutError as err: + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout') from err + except Exception: + print('Some error occurred, most probably it is etcd that is erroring out.') + logger.exception('Some etcd error occurred') + return wrapper + + class Etcd3Wrapper: def __init__(self, *args, **kwargs): self.client = etcd3.client(*args, **kwargs) + @readable_errors def get(self, *args, value_in_json=False, **kwargs): _value, _key = self.client.get(*args, **kwargs) if _key is None or _value is None: return None return EtcdEntry(_key, _value, value_in_json=value_in_json) + @readable_errors def put(self, *args, value_in_json=False, **kwargs): _key, _value = args if value_in_json: _value = json.dumps(_value) if not isinstance(_key, str): - _key = _key.decode("utf-8") + _key = _key.decode('utf-8') return self.client.put(_key, _value, **kwargs) + @readable_errors def get_prefix(self, *args, value_in_json=False, **kwargs): r = self.client.get_prefix(*args, **kwargs) for entry in r: @@ -45,10 +68,11 @@ class Etcd3Wrapper: if e.value: yield e + @readable_errors def watch_prefix(self, key, timeout=0, value_in_json=False): - timeout_event = EtcdEntry(PseudoEtcdMeta(key=b"TIMEOUT"), - value=str.encode(json.dumps({"status": "TIMEOUT", - "type": "TIMEOUT"})), + timeout_event = EtcdEntry(PseudoEtcdMeta(key=b'TIMEOUT'), + value=str.encode(json.dumps({'status': 'TIMEOUT', + 'type': 'TIMEOUT'})), value_in_json=value_in_json) event_queue = queue.Queue() @@ -71,4 +95,4 @@ class Etcd3Wrapper: class PsuedoEtcdEntry(EtcdEntry): def __init__(self, key, value, value_in_json=False): - super().__init__(PseudoEtcdMeta(key=key.encode("utf-8")), value, value_in_json=value_in_json) + super().__init__(PseudoEtcdMeta(key=key.encode('utf-8')), value, value_in_json=value_in_json) diff --git a/ucloud/common/network.py b/ucloud/common/network.py new file mode 100644 index 0000000..6a6c6e2 --- /dev/null +++ b/ucloud/common/network.py @@ -0,0 +1,60 @@ +import subprocess as sp +import random +import logging +import socket +from contextlib import closing + +logger = logging.getLogger(__name__) + + +def random_bytes(num=6): + return [random.randrange(256) for _ in range(num)] + + +def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): + mac = random_bytes() + if oui: + if type(oui) == str: + oui = [int(chunk) for chunk in oui.split(separator)] + mac = oui + random_bytes(num=6 - len(oui)) + else: + if multicast: + mac[0] |= 1 # set bit 0 + else: + mac[0] &= ~1 # clear bit 0 + if uaa: + mac[0] &= ~(1 << 1) # clear bit 1 + else: + mac[0] |= 1 << 1 # set bit 1 + return separator.join(byte_fmt % b for b in mac) + + +def create_dev(script, _id, dev, ip=None): + command = [script, _id, dev] + if ip: + command.append(ip) + try: + output = sp.check_output(command, stderr=sp.PIPE) + except Exception as e: + print(e) + return None + else: + return output.decode('utf-8').strip() + + +def delete_network_interface(iface): + try: + sp.check_output(['ip', 'link', 'del', iface]) + except Exception: + logger.exception('Interface Deletion failed') + + +def find_free_port(): + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + try: + s.bind(('', 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + except Exception: + return None + else: + return s.getsockname()[1] diff --git a/ucloud/common/storage_handlers.py b/ucloud/common/storage_handlers.py index 8b1097a..d2bd452 100644 --- a/ucloud/common/storage_handlers.py +++ b/ucloud/common/storage_handlers.py @@ -7,6 +7,8 @@ from abc import ABC from . import logger from os.path import join as join_path +from ucloud.settings import settings as config + class ImageStorageHandler(ABC): def __init__(self, image_base, vm_base): @@ -156,3 +158,19 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): path = join_path(self.vm_base, path) command = ["rbd", "info", path] return self.execute_command(command, report=False) + + +def get_storage_handler(): + __storage_backend = config['storage']['storage_backend'] + if __storage_backend == 'filesystem': + return FileSystemBasedImageStorageHandler( + vm_base=config['storage']['vm_dir'], + image_base=config['storage']['image_dir'] + ) + elif __storage_backend == 'ceph': + return CEPHBasedImageStorageHandler( + vm_base=config['storage']['ceph_vm_pool'], + image_base=config['storage']['ceph_image_pool'] + ) + else: + raise Exception('Unknown Image Storage Handler') diff --git a/ucloud/config.py b/ucloud/config.py deleted file mode 100644 index 97d1561..0000000 --- a/ucloud/config.py +++ /dev/null @@ -1,39 +0,0 @@ -import configparser -import os -import logging - -from ucloud.common.host import HostPool -from ucloud.common.request import RequestPool -from ucloud.common.vm import VmPool -from ucloud.common.storage_handlers import (FileSystemBasedImageStorageHandler, - CEPHBasedImageStorageHandler) -from ucloud.common.etcd_wrapper import Etcd3Wrapper -from ucloud.settings import Settings -from os.path import join as join_path - -logger = logging.getLogger('ucloud.config') - - - -config = Settings() -etcd_client = config.get_etcd_client() - -host_pool = HostPool(etcd_client, config['etcd']['host_prefix']) -vm_pool = VmPool(etcd_client, config['etcd']['vm_prefix']) -request_pool = RequestPool(etcd_client, config['etcd']['request_prefix']) - -running_vms = [] - -__storage_backend = config['storage']['storage_backend'] -if __storage_backend == 'filesystem': - image_storage_handler = FileSystemBasedImageStorageHandler( - vm_base=config['storage']['vm_dir'], - image_base=config['storage']['image_dir'] - ) -elif __storage_backend == 'ceph': - image_storage_handler = CEPHBasedImageStorageHandler( - vm_base=config['storage']['ceph_vm_pool'], - image_base=config['storage']['ceph_image_pool'] - ) -else: - raise Exception('Unknown Image Storage Handler') diff --git a/ucloud/configure/main.py b/ucloud/configure/main.py index 0baa8eb..71e07a1 100644 --- a/ucloud/configure/main.py +++ b/ucloud/configure/main.py @@ -2,21 +2,19 @@ import argparse import sys import os -from ucloud.settings import Settings +from ucloud.settings import settings +from ucloud.shared import shared -config = Settings() -etcd_client = config.get_etcd_client() def update_config(section, kwargs): - uncloud_config = etcd_client.get(config.config_key, - value_in_json=True) + uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - etcd_client.put(config.config_key, uncloud_config, value_in_json=True) + shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) def configure_parser(parser): diff --git a/ucloud/filescanner/main.py b/ucloud/filescanner/main.py index 14a77cf..ff38748 100755 --- a/ucloud/filescanner/main.py +++ b/ucloud/filescanner/main.py @@ -8,8 +8,8 @@ import sys from uuid import uuid4 from . import logger -from ucloud.config import config, etcd_client - +from ucloud.settings import settings +from ucloud.shared import shared def getxattr(file, attr): """Get specified user extended attribute (arg:attr) of a file (arg:file)""" @@ -69,11 +69,10 @@ except Exception as e: def main(): - BASE_DIR = config['storage']['file_dir'] - FILE_PREFIX = config['etcd']['file_prefix'] + base_dir = settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR - files = glob.glob("{}/**".format(BASE_DIR), recursive=True) + files = glob.glob("{}/**".format(base_dir), recursive=True) # Retain only Files files = list(filter(os.path.isfile, files)) @@ -89,7 +88,7 @@ def main(): file_id = uuid4() # Get Username - owner = pathlib.Path(file).parts[len(pathlib.Path(BASE_DIR).parts)] + owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] # Get Creation Date of File # Here, we are assuming that ctime is creation time @@ -105,7 +104,7 @@ def main(): file_path = pathlib.Path(file).parts[-1] # Create Entry - entry_key = os.path.join(FILE_PREFIX, str(file_id)) + entry_key = os.path.join(settings['etcd']['file_prefix'], str(file_id)) entry_value = { "filename": file_path, "owner": owner, @@ -115,8 +114,8 @@ def main(): } logger.info("Tracking %s", file) - # Insert Entry - etcd_client.put(entry_key, entry_value, value_in_json=True) + + shared.etcd_client.put(entry_key, entry_value, value_in_json=True) setxattr(file, "utracked", True) diff --git a/ucloud/host/helper.py b/ucloud/host/helper.py deleted file mode 100644 index edcb82d..0000000 --- a/ucloud/host/helper.py +++ /dev/null @@ -1,13 +0,0 @@ -import socket -from contextlib import closing - - -def find_free_port(): - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - try: - s.bind(('', 0)) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except Exception: - return None - else: - return s.getsockname()[1] diff --git a/ucloud/host/main.py b/ucloud/host/main.py index ddc52c7..f78f629 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -3,22 +3,21 @@ import multiprocessing as mp import time import sys -from os.path import isdir -from ucloud.common.etcd_wrapper import Etcd3Wrapper from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import (vm_pool, request_pool, - etcd_client, running_vms, - HostPool, config) +from ucloud.common.host import HostPool +from ucloud.shared import shared +from ucloud.settings import settings -from .helper import find_free_port from . import virtualmachine, logger +vmm = virtualmachine.VMM() + def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" - client = config.get_etcd_client() - host_pool = HostPool(client, config['etcd']['host_prefix']) + client = shared.etcd_client + host_pool = HostPool(client) this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) while True: @@ -27,122 +26,55 @@ def update_heartbeat(hostname): time.sleep(10) -def maintenance(host): - # To capture vm running according to running_vms list - - # This is to capture successful migration of a VM. - # Suppose, this host is running "vm1" and user initiated - # request to migrate this "vm1" to some other host. On, - # successful migration the destination host would set - # the vm hostname to itself. Thus, we are checking - # whether this host vm is successfully migrated. If yes - # then we shutdown "vm1" on this host. - - to_be_removed = [] - for running_vm in running_vms: - with vm_pool.get_put(running_vm.key) as vm_entry: - if vm_entry.hostname != host.key and not vm_entry.in_migration: - running_vm.handle.shutdown() - logger.info("VM migration not completed successfully.") - to_be_removed.append(running_vm) - - for r in to_be_removed: - running_vms.remove(r) - - # To check vm running according to etcd entries - alleged_running_vms = vm_pool.by_status("RUNNING", vm_pool.by_host(host.key)) - - for vm_entry in alleged_running_vms: - _vm = virtualmachine.get_vm(running_vms, vm_entry.key) - # Whether, the allegedly running vm is in our - # running_vms list or not if it is said to be - # running on this host but it is not then we - # need to shut it down - - # This is to capture poweroff/shutdown of a VM - # initiated by user inside VM. OR crash of VM by some - # user running process - if (_vm and not _vm.handle.is_running()) or not _vm: - logger.debug("_vm = %s, is_running() = %s" % (_vm, _vm.handle.is_running())) - vm_entry.add_log("""{} is not running but is said to be running. - So, shutting it down and declare it killed""".format(vm_entry.key)) - vm_entry.declare_killed() - vm_pool.put(vm_entry) - if _vm: - running_vms.remove(_vm) - -def check(): - if config['storage']['storage_backend'] == 'filesystem' and \ - not isdir(config['storage']['vm_dir']): - - print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" - " in /etc/ucloud/ucloud.conf file must exists. But, it don't.") - sys.exit(1) - - def main(hostname): - check() - - heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) - - host_pool = HostPool(etcd_client, config['etcd']['host_prefix']) + host_pool = HostPool(shared.etcd_client) host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) assert host is not None, "No such host with name = {}".format(hostname) try: + heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() except Exception as e: - logger.info("No Need To Go Further. Our heartbeat updating mechanism is not working") logger.exception(e) - exit(-1) - - logger.info("%s Session Started %s", '*' * 5, '*' * 5) - - # It is seen that under heavy load, timeout event doesn't come - # in a predictive manner (which is intentional because we give - # higher priority to customer's requests) which delays heart - # beat update which in turn misunderstood by scheduler that the - # host is dead when it is actually alive. So, to ensure that we - # update the heart beat in a predictive manner we start Heart - # beat updating mechanism in separated thread + sys.exit("No Need To Go Further. ucloud-host heartbeat updating mechanism is not working") for events_iterator in [ - etcd_client.get_prefix(config['etcd']['request_prefix'], value_in_json=True), - etcd_client.watch_prefix(config['etcd']['request_prefix'], timeout=10, value_in_json=True), + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True), + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], timeout=10, value_in_json=True), ]: for request_event in events_iterator: request_event = RequestEntry(request_event) if request_event.type == "TIMEOUT": - maintenance(host) + vmm.maintenance(host) continue # If the event is directed toward me OR I am destination of a InitVMMigration if request_event.hostname == host.key or request_event.destination == host.key: logger.debug("VM Request: %s", request_event) - request_pool.client.client.delete(request_event.key) - vm_entry = vm_pool.get(request_event.uuid) + shared.request_pool.client.client.delete(request_event.key) + vm_entry = shared.vm_pool.get(request_event.uuid) if vm_entry: if request_event.type == RequestType.StartVM: - virtualmachine.start(vm_entry) + vmm.start(vm_entry) elif request_event.type == RequestType.StopVM: - virtualmachine.stop(vm_entry) + vmm.stop(vm_entry) elif request_event.type == RequestType.DeleteVM: - virtualmachine.delete(vm_entry) + vmm.delete(vm_entry) elif request_event.type == RequestType.InitVMMigration: - virtualmachine.start(vm_entry, host.key, find_free_port()) + vmm.start(vm_entry, host.key) elif request_event.type == RequestType.TransferVM: - virtualmachine.transfer(request_event) + vmm.transfer(request_event) else: logger.info("VM Entry missing") - logger.info("Running VMs %s", running_vms) + logger.info("Running VMs %s", vmm.running_vms) if __name__ == "__main__": diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index 4a7584a..cc06ce3 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -5,7 +5,6 @@ # https://qemu.weilnetz.de/doc/qemu-doc.html#pcsys_005fmonitor import os -import random import subprocess as sp import tempfile import time @@ -21,11 +20,12 @@ import sshtunnel from ucloud.common.helpers import get_ipv6_address from ucloud.common.request import RequestEntry, RequestType from ucloud.common.vm import VMEntry, VMStatus -from ucloud.config import (etcd_client, request_pool, - running_vms, vm_pool, config, - image_storage_handler) -from . import qmp +from ucloud.common.network import create_dev, delete_network_interface, find_free_port from ucloud.host import logger +from ucloud.shared import shared +from ucloud.settings import settings + +from . import qmp class VM: @@ -38,193 +38,22 @@ class VM: return "VM({})".format(self.key) -def delete_network_interface(iface): - try: - sp.check_output(['ip', 'link', 'del', iface]) - except Exception: - pass +def capture_all_exception(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + func(*args, **kwargs) + except Exception: + logger.info("Exception absorbed by captual_all_exception()") + logger.exception(func.__name__) - -def resolve_network(network_name, network_owner): - network = etcd_client.get(join_path(config['etcd']['network_prefix'], - network_owner, - network_name), - value_in_json=True) - return network - - -def delete_vm_network(vm_entry): - try: - for network in vm_entry.network: - network_name = network[0] - tap_mac = network[1] - tap_id = network[2] - - delete_network_interface('tap{}'.format(tap_id)) - - owners_vms = vm_pool.by_owner(vm_entry.owner) - owners_running_vms = vm_pool.by_status(VMStatus.running, - _vms=owners_vms) - - networks = map(lambda n: n[0], - map(lambda vm: vm.network, owners_running_vms) - ) - networks_in_use_by_user_vms = [vm[0] for vm in networks] - if network_name not in networks_in_use_by_user_vms: - network_entry = resolve_network(network[0], vm_entry.owner) - if network_entry: - network_type = network_entry.value["type"] - network_id = network_entry.value["id"] - if network_type == "vxlan": - delete_network_interface('br{}'.format(network_id)) - delete_network_interface('vxlan{}'.format(network_id)) - except Exception: - logger.exception("Exception in network interface deletion") - - -def create_dev(script, _id, dev, ip=None): - command = [script, _id, dev] - if ip: - command.append(ip) - try: - output = sp.check_output(command, stderr=sp.PIPE) - except Exception as e: - print(e.stderr) - return None - else: - return output.decode("utf-8").strip() - - -def create_vxlan_br_tap(_id, _dev, tap_id, ip=None): - network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') - vxlan = create_dev(script=os.path.join(network_script_base, 'create-vxlan.sh'), - _id=_id, dev=_dev) - if vxlan: - bridge = create_dev(script=os.path.join(network_script_base, 'create-bridge.sh'), - _id=_id, dev=vxlan, ip=ip) - if bridge: - tap = create_dev(script=os.path.join(network_script_base, 'create-tap.sh'), - _id=str(tap_id), dev=bridge) - if tap: - return tap - - -def random_bytes(num=6): - return [random.randrange(256) for _ in range(num)] - - -def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): - mac = random_bytes() - if oui: - if type(oui) == str: - oui = [int(chunk) for chunk in oui.split(separator)] - mac = oui + random_bytes(num=6 - len(oui)) - else: - if multicast: - mac[0] |= 1 # set bit 0 - else: - mac[0] &= ~1 # clear bit 0 - if uaa: - mac[0] &= ~(1 << 1) # clear bit 1 - else: - mac[0] |= 1 << 1 # set bit 1 - return separator.join(byte_fmt % b for b in mac) - - -def update_radvd_conf(etcd_client): - network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') - - networks = { - net.value['ipv6']: net.value['id'] - for net in etcd_client.get_prefix('/v1/network/', value_in_json=True) - if net.value.get('ipv6') - } - radvd_template = open(os.path.join(network_script_base, - 'radvd-template.conf'), 'r').read() - radvd_template = Template(radvd_template) - - content = [radvd_template.safe_substitute(bridge='br{}'.format(networks[net]), - prefix=net) - for net in networks if networks.get(net)] - - with open('/etc/radvd.conf', 'w') as radvd_conf: - radvd_conf.writelines(content) - try: - sp.check_output(['systemctl', 'restart', 'radvd']) - except Exception: - sp.check_output(['service', 'radvd', 'restart']) - - -def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, migration_port=None): - threads_per_core = 1 - vm_memory = int(bitmath.parse_string_unsafe(vm_entry.specs["ram"]).to_MB()) - vm_cpus = int(vm_entry.specs["cpu"]) - vm_uuid = vm_entry.uuid - vm_networks = vm_entry.network - - command = "-name {}_{}".format(vm_entry.owner, vm_entry.name) - - command += " -drive file={},format=raw,if=virtio,cache=none".format( - image_storage_handler.qemu_path_string(vm_uuid) - ) - command += " -device virtio-rng-pci -vnc unix:{}".format(vnc_sock_filename) - command += " -m {} -smp cores={},threads={}".format( - vm_memory, vm_cpus, threads_per_core - ) - - if migration: - command += " -incoming tcp:[::]:{}".format(migration_port) - - tap = None - for network_mac_and_tap in vm_networks: - network_name, mac, tap = network_mac_and_tap - - _key = os.path.join(config['etcd']['network_prefix'], vm_entry.owner, network_name) - network = etcd_client.get(_key, value_in_json=True) - network_type = network.value["type"] - network_id = str(network.value["id"]) - network_ipv6 = network.value["ipv6"] - - if network_type == "vxlan": - tap = create_vxlan_br_tap(_id=network_id, - _dev=config['network']['vxlan_phy_dev'], - tap_id=tap, - ip=network_ipv6) - update_radvd_conf(etcd_client) - - command += " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" \ - " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" \ - .format(tap=tap, net_id=network_id, mac=mac) - - return command.split(" ") - - -def create_vm_object(vm_entry, migration=False, migration_port=None): - # NOTE: If migration suddenly stop working, having different - # VNC unix filename on source and destination host can - # be a possible cause of it. - - # REQUIREMENT: Use Unix Socket instead of TCP Port for VNC - vnc_sock_file = tempfile.NamedTemporaryFile() - - qemu_args = get_start_command_args( - vm_entry=vm_entry, - vnc_sock_filename=vnc_sock_file.name, - migration=migration, - migration_port=migration_port, - ) - qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64", args=qemu_args) - return VM(vm_entry.key, qemu_machine, vnc_sock_file) - - -def get_vm(vm_list: list, vm_key) -> Union[VM, None]: - return next((vm for vm in vm_list if vm.key == vm_key), None) + return wrapper def need_running_vm(func): @wraps(func) - def wrapper(e): - vm = get_vm(running_vms, e.key) + def wrapper(self, e): + vm = self.get_vm(self.running_vms, e.key) if vm: try: status = vm.handle.command("query-status") @@ -242,143 +71,336 @@ def need_running_vm(func): return wrapper -def create(vm_entry: VMEntry): - if image_storage_handler.is_vm_image_exists(vm_entry.uuid): - # File Already exists. No Problem Continue - logger.debug("Image for vm %s exists", vm_entry.uuid) - else: - vm_hdd = int(bitmath.parse_string_unsafe(vm_entry.specs["os-ssd"]).to_MB()) - if image_storage_handler.make_vm_image(src=vm_entry.image_uuid, dest=vm_entry.uuid): - if not image_storage_handler.resize_vm_image(path=vm_entry.uuid, size=vm_hdd): - vm_entry.status = VMStatus.error - else: - logger.info("New VM Created") +class VMM: + def __init__(self): + self.etcd_client = shared.etcd_client + self.storage_handler = shared.storage_handler + self.running_vms = [] + def get_start_command_args(self, vm_entry, vnc_sock_filename: str, migration=False, migration_port=None): + threads_per_core = 1 + vm_memory = int(bitmath.parse_string_unsafe(vm_entry.specs['ram']).to_MB()) + vm_cpus = int(vm_entry.specs['cpu']) + vm_uuid = vm_entry.uuid + vm_networks = vm_entry.network -def start(vm_entry: VMEntry, destination_host_key=None, migration_port=None): - _vm = get_vm(running_vms, vm_entry.key) + command = '-name {}_{}'.format(vm_entry.owner, vm_entry.name) - # VM already running. No need to proceed further. - if _vm: - logger.info("VM %s already running" % vm_entry.uuid) - return - else: - logger.info("Trying to start %s" % vm_entry.uuid) - if destination_host_key: - launch_vm(vm_entry, migration=True, migration_port=migration_port, - destination_host_key=destination_host_key) - else: - create(vm_entry) - launch_vm(vm_entry) - - -@need_running_vm -def stop(vm_entry): - vm = get_vm(running_vms, vm_entry.key) - vm.handle.shutdown() - if not vm.handle.is_running(): - vm_entry.add_log("Shutdown successfully") - vm_entry.declare_stopped() - vm_pool.put(vm_entry) - running_vms.remove(vm) - delete_vm_network(vm_entry) - - -def delete(vm_entry): - logger.info("Deleting VM | %s", vm_entry) - stop(vm_entry) - - if image_storage_handler.is_vm_image_exists(vm_entry.uuid): - r_status = image_storage_handler.delete_vm_image(vm_entry.uuid) - if r_status: - etcd_client.client.delete(vm_entry.key) - else: - etcd_client.client.delete(vm_entry.key) - -def transfer(request_event): - # This function would run on source host i.e host on which the vm - # is running initially. This host would be responsible for transferring - # vm state to destination host. - - _host, _port = request_event.parameters["host"], request_event.parameters["port"] - _uuid = request_event.uuid - _destination = request_event.destination_host_key - vm = get_vm(running_vms, join_path(config['etcd']['vm_prefix'], _uuid)) - - if vm: - tunnel = sshtunnel.SSHTunnelForwarder( - _host, - ssh_username=config['ssh']['username'], - ssh_pkey=config['ssh']['private_key_path'], - remote_bind_address=("127.0.0.1", _port), - ssh_proxy_enabled=True, - ssh_proxy=(_host, 22) + command += ' -drive file={},format=raw,if=virtio,cache=none'.format( + self.storage_handler.qemu_path_string(vm_uuid) ) - try: - tunnel.start() - except sshtunnel.BaseSSHTunnelForwarderError: - logger.exception("Couldn't establish connection to (%s, 22)", _host) + command += ' -device virtio-rng-pci -vnc unix:{}'.format(vnc_sock_filename) + command += ' -m {} -smp cores={},threads={}'.format( + vm_memory, vm_cpus, threads_per_core + ) + + if migration: + command += ' -incoming tcp:[::]:{}'.format(migration_port) + + for network_mac_and_tap in vm_networks: + network_name, mac, tap = network_mac_and_tap + + _key = os.path.join(settings['etcd']['network_prefix'], vm_entry.owner, network_name) + network = self.etcd_client.get(_key, value_in_json=True) + network_type = network.value["type"] + network_id = str(network.value["id"]) + network_ipv6 = network.value["ipv6"] + + if network_type == "vxlan": + tap = create_vxlan_br_tap(_id=network_id, + _dev=settings['network']['vxlan_phy_dev'], + tap_id=tap, + ip=network_ipv6) + all_networks = self.etcd_client.get_prefix('/v1/network/', value_in_json=True) + update_radvd_conf(all_networks) + + command += " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" \ + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" \ + .format(tap=tap, net_id=network_id, mac=mac) + + return command.split(" ") + + def create_vm_object(self, vm_entry, migration=False, migration_port=None): + vnc_sock_file = tempfile.NamedTemporaryFile() + + qemu_args = self.get_start_command_args( + vm_entry=vm_entry, + vnc_sock_filename=vnc_sock_file.name, + migration=migration, + migration_port=migration_port, + ) + qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64", args=qemu_args) + return VM(vm_entry.key, qemu_machine, vnc_sock_file) + + @staticmethod + def get_vm(vm_list: list, vm_key) -> Union[VM, None]: + return next((vm for vm in vm_list if vm.key == vm_key), None) + + @capture_all_exception + def create(self, vm_entry: VMEntry): + if self.storage_handler.is_vm_image_exists(vm_entry.uuid): + # File Already exists. No Problem Continue + logger.debug("Image for vm %s exists", vm_entry.uuid) + return None else: - vm.handle.command( - "migrate", uri="tcp:0.0.0.0:{}".format(tunnel.local_bind_port) - ) + vm_hdd = int(bitmath.parse_string_unsafe(vm_entry.specs["os-ssd"]).to_MB()) + if self.storage_handler.make_vm_image(src=vm_entry.image_uuid, dest=vm_entry.uuid): + if not self.storage_handler.resize_vm_image(path=vm_entry.uuid, size=vm_hdd): + vm_entry.status = VMStatus.error + else: + logger.info("New VM Created") - status = vm.handle.command("query-migrate")["status"] - while status not in ["failed", "completed"]: - time.sleep(2) - status = vm.handle.command("query-migrate")["status"] + @capture_all_exception + def start(self, vm_entry: VMEntry, destination_host_key=None): + _vm = self.get_vm(self.running_vms, vm_entry.key) - with vm_pool.get_put(request_event.uuid) as source_vm: - if status == "failed": - source_vm.add_log("Migration Failed") - elif status == "completed": - # If VM is successfully migrated then shutdown the VM - # on this host and update hostname to destination host key - source_vm.add_log("Successfully migrated") - source_vm.hostname = _destination - running_vms.remove(vm) - vm.handle.shutdown() - source_vm.in_migration = False # VM transfer finished - finally: - tunnel.close() + # VM already running. No need to proceed further. + if _vm: + logger.info("VM %s already running" % vm_entry.uuid) + return + else: + logger.info("Trying to start %s" % vm_entry.uuid) + if destination_host_key: + migration_port = find_free_port() + self.launch_vm(vm_entry, migration=True, migration_port=migration_port, + destination_host_key=destination_host_key) + else: + self.create(vm_entry) + self.launch_vm(vm_entry) - -def launch_vm(vm_entry, migration=False, migration_port=None, destination_host_key=None): - logger.info("Starting %s" % vm_entry.key) - - vm = create_vm_object(vm_entry, migration=migration, migration_port=migration_port) - try: - vm.handle.launch() - except Exception: - logger.exception("Error Occured while starting VM") + @need_running_vm + @capture_all_exception + def stop(self, vm_entry): + vm = self.get_vm(self.running_vms, vm_entry.key) vm.handle.shutdown() + if not vm.handle.is_running(): + vm_entry.add_log("Shutdown successfully") + vm_entry.declare_stopped() + shared.vm_pool.put(vm_entry) + self.running_vms.remove(vm) + delete_vm_network(vm_entry) - if migration: - # We don't care whether MachineError or any other error occurred - pass + @capture_all_exception + def delete(self, vm_entry): + logger.info("Deleting VM | %s", vm_entry) + self.stop(vm_entry) + + if self.storage_handler.is_vm_image_exists(vm_entry.uuid): + r_status = self.storage_handler.delete_vm_image(vm_entry.uuid) + if r_status: + shared.etcd_client.client.delete(vm_entry.key) else: - # Error during typical launch of a vm - vm.handle.shutdown() - vm_entry.declare_killed() - vm_pool.put(vm_entry) - else: - vm_entry.vnc_socket = vm.vnc_socket_file.name - running_vms.append(vm) + shared.etcd_client.client.delete(vm_entry.key) - if migration: - vm_entry.in_migration = True - r = RequestEntry.from_scratch( - type=RequestType.TransferVM, - hostname=vm_entry.hostname, - parameters={"host": get_ipv6_address(), "port": migration_port}, - uuid=vm_entry.uuid, - destination_host_key=destination_host_key, - request_prefix=config['etcd']['request_prefix'] + @capture_all_exception + def transfer(self, request_event): + # This function would run on source host i.e host on which the vm + # is running initially. This host would be responsible for transferring + # vm state to destination host. + + _host, _port = request_event.parameters["host"], request_event.parameters["port"] + _uuid = request_event.uuid + _destination = request_event.destination_host_key + vm = self.get_vm(self.running_vms, join_path(settings['etcd']['vm_prefix'], _uuid)) + + if vm: + tunnel = sshtunnel.SSHTunnelForwarder( + _host, + ssh_username=settings['ssh']['username'], + ssh_pkey=settings['ssh']['private_key_path'], + remote_bind_address=("127.0.0.1", _port), + ssh_proxy_enabled=True, + ssh_proxy=(_host, 22) ) - request_pool.put(r) - else: - # Typical launching of a vm - vm_entry.status = VMStatus.running - vm_entry.add_log("Started successfully") + try: + tunnel.start() + except sshtunnel.BaseSSHTunnelForwarderError: + logger.exception("Couldn't establish connection to (%s, 22)", _host) + else: + vm.handle.command( + "migrate", uri="tcp:0.0.0.0:{}".format(tunnel.local_bind_port) + ) - vm_pool.put(vm_entry) + status = vm.handle.command("query-migrate")["status"] + while status not in ["failed", "completed"]: + time.sleep(2) + status = vm.handle.command("query-migrate")["status"] + + with shared.vm_pool.get_put(request_event.uuid) as source_vm: + if status == "failed": + source_vm.add_log("Migration Failed") + elif status == "completed": + # If VM is successfully migrated then shutdown the VM + # on this host and update hostname to destination host key + source_vm.add_log("Successfully migrated") + source_vm.hostname = _destination + self.running_vms.remove(vm) + vm.handle.shutdown() + source_vm.in_migration = False # VM transfer finished + finally: + tunnel.close() + + @capture_all_exception + def launch_vm(self, vm_entry, migration=False, migration_port=None, destination_host_key=None): + logger.info("Starting %s" % vm_entry.key) + + vm = self.create_vm_object(vm_entry, migration=migration, migration_port=migration_port) + try: + vm.handle.launch() + except Exception: + logger.exception("Error Occured while starting VM") + vm.handle.shutdown() + + if migration: + # We don't care whether MachineError or any other error occurred + pass + else: + # Error during typical launch of a vm + vm.handle.shutdown() + vm_entry.declare_killed() + shared.vm_pool.put(vm_entry) + else: + vm_entry.vnc_socket = vm.vnc_socket_file.name + self.running_vms.append(vm) + + if migration: + vm_entry.in_migration = True + r = RequestEntry.from_scratch( + type=RequestType.TransferVM, + hostname=vm_entry.hostname, + parameters={"host": get_ipv6_address(), "port": migration_port}, + uuid=vm_entry.uuid, + destination_host_key=destination_host_key, + request_prefix=settings['etcd']['request_prefix'] + ) + shared.request_pool.put(r) + else: + # Typical launching of a vm + vm_entry.status = VMStatus.running + vm_entry.add_log("Started successfully") + + shared.vm_pool.put(vm_entry) + + @capture_all_exception + def maintenance(self, host): + # To capture vm running according to running_vms list + + # This is to capture successful migration of a VM. + # Suppose, this host is running "vm1" and user initiated + # request to migrate this "vm1" to some other host. On, + # successful migration the destination host would set + # the vm hostname to itself. Thus, we are checking + # whether this host vm is successfully migrated. If yes + # then we shutdown "vm1" on this host. + logger.debug("Starting Maintenance!!") + to_be_removed = [] + for running_vm in self.running_vms: + with shared.vm_pool.get_put(running_vm.key) as vm_entry: + if vm_entry.hostname != host.key and not vm_entry.in_migration: + running_vm.handle.shutdown() + logger.info("VM migration not completed successfully.") + to_be_removed.append(running_vm) + + for r in to_be_removed: + self.running_vms.remove(r) + + # To check vm running according to etcd entries + alleged_running_vms = shared.vm_pool.by_status("RUNNING", shared.vm_pool.by_host(host.key)) + + for vm_entry in alleged_running_vms: + _vm = self.get_vm(self.running_vms, vm_entry.key) + # Whether, the allegedly running vm is in our + # running_vms list or not if it is said to be + # running on this host but it is not then we + # need to shut it down + + # This is to capture poweroff/shutdown of a VM + # initiated by user inside VM. OR crash of VM by some + # user running process + if (_vm and not _vm.handle.is_running()) or not _vm: + logger.debug("_vm = %s, is_running() = %s" % (_vm, _vm.handle.is_running())) + vm_entry.add_log("""{} is not running but is said to be running. + So, shutting it down and declare it killed""".format(vm_entry.key)) + vm_entry.declare_killed() + shared.vm_pool.put(vm_entry) + if _vm: + self.running_vms.remove(_vm) + + +def resolve_network(network_name, network_owner): + network = shared.etcd_client.get(join_path(settings['etcd']['network_prefix'], + network_owner, + network_name), + value_in_json=True) + return network + + +def delete_vm_network(vm_entry): + try: + for network in vm_entry.network: + network_name = network[0] + tap_mac = network[1] + tap_id = network[2] + + delete_network_interface('tap{}'.format(tap_id)) + + owners_vms = shared.vm_pool.by_owner(vm_entry.owner) + owners_running_vms = shared.vm_pool.by_status(VMStatus.running, + _vms=owners_vms) + + networks = map( + lambda n: n[0], map(lambda vm: vm.network, owners_running_vms) + ) + networks_in_use_by_user_vms = [vm[0] for vm in networks] + if network_name not in networks_in_use_by_user_vms: + network_entry = resolve_network(network[0], vm_entry.owner) + if network_entry: + network_type = network_entry.value["type"] + network_id = network_entry.value["id"] + if network_type == "vxlan": + delete_network_interface('br{}'.format(network_id)) + delete_network_interface('vxlan{}'.format(network_id)) + except Exception: + logger.exception("Exception in network interface deletion") + + +def create_vxlan_br_tap(_id, _dev, tap_id, ip=None): + network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') + vxlan = create_dev(script=os.path.join(network_script_base, 'create-vxlan.sh'), + _id=_id, dev=_dev) + if vxlan: + bridge = create_dev(script=os.path.join(network_script_base, 'create-bridge.sh'), + _id=_id, dev=vxlan, ip=ip) + if bridge: + tap = create_dev(script=os.path.join(network_script_base, 'create-tap.sh'), + _id=str(tap_id), dev=bridge) + if tap: + return tap + + +def update_radvd_conf(all_networks): + network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') + + networks = { + net.value['ipv6']: net.value['id'] + for net in all_networks + if net.value.get('ipv6') + } + radvd_template = open(os.path.join(network_script_base, + 'radvd-template.conf'), 'r').read() + radvd_template = Template(radvd_template) + + content = [ + radvd_template.safe_substitute( + bridge='br{}'.format(networks[net]), + prefix=net + ) + for net in networks if networks.get(net) + ] + + with open('/etc/radvd.conf', 'w') as radvd_conf: + radvd_conf.writelines(content) + try: + sp.check_output(['systemctl', 'restart', 'radvd']) + except Exception: + sp.check_output(['service', 'radvd', 'restart']) diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index 135f8cb..d164ea3 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -5,7 +5,8 @@ import sys from os.path import isdir from os.path import join as join_path -from ucloud.config import etcd_client, config, image_storage_handler +from ucloud.settings import settings +from ucloud.shared import shared from ucloud.imagescanner import logger @@ -22,9 +23,9 @@ def qemu_img_type(path): def check(): """ check whether settings are sane, refuse to start if they aren't """ - if config['storage']['storage_backend'] == 'filesystem' and not isdir(config['storage']['image_dir']): + if settings['storage']['storage_backend'] == 'filesystem' and not isdir(settings['storage']['image_dir']): sys.exit("You have set STORAGE_BACKEND to filesystem, but " - "{} does not exist. Refusing to start".format(config['storage']['image_dir']) + "{} does not exist. Refusing to start".format(settings['storage']['image_dir']) ) try: @@ -36,7 +37,7 @@ def check(): def main(): # We want to get images entries that requests images to be created - images = etcd_client.get_prefix(config['etcd']['image_prefix'], value_in_json=True) + images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True) images_to_be_created = list(filter(lambda im: im.value['status'] == 'TO_BE_CREATED', images)) for image in images_to_be_created: @@ -45,9 +46,9 @@ def main(): image_owner = image.value['owner'] image_filename = image.value['filename'] image_store_name = image.value['store_name'] - image_full_path = join_path(config['storage']['file_dir'], image_owner, image_filename) + image_full_path = join_path(settings['storage']['file_dir'], image_owner, image_filename) - image_stores = etcd_client.get_prefix(config['etcd']['image_store_prefix'], + image_stores = shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix'], value_in_json=True) user_image_store = next(filter( lambda s, store_name=image_store_name: s.value["name"] == store_name, @@ -71,18 +72,18 @@ def main(): logger.exception(e) else: # Import and Protect - r_status = image_storage_handler.import_image(src="image.raw", + r_status = shared.storage_handler.import_image(src="image.raw", dest=image_uuid, protect=True) if r_status: # Everything is successfully done image.value["status"] = "CREATED" - etcd_client.put(image.key, json.dumps(image.value)) + shared.etcd_client.put(image.key, json.dumps(image.value)) else: # The user provided image is either not found or of invalid format image.value["status"] = "INVALID_IMAGE" - etcd_client.put(image.key, json.dumps(image.value)) + shared.etcd_client.put(image.key, json.dumps(image.value)) try: os.remove("image.raw") diff --git a/ucloud/metadata/main.py b/ucloud/metadata/main.py index 16b7c6d..5526084 100644 --- a/ucloud/metadata/main.py +++ b/ucloud/metadata/main.py @@ -2,15 +2,15 @@ import os from flask import Flask, request from flask_restful import Resource, Api - -from ucloud.config import etcd_client, config, vm_pool +from ucloud.settings import settings +from ucloud.shared import shared app = Flask(__name__) api = Api(app) def get_vm_entry(mac_addr): - return next(filter(lambda vm: mac_addr in list(zip(*vm.network))[1], vm_pool.vms), None) + return next(filter(lambda vm: mac_addr in list(zip(*vm.network))[1], shared.vm_pool.vms), None) # https://stackoverflow.com/questions/37140846/how-to-convert-ipv6-link-local-address-to-mac-address-in-python @@ -43,10 +43,10 @@ class Root(Resource): if not data: return {'message': 'Metadata for such VM does not exists.'}, 404 else: - etcd_key = os.path.join(config['etcd']['user_prefix'], + etcd_key = os.path.join(settings['etcd']['user_prefix'], data.value['owner_realm'], data.value['owner'], 'key') - etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True) + etcd_entry = shared.etcd_client.get_prefix(etcd_key, value_in_json=True) user_personal_ssh_keys = [key.value for key in etcd_entry] data.value['metadata']['ssh-keys'] += user_personal_ssh_keys return data.value['metadata'], 200 diff --git a/ucloud/scheduler/helper.py b/ucloud/scheduler/helper.py index 560bdbc..643e8e9 100755 --- a/ucloud/scheduler/helper.py +++ b/ucloud/scheduler/helper.py @@ -6,7 +6,8 @@ import bitmath from ucloud.common.host import HostStatus from ucloud.common.request import RequestEntry, RequestType from ucloud.common.vm import VMStatus -from ucloud.config import vm_pool, host_pool, request_pool, config +from ucloud.shared import shared +from ucloud.settings import settings def accumulated_specs(vms_specs): @@ -46,14 +47,14 @@ class NoSuitableHostFound(Exception): def get_suitable_host(vm_specs, hosts=None): if hosts is None: - hosts = host_pool.by_status(HostStatus.alive) + hosts = shared.host_pool.by_status(HostStatus.alive) for host in hosts: # Filter them by host_name - vms = vm_pool.by_host(host.key) + vms = shared.vm_pool.by_host(host.key) # Filter them by status - vms = vm_pool.by_status(VMStatus.running, vms) + vms = shared.vm_pool.by_status(VMStatus.running, vms) running_vms_specs = [vm.specs for vm in vms] @@ -75,7 +76,7 @@ def get_suitable_host(vm_specs, hosts=None): def dead_host_detection(): # Bring out your dead! - Monty Python and the Holy Grail - hosts = host_pool.by_status(HostStatus.alive) + hosts = shared.host_pool.by_status(HostStatus.alive) dead_hosts_keys = [] for host in hosts: @@ -89,25 +90,25 @@ def dead_host_detection(): def dead_host_mitigation(dead_hosts_keys): for host_key in dead_hosts_keys: - host = host_pool.get(host_key) + host = shared.host_pool.get(host_key) host.declare_dead() - vms_hosted_on_dead_host = vm_pool.by_host(host_key) + vms_hosted_on_dead_host = shared.vm_pool.by_host(host_key) for vm in vms_hosted_on_dead_host: vm.declare_killed() - vm_pool.put(vm) - host_pool.put(host) + shared.vm_pool.put(vm) + shared.host_pool.put(host) def assign_host(vm): vm.hostname = get_suitable_host(vm.specs) - vm_pool.put(vm) + shared.vm_pool.put(vm) r = RequestEntry.from_scratch(type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=config['etcd']['request_prefix']) - request_pool.put(r) + request_prefix=settings['etcd']['request_prefix']) + shared.request_pool.put(r) vm.log.append("VM scheduled for starting") return vm.hostname diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index 49d6291..3412545 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -5,8 +5,8 @@ # maybe expose a prometheus compatible output from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import etcd_client -from ucloud.config import host_pool, request_pool, vm_pool, config +from ucloud.shared import shared +from ucloud.settings import settings from .helper import (get_suitable_host, dead_host_mitigation, dead_host_detection, assign_host, NoSuitableHostFound) from . import logger @@ -16,8 +16,8 @@ def main(): pending_vms = [] for request_iterator in [ - etcd_client.get_prefix(config['etcd']['request_prefix'], value_in_json=True), - etcd_client.watch_prefix(config['etcd']['request_prefix'], timeout=5, value_in_json=True), + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True), + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], timeout=5, value_in_json=True), ]: for request_event in request_iterator: request_entry = RequestEntry(request_event) @@ -44,17 +44,17 @@ def main(): r = RequestEntry.from_scratch(type="ScheduleVM", uuid=pending_vm_entry.uuid, hostname=pending_vm_entry.hostname, - request_prefix=config['etcd']['request_prefix']) - request_pool.put(r) + request_prefix=settings['etcd']['request_prefix']) + shared.request_pool.put(r) elif request_entry.type == RequestType.ScheduleVM: logger.debug("%s, %s", request_entry.key, request_entry.value) - vm_entry = vm_pool.get(request_entry.uuid) + vm_entry = shared.vm_pool.get(request_entry.uuid) if vm_entry is None: logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) continue - etcd_client.client.delete(request_entry.key) # consume Request + shared.etcd_client.client.delete(request_entry.key) # consume Request # If the Request is about a VM which is labelled as "migration" # and has a destination @@ -62,7 +62,7 @@ def main(): and hasattr(request_entry, "destination") and request_entry.destination: try: get_suitable_host(vm_specs=vm_entry.specs, - hosts=[host_pool.get(request_entry.destination)]) + hosts=[shared.host_pool.get(request_entry.destination)]) except NoSuitableHostFound: logger.info("Requested destination host doesn't have enough capacity" "to hold %s" % vm_entry.uuid) @@ -70,8 +70,8 @@ def main(): r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, uuid=request_entry.uuid, destination=request_entry.destination, - request_prefix=config['etcd']['request_prefix']) - request_pool.put(r) + request_prefix=settings['etcd']['request_prefix']) + shared.request_pool.put(r) # If the Request is about a VM that just want to get started/created else: @@ -81,7 +81,7 @@ def main(): assign_host(vm_entry) except NoSuitableHostFound: vm_entry.add_log("Can't schedule VM. No Resource Left.") - vm_pool.put(vm_entry) + shared.vm_pool.put(vm_entry) pending_vms.append(vm_entry) logger.info("No Resource Left. Emailing admin....") diff --git a/ucloud/scheduler/main.py.old b/ucloud/scheduler/main.py.old deleted file mode 100755 index e2c975a..0000000 --- a/ucloud/scheduler/main.py.old +++ /dev/null @@ -1,93 +0,0 @@ -# TODO -# 1. send an email to an email address defined by env['admin-email'] -# if resources are finished -# 2. Introduce a status endpoint of the scheduler - -# maybe expose a prometheus compatible output - -from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import etcd_client -from ucloud.config import host_pool, request_pool, vm_pool, env_vars -from .helper import (get_suitable_host, dead_host_mitigation, dead_host_detection, - assign_host, NoSuitableHostFound) -from . import logger - - -def main(): - logger.info("%s SESSION STARTED %s", '*' * 5, '*' * 5) - - pending_vms = [] - - for request_iterator in [ - etcd_client.get_prefix(env_vars.get('REQUEST_PREFIX'), value_in_json=True), - etcd_client.watch_prefix(env_vars.get('REQUEST_PREFIX'), timeout=5, value_in_json=True), - ]: - for request_event in request_iterator: - request_entry = RequestEntry(request_event) - # Never Run time critical mechanism inside timeout - # mechanism because timeout mechanism only comes - # when no other event is happening. It means under - # heavy load there would not be a timeout event. - if request_entry.type == "TIMEOUT": - - # Detect hosts that are dead and set their status - # to "DEAD", and their VMs' status to "KILLED" - dead_hosts = dead_host_detection() - if dead_hosts: - logger.debug("Dead hosts: %s", dead_hosts) - dead_host_mitigation(dead_hosts) - - # If there are VMs that weren't assigned a host - # because there wasn't a host available which - # meets requirement of that VM then we would - # create a new ScheduleVM request for that VM - # on our behalf. - while pending_vms: - pending_vm_entry = pending_vms.pop() - r = RequestEntry.from_scratch(type="ScheduleVM", - uuid=pending_vm_entry.uuid, - hostname=pending_vm_entry.hostname, - request_prefix=env_vars.get("REQUEST_PREFIX")) - request_pool.put(r) - - elif request_entry.type == RequestType.ScheduleVM: - logger.debug("%s, %s", request_entry.key, request_entry.value) - - vm_entry = vm_pool.get(request_entry.uuid) - if vm_entry is None: - logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) - continue - etcd_client.client.delete(request_entry.key) # consume Request - - # If the Request is about a VM which is labelled as "migration" - # and has a destination - if hasattr(request_entry, "migration") and request_entry.migration \ - and hasattr(request_entry, "destination") and request_entry.destination: - try: - get_suitable_host(vm_specs=vm_entry.specs, - hosts=[host_pool.get(request_entry.destination)]) - except NoSuitableHostFound: - logger.info("Requested destination host doesn't have enough capacity" - "to hold %s" % vm_entry.uuid) - else: - r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, - uuid=request_entry.uuid, - destination=request_entry.destination, - request_prefix=env_vars.get("REQUEST_PREFIX")) - request_pool.put(r) - - # If the Request is about a VM that just want to get started/created - else: - # assign_host only returns None when we couldn't be able to assign - # a host to a VM because of resource constraints - try: - assign_host(vm_entry) - except NoSuitableHostFound: - vm_entry.add_log("Can't schedule VM. No Resource Left.") - vm_pool.put(vm_entry) - - pending_vms.append(vm_entry) - logger.info("No Resource Left. Emailing admin....") - - -if __name__ == "__main__": - main() diff --git a/ucloud/settings/__init__.py b/ucloud/settings/__init__.py index 5f29c41..2c77300 100644 --- a/ucloud/settings/__init__.py +++ b/ucloud/settings/__init__.py @@ -23,28 +23,28 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): def __init__(self, config_key='/uncloud/config/'): conf_name = 'ucloud.conf' - conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud') - config_file = os.path.join(conf_dir, conf_name) + conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/ucloud/')) + self.config_file = os.path.join(conf_dir, conf_name) self.config_parser = CustomConfigParser(allow_no_value=True) self.config_key = config_key self.read_internal_values() - self.read_config_file_values(config_file) + self.config_parser.read(self.config_file) - self.etcd_wrapper_args = tuple() - self.etcd_wrapper_kwargs = { - 'host': self.config_parser['etcd']['url'], - 'port': self.config_parser['etcd']['port'], - 'ca_cert': self.config_parser['etcd']['ca_cert'], - 'cert_cert': self.config_parser['etcd']['cert_cert'], - 'cert_key': self.config_parser['etcd']['cert_key'] - } - - def get_etcd_client(self): - args = self.etcd_wrapper_args - kwargs = self.etcd_wrapper_kwargs + args = tuple() + try: + kwargs = { + 'host': self.config_parser.get('etcd', 'url'), + 'port': self.config_parser.get('etcd', 'port'), + 'ca_cert': self.config_parser.get('etcd', 'ca_cert'), + 'cert_cert': self.config_parser.get('etcd','cert_cert'), + 'cert_key': self.config_parser.get('etcd','cert_key') + } + except configparser.Error as err: + raise configparser.Error('{} in config file {}'.format(err.message, self.config_file)) from err + return Etcd3Wrapper(*args, **kwargs) def read_internal_values(self): @@ -78,9 +78,11 @@ class Settings(object): if config_from_etcd: self.config_parser.read_dict(config_from_etcd.value) else: - return - sys.exit("No settings found in etcd at key {}".format(self.config_key)) - + raise KeyError("Key '{}' not found in etcd".format(self.config_key)) + def __getitem__(self, key): self.read_values_from_etcd() return self.config_parser[key] + + +settings = Settings() diff --git a/ucloud/shared/__init__.py b/ucloud/shared/__init__.py new file mode 100644 index 0000000..7a296e9 --- /dev/null +++ b/ucloud/shared/__init__.py @@ -0,0 +1,30 @@ +from ucloud.settings import settings +from ucloud.common.vm import VmPool +from ucloud.common.host import HostPool +from ucloud.common.request import RequestPool +from ucloud.common.storage_handlers import get_storage_handler + + +class Shared: + @property + def etcd_client(self): + return settings.get_etcd_client() + + @property + def host_pool(self): + return HostPool(self.etcd_client, settings['etcd']['host_prefix']) + + @property + def vm_pool(self): + return VmPool(self.etcd_client, settings['etcd']['vm_prefix']) + + @property + def request_pool(self): + return RequestPool(self.etcd_client, settings['etcd']['request_prefix']) + + @property + def storage_handler(self): + return get_storage_handler() + + +shared = Shared() From 88b4d34e1a7d2da47229303ce501a9b899a7cfe8 Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 22 Dec 2019 12:33:59 +0500 Subject: [PATCH 039/163] workaround of setuptools bug that fails to install Flask without version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e273d68..5a624db 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ setup(name='ucloud', packages=find_packages(), install_requires=[ 'requests', - 'Flask', + 'Flask>=1.1.1', 'flask-restful', 'bitmath', 'pyotp', From e4d2c98fb5822e510f6bb979a807765e95d8d6ec Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 22 Dec 2019 13:14:42 +0500 Subject: [PATCH 040/163] Better logging. Errors without stacktrace are now printed to stderr --- scripts/ucloud | 47 +++++++++++++++++++++++-------- ucloud/common/etcd_wrapper.py | 1 - ucloud/common/network.py | 1 + ucloud/common/storage_handlers.py | 1 - 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/scripts/ucloud b/scripts/ucloud index 7f3ef3a..f741663 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -6,6 +6,8 @@ import importlib import multiprocessing as mp import sys +from logging.handlers import SysLogHandler + from ucloud.configure.main import configure_parser @@ -14,10 +16,20 @@ def exception_hook(exc_type, exc_value, exc_traceback): 'Uncaught exception', exc_info=(exc_type, exc_value, exc_traceback) ) - print('Error: ', end='') - print(exc_type, exc_value, exc_traceback) + # print('Error: ', end='') + # print(exc_type, exc_value, exc_traceback) +class NoTracebackStreamHandler(logging.StreamHandler): + def handle(self, record): + info, cache = record.exc_info, record.exc_text + record.exc_info, record.exc_text = None, None + try: + super().handle(record) + finally: + record.exc_info = info + record.exc_text = cache + if __name__ == '__main__': sys.excepthook = exception_hook @@ -44,12 +56,26 @@ if __name__ == '__main__': if not args.command: arg_parser.print_help() else: - logging.basicConfig( - level=logging.DEBUG, - format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s', - handlers=[logging.handlers.SysLogHandler(address = '/dev/log')] - ) - logger = logging.getLogger("ucloud") + # Setting up root logger + logger = logging.getLogger('ucloud') + + syslog_handler = SysLogHandler(address='/dev/log') + syslog_handler.setLevel(logging.DEBUG) + syslog_formatter = logging.Formatter('%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s') + syslog_handler.setFormatter(syslog_formatter) + + stream_handler = NoTracebackStreamHandler() + stream_handler.setLevel(logging.WARNING) + stream_formatter = logging.Formatter('%(message)s') + stream_handler.setFormatter(stream_formatter) + + logger.addHandler(syslog_handler) + logger.addHandler(stream_handler) + + # if we start etcd in seperate process with default settings + # i.e inheriting few things from parent process etcd3 module + # errors out, so the following command configure multiprocessing + # module to not inherit anything from parent. mp.set_start_method('spawn') arguments = vars(args) @@ -58,6 +84,5 @@ if __name__ == '__main__': mod = importlib.import_module("ucloud.{}.main".format(name)) main = getattr(mod, "main") main(**arguments) - except Exception as e: - logger.exception('Error') - print(e) \ No newline at end of file + except Exception as err: + logger.exception(err) diff --git a/ucloud/common/etcd_wrapper.py b/ucloud/common/etcd_wrapper.py index e249e6c..91149b8 100644 --- a/ucloud/common/etcd_wrapper.py +++ b/ucloud/common/etcd_wrapper.py @@ -33,7 +33,6 @@ def readable_errors(func): except etcd3.exceptions.ConnectionTimeoutError as err: raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout') from err except Exception: - print('Some error occurred, most probably it is etcd that is erroring out.') logger.exception('Some etcd error occurred') return wrapper diff --git a/ucloud/common/network.py b/ucloud/common/network.py index 6a6c6e2..df7151b 100644 --- a/ucloud/common/network.py +++ b/ucloud/common/network.py @@ -36,6 +36,7 @@ def create_dev(script, _id, dev, ip=None): try: output = sp.check_output(command, stderr=sp.PIPE) except Exception as e: + logger.exception('Creation of interface %s failed.', dev) print(e) return None else: diff --git a/ucloud/common/storage_handlers.py b/ucloud/common/storage_handlers.py index d2bd452..eaad1a5 100644 --- a/ucloud/common/storage_handlers.py +++ b/ucloud/common/storage_handlers.py @@ -51,7 +51,6 @@ class ImageStorageHandler(ABC): output = sp.check_output(command, stderr=sp.PIPE) except Exception as e: if report: - print(e) logger.exception(e) return False return True From eea6c1568e2ad9675032c3950902ead88bbeb5fc Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 22 Dec 2019 13:47:16 +0500 Subject: [PATCH 041/163] colored error output --- scripts/ucloud | 11 +++++++++++ setup.py | 1 + 2 files changed, 12 insertions(+) diff --git a/scripts/ucloud b/scripts/ucloud index f741663..0a6c5ec 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -6,6 +6,8 @@ import importlib import multiprocessing as mp import sys +import colorama + from logging.handlers import SysLogHandler from ucloud.configure.main import configure_parser @@ -24,11 +26,20 @@ class NoTracebackStreamHandler(logging.StreamHandler): def handle(self, record): info, cache = record.exc_info, record.exc_text record.exc_info, record.exc_text = None, None + + if record.levelname == 'WARNING': + color = colorama.Fore.YELLOW + elif record.levelname == 'ERROR': + color = colorama.Fore.LIGHTRED_EX + else: + color = colorama.Fore.RED try: + print(color) super().handle(record) finally: record.exc_info = info record.exc_text = cache + print(colorama.Style.RESET_ALL) if __name__ == '__main__': sys.excepthook = exception_hook diff --git a/setup.py b/setup.py index 5a624db..b4341d3 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ setup(name='ucloud', 'sshtunnel', 'sphinx', 'pynetbox', + 'colorama', 'sphinx-rtd-theme', 'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3', ], From 972bb5a92099ce82c9b777a5040f094b13208241 Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 23 Dec 2019 12:58:04 +0500 Subject: [PATCH 042/163] - Better error reporting. - Flask now uses application's logger instead of its own. - ucloud file scanner refactored. --- scripts/ucloud | 29 ++------ setup.py | 1 + ucloud/api/helper.py | 3 +- ucloud/api/main.py | 32 ++++++++- ucloud/common/etcd_wrapper.py | 2 +- ucloud/common/logging.py | 24 +++++++ ucloud/common/storage_handlers.py | 30 ++++++--- ucloud/filescanner/main.py | 107 ++++++++++-------------------- ucloud/host/main.py | 5 +- ucloud/host/virtualmachine.py | 25 +------ ucloud/imagescanner/main.py | 27 ++++---- ucloud/metadata/__init__.py | 3 + ucloud/metadata/main.py | 15 +++++ ucloud/settings/__init__.py | 8 +-- 14 files changed, 157 insertions(+), 154 deletions(-) create mode 100644 ucloud/common/logging.py diff --git a/scripts/ucloud b/scripts/ucloud index 0a6c5ec..5e8bce6 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -6,44 +6,23 @@ import importlib import multiprocessing as mp import sys -import colorama - from logging.handlers import SysLogHandler from ucloud.configure.main import configure_parser +from ucloud.common.logging import NoTracebackStreamHandler def exception_hook(exc_type, exc_value, exc_traceback): + logger = logging.getLogger(__name__) logger.error( 'Uncaught exception', exc_info=(exc_type, exc_value, exc_traceback) ) - # print('Error: ', end='') - # print(exc_type, exc_value, exc_traceback) -class NoTracebackStreamHandler(logging.StreamHandler): - def handle(self, record): - info, cache = record.exc_info, record.exc_text - record.exc_info, record.exc_text = None, None - - if record.levelname == 'WARNING': - color = colorama.Fore.YELLOW - elif record.levelname == 'ERROR': - color = colorama.Fore.LIGHTRED_EX - else: - color = colorama.Fore.RED - try: - print(color) - super().handle(record) - finally: - record.exc_info = info - record.exc_text = cache - print(colorama.Style.RESET_ALL) +sys.excepthook = exception_hook if __name__ == '__main__': - sys.excepthook = exception_hook - arg_parser = argparse.ArgumentParser() subparsers = arg_parser.add_subparsers(dest="command") @@ -68,7 +47,7 @@ if __name__ == '__main__': arg_parser.print_help() else: # Setting up root logger - logger = logging.getLogger('ucloud') + logger = logging.getLogger(__name__) syslog_handler = SysLogHandler(address='/dev/log') syslog_handler.setLevel(logging.DEBUG) diff --git a/setup.py b/setup.py index b4341d3..2c1c2cb 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ setup(name='ucloud', 'colorama', 'sphinx-rtd-theme', 'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3', + 'werkzeug' ], scripts=['scripts/ucloud'], data_files=[(os.path.expanduser('~/ucloud/'), ['conf/ucloud.conf'])], diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index 9cda36e..e275e46 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -10,7 +10,8 @@ from pyotp import TOTP from ucloud.shared import shared from ucloud.settings import settings -logger = logging.getLogger("ucloud.api.helper") +logger = logging.getLogger(__name__) + def check_otp(name, realm, token): try: diff --git a/ucloud/api/main.py b/ucloud/api/main.py index 05972ff..92c73f5 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -1,11 +1,14 @@ import json import pynetbox +import logging +import urllib3 from uuid import uuid4 from os.path import join as join_path from flask import Flask, request from flask_restful import Resource, Api +from werkzeug.exceptions import HTTPException from ucloud.common import counters from ucloud.common.vm import VMStatus @@ -15,10 +18,33 @@ from ucloud.shared import shared from . import schemas from .helper import generate_mac, mac2ipv6 -from . import logger + + +def get_parent(obj, attr): + parent = getattr(obj, attr) + child = parent + while parent is not None: + child = parent + parent = getattr(parent, attr) + return child + + +logger = logging.getLogger(__name__) app = Flask(__name__) api = Api(app) +app.logger.handlers.clear() + + +@app.errorhandler(Exception) +def handle_exception(e): + app.logger.error(e) + # pass through HTTP errors + if isinstance(e, HTTPException): + return e + + # now you're handling non-HTTP exceptions only + return {'message': 'Server Error'}, 500 class CreateVM(Resource): @@ -438,8 +464,8 @@ class CreateNetwork(Resource): "is_pool": True, } ) - except Exception: - logger.exception("Exception occur while contacting netbox") + except Exception as err: + app.logger.error(err) return {"message": "Error occured while creating network."} else: network_entry["ipv6"] = prefix["prefix"] diff --git a/ucloud/common/etcd_wrapper.py b/ucloud/common/etcd_wrapper.py index 91149b8..eecf4c7 100644 --- a/ucloud/common/etcd_wrapper.py +++ b/ucloud/common/etcd_wrapper.py @@ -27,7 +27,7 @@ def readable_errors(func): @wraps(func) def wrapper(*args, **kwargs): try: - func(*args, **kwargs) + return func(*args, **kwargs) except etcd3.exceptions.ConnectionFailedError as err: raise etcd3.exceptions.ConnectionFailedError('etcd connection failed') from err except etcd3.exceptions.ConnectionTimeoutError as err: diff --git a/ucloud/common/logging.py b/ucloud/common/logging.py new file mode 100644 index 0000000..945f473 --- /dev/null +++ b/ucloud/common/logging.py @@ -0,0 +1,24 @@ +import logging +import colorama + + +class NoTracebackStreamHandler(logging.StreamHandler): + def handle(self, record): + info, cache = record.exc_info, record.exc_text + record.exc_info, record.exc_text = None, None + + if record.levelname == 'WARNING': + color = colorama.Fore.YELLOW + elif record.levelname in ['ERROR', 'EXCEPTION']: + color = colorama.Fore.LIGHTRED_EX + elif record.levelname == 'INFO': + color = colorama.Fore.LIGHTBLUE_EX + else: + color = colorama.Fore.WHITE + try: + print(color, end='', flush=True) + super().handle(record) + finally: + record.exc_info = info + record.exc_text = cache + print(colorama.Style.RESET_ALL, end='', flush=True) diff --git a/ucloud/common/storage_handlers.py b/ucloud/common/storage_handlers.py index eaad1a5..4b7928e 100644 --- a/ucloud/common/storage_handlers.py +++ b/ucloud/common/storage_handlers.py @@ -11,6 +11,8 @@ from ucloud.settings import settings as config class ImageStorageHandler(ABC): + handler_name = 'base' + def __init__(self, image_base, vm_base): self.image_base = image_base self.vm_base = vm_base @@ -45,13 +47,17 @@ class ImageStorageHandler(ABC): def delete_vm_image(self, path): raise NotImplementedError() - def execute_command(self, command, report=True): + def execute_command(self, command, report=True, error_origin=None): + if not error_origin: + error_origin = self.handler_name + command = list(map(str, command)) try: - output = sp.check_output(command, stderr=sp.PIPE) - except Exception as e: + sp.check_output(command, stderr=sp.PIPE) + except sp.CalledProcessError as e: + _stderr = e.stderr.decode('utf-8').strip() if report: - logger.exception(e) + logger.exception('%s:- %s', error_origin, _stderr) return False return True @@ -66,6 +72,8 @@ class ImageStorageHandler(ABC): class FileSystemBasedImageStorageHandler(ImageStorageHandler): + handler_name = 'Filesystem' + def import_image(self, src, dest, protect=False): dest = join_path(self.image_base, dest) try: @@ -118,17 +126,23 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler): class CEPHBasedImageStorageHandler(ImageStorageHandler): + handler_name = 'Ceph' + def import_image(self, src, dest, protect=False): dest = join_path(self.image_base, dest) - command = ["rbd", "import", src, dest] + import_command = ["rbd", "import", src, dest] + commands = [import_command] if protect: snap_create_command = ["rbd", "snap", "create", "{}@protected".format(dest)] snap_protect_command = ["rbd", "snap", "protect", "{}@protected".format(dest)] + commands.append(snap_create_command) + commands.append(snap_protect_command) - return self.execute_command(command) and self.execute_command(snap_create_command) and\ - self.execute_command(snap_protect_command) + result = True + for command in commands: + result = result and self.execute_command(command) - return self.execute_command(command) + return result def make_vm_image(self, src, dest): src = join_path(self.image_base, src) diff --git a/ucloud/filescanner/main.py b/ucloud/filescanner/main.py index ff38748..778e942 100755 --- a/ucloud/filescanner/main.py +++ b/ucloud/filescanner/main.py @@ -3,7 +3,6 @@ import os import pathlib import subprocess as sp import time -import sys from uuid import uuid4 @@ -11,30 +10,6 @@ from . import logger from ucloud.settings import settings from ucloud.shared import shared -def getxattr(file, attr): - """Get specified user extended attribute (arg:attr) of a file (arg:file)""" - try: - attr = "user." + attr - value = sp.check_output(['getfattr', file, - '--name', attr, - '--only-values', - '--absolute-names'], stderr=sp.DEVNULL) - value = value.decode("utf-8") - except sp.CalledProcessError as e: - value = None - - return value - - -def setxattr(file, attr, value): - """Set specified user extended attribute (arg:attr) equal to (arg:value) - of a file (arg:file)""" - - attr = "user." + attr - sp.check_output(['setfattr', file, - '--name', attr, - '--value', str(value)]) - def sha512sum(file: str): """Use sha512sum utility to compute sha512 sum of arg:file @@ -60,12 +35,33 @@ def sha512sum(file: str): return None -try: - sp.check_output(['which', 'getfattr']) - sp.check_output(['which', 'setfattr']) -except Exception as e: - logger.error("You don't seems to have both getfattr and setfattr") - sys.exit(1) +def track_file(file, base_dir): + file_id = uuid4() + + # Get Username + owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] + + # Get Creation Date of File + # Here, we are assuming that ctime is creation time + # which is mostly not true. + creation_date = time.ctime(os.stat(file).st_ctime) + + file_path = pathlib.Path(file).parts[-1] + + # Create Entry + entry_key = os.path.join(settings['etcd']['file_prefix'], str(file_id)) + entry_value = { + "filename": file_path, + "owner": owner, + "sha512sum": sha512sum(file), + "creation_date": creation_date, + "size": os.path.getsize(file) + } + + logger.info("Tracking %s", file) + + shared.etcd_client.put(entry_key, entry_value, value_in_json=True) + os.setxattr(file, 'user.utracked', b'True') def main(): @@ -75,48 +71,15 @@ def main(): files = glob.glob("{}/**".format(base_dir), recursive=True) # Retain only Files - files = list(filter(os.path.isfile, files)) + files = [file for file in files if os.path.isfile(file)] - untracked_files = list( - filter(lambda f: not bool(getxattr(f, "utracked")), files) - ) - - tracked_files = list( - filter(lambda f: f not in untracked_files, files) - ) - for file in untracked_files: - file_id = uuid4() - - # Get Username - owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] - - # Get Creation Date of File - # Here, we are assuming that ctime is creation time - # which is mostly not true. - creation_date = time.ctime(os.stat(file).st_ctime) - - # Get File Size - size = os.path.getsize(file) - - # Compute sha512 sum - sha_sum = sha512sum(file) - - file_path = pathlib.Path(file).parts[-1] - - # Create Entry - entry_key = os.path.join(settings['etcd']['file_prefix'], str(file_id)) - entry_value = { - "filename": file_path, - "owner": owner, - "sha512sum": sha_sum, - "creation_date": creation_date, - "size": size - } - - logger.info("Tracking %s", file) - - shared.etcd_client.put(entry_key, entry_value, value_in_json=True) - setxattr(file, "utracked", True) + untracked_files = [] + for file in files: + try: + os.getxattr(file, 'user.utracked') + except OSError: + track_file(file, base_dir) + untracked_files.append(file) if __name__ == "__main__": diff --git a/ucloud/host/main.py b/ucloud/host/main.py index f78f629..be4f501 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -16,8 +16,7 @@ vmm = virtualmachine.VMM() def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" - client = shared.etcd_client - host_pool = HostPool(client) + host_pool = shared.host_pool this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) while True: @@ -27,7 +26,7 @@ def update_heartbeat(hostname): def main(hostname): - host_pool = HostPool(shared.etcd_client) + host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) assert host is not None, "No such host with name = {}".format(hostname) diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index cc06ce3..b3a1d2a 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -44,29 +44,7 @@ def capture_all_exception(func): try: func(*args, **kwargs) except Exception: - logger.info("Exception absorbed by captual_all_exception()") - logger.exception(func.__name__) - - return wrapper - - -def need_running_vm(func): - @wraps(func) - def wrapper(self, e): - vm = self.get_vm(self.running_vms, e.key) - if vm: - try: - status = vm.handle.command("query-status") - logger.debug("VM Status Check - %s", status) - except Exception as exception: - logger.info("%s failed - VM %s %s", func.__name__, e, exception) - else: - return func(e) - - return None - else: - logger.info("%s failed because VM %s is not running", func.__name__, e.key) - return None + logger.exception('Unhandled exception occur in %s. For more details see Syslog.', __name__) return wrapper @@ -168,7 +146,6 @@ class VMM: self.create(vm_entry) self.launch_vm(vm_entry) - @need_running_vm @capture_all_exception def stop(self, vm_entry): vm = self.get_vm(self.running_vms, vm_entry.key) diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index d164ea3..0d2fbf2 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -1,6 +1,6 @@ import json import os -import subprocess +import subprocess as sp import sys from os.path import isdir @@ -13,7 +13,7 @@ from ucloud.imagescanner import logger def qemu_img_type(path): qemu_img_info_command = ["qemu-img", "info", "--output", "json", path] try: - qemu_img_info = subprocess.check_output(qemu_img_info_command) + qemu_img_info = sp.check_output(qemu_img_info_command) except Exception as e: logger.exception(e) return None @@ -29,7 +29,7 @@ def check(): ) try: - subprocess.check_output(['which', 'qemu-img']) + sp.check_output(['which', 'qemu-img']) except Exception: print("qemu-img missing") sys.exit(1) @@ -67,29 +67,30 @@ def main(): if qemu_img_type(image_full_path) == "qcow2": try: # Convert .qcow2 to .raw - subprocess.check_output(qemu_img_convert_command) - except Exception as e: - logger.exception(e) + sp.check_output(qemu_img_convert_command,) + + except sp.CalledProcessError: + logger.exception('Image convertion from .qcow2 to .raw failed.') else: # Import and Protect r_status = shared.storage_handler.import_image(src="image.raw", - dest=image_uuid, - protect=True) + dest=image_uuid, + protect=True) if r_status: # Everything is successfully done image.value["status"] = "CREATED" shared.etcd_client.put(image.key, json.dumps(image.value)) + finally: + try: + os.remove("image.raw") + except Exception: + pass else: # The user provided image is either not found or of invalid format image.value["status"] = "INVALID_IMAGE" shared.etcd_client.put(image.key, json.dumps(image.value)) - try: - os.remove("image.raw") - except Exception: - pass - if __name__ == "__main__": main() diff --git a/ucloud/metadata/__init__.py b/ucloud/metadata/__init__.py index e69de29..eea436a 100644 --- a/ucloud/metadata/__init__.py +++ b/ucloud/metadata/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/ucloud/metadata/main.py b/ucloud/metadata/main.py index 5526084..adec9e7 100644 --- a/ucloud/metadata/main.py +++ b/ucloud/metadata/main.py @@ -2,12 +2,27 @@ import os from flask import Flask, request from flask_restful import Resource, Api +from werkzeug.exceptions import HTTPException + from ucloud.settings import settings from ucloud.shared import shared app = Flask(__name__) api = Api(app) +app.logger.handlers.clear() + + +@app.errorhandler(Exception) +def handle_exception(e): + app.logger.error(e) + # pass through HTTP errors + if isinstance(e, HTTPException): + return e + + # now you're handling non-HTTP exceptions only + return {'message': 'Server Error'}, 500 + def get_vm_entry(mac_addr): return next(filter(lambda vm: mac_addr in list(zip(*vm.network))[1], shared.vm_pool.vms), None) diff --git a/ucloud/settings/__init__.py b/ucloud/settings/__init__.py index 2c77300..b651aa2 100644 --- a/ucloud/settings/__init__.py +++ b/ucloud/settings/__init__.py @@ -5,7 +5,6 @@ import os from ucloud.common.etcd_wrapper import Etcd3Wrapper - logger = logging.getLogger(__name__) @@ -14,8 +13,9 @@ class CustomConfigParser(configparser.RawConfigParser): try: result = super().__getitem__(key) except KeyError as err: - raise KeyError("Key '{}' not found in config file"\ - .format(key)) from err + raise KeyError( + 'Key \'{}\' not found in configuration. Make sure you configure ucloud.'.format(key) + ) from err else: return result @@ -78,7 +78,7 @@ class Settings(object): if config_from_etcd: self.config_parser.read_dict(config_from_etcd.value) else: - raise KeyError("Key '{}' not found in etcd".format(self.config_key)) + raise KeyError("Key '{}' not found in etcd. Please configure ucloud.".format(self.config_key)) def __getitem__(self, key): self.read_values_from_etcd() From f79097cae9fbe7894c31a6c0237619c51728725a Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 24 Dec 2019 15:27:21 +0500 Subject: [PATCH 043/163] Fix logging --- scripts/ucloud | 2 +- ucloud/imagescanner/main.py | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/scripts/ucloud b/scripts/ucloud index 5e8bce6..3ddbb5a 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -47,7 +47,7 @@ if __name__ == '__main__': arg_parser.print_help() else: # Setting up root logger - logger = logging.getLogger(__name__) + logger = logging.getLogger('ucloud') syslog_handler = SysLogHandler(address='/dev/log') syslog_handler.setLevel(logging.DEBUG) diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index 0d2fbf2..e215c88 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -1,9 +1,7 @@ import json import os import subprocess as sp -import sys -from os.path import isdir from os.path import join as join_path from ucloud.settings import settings from ucloud.shared import shared @@ -21,19 +19,6 @@ def qemu_img_type(path): qemu_img_info = json.loads(qemu_img_info.decode("utf-8")) return qemu_img_info["format"] -def check(): - """ check whether settings are sane, refuse to start if they aren't """ - if settings['storage']['storage_backend'] == 'filesystem' and not isdir(settings['storage']['image_dir']): - sys.exit("You have set STORAGE_BACKEND to filesystem, but " - "{} does not exist. Refusing to start".format(settings['storage']['image_dir']) - ) - - try: - sp.check_output(['which', 'qemu-img']) - except Exception: - print("qemu-img missing") - sys.exit(1) - def main(): # We want to get images entries that requests images to be created From ec3cf49799e9cf17dcd23fa3d09209597d1d85ea Mon Sep 17 00:00:00 2001 From: meow Date: Thu, 26 Dec 2019 12:24:19 +0500 Subject: [PATCH 044/163] Create radvd configuration and start it <--> VM's which is being started has IPv6 network which is global --- ucloud/host/virtualmachine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index b3a1d2a..dbd13de 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -8,6 +8,7 @@ import os import subprocess as sp import tempfile import time +import ipaddress from functools import wraps from string import Template @@ -90,7 +91,9 @@ class VMM: tap_id=tap, ip=network_ipv6) all_networks = self.etcd_client.get_prefix('/v1/network/', value_in_json=True) - update_radvd_conf(all_networks) + + if ipaddress.ip_network(network_ipv6).is_global: + update_radvd_conf(all_networks) command += " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" \ " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" \ @@ -361,7 +364,7 @@ def update_radvd_conf(all_networks): networks = { net.value['ipv6']: net.value['id'] for net in all_networks - if net.value.get('ipv6') + if net.value.get('ipv6') and ipaddress.ip_network(net.value.get('ipv6')).is_global } radvd_template = open(os.path.join(network_script_base, 'radvd-template.conf'), 'r').read() From cd9d4cb78c9069e166bb618c78cd302514dd9c70 Mon Sep 17 00:00:00 2001 From: meow Date: Thu, 26 Dec 2019 14:30:15 +0500 Subject: [PATCH 045/163] Fix bug that cause failure of image resizing when creating vm --- ucloud/common/storage_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ucloud/common/storage_handlers.py b/ucloud/common/storage_handlers.py index 4b7928e..4a17ec7 100644 --- a/ucloud/common/storage_handlers.py +++ b/ucloud/common/storage_handlers.py @@ -89,7 +89,7 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler): src = join_path(self.image_base, src) dest = join_path(self.vm_base, dest) try: - shutil.copy(src, dest) + shutil.copyfile(src, dest) except Exception as e: logger.exception(e) return False From ba515f0b48a079b9b6bd7e74cbdd842c5696ca65 Mon Sep 17 00:00:00 2001 From: meow Date: Sat, 28 Dec 2019 15:39:11 +0500 Subject: [PATCH 046/163] Refactoring, VMM added, uncloud-host mostly new, migration is better now --- setup.py | 2 +- ucloud/api/main.py | 17 +- ucloud/common/network.py | 2 +- ucloud/common/request.py | 1 + ucloud/common/schemas.py | 39 +++ ucloud/common/storage_handlers.py | 8 +- ucloud/common/vm.py | 6 + ucloud/host/main.py | 45 +-- ucloud/host/virtualmachine.py | 449 ++++++++++-------------------- ucloud/scheduler/helper.py | 2 +- ucloud/scheduler/main.py | 35 +-- ucloud/vmm/__init__.py | 181 ++++++++++++ 12 files changed, 423 insertions(+), 364 deletions(-) create mode 100644 ucloud/common/schemas.py create mode 100644 ucloud/vmm/__init__.py diff --git a/setup.py b/setup.py index 2c1c2cb..51d21b8 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup(name='ucloud', 'colorama', 'sphinx-rtd-theme', 'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3', - 'werkzeug' + 'werkzeug', 'marshmallow' ], scripts=['scripts/ucloud'], data_files=[(os.path.expanduser('~/ucloud/'), ['conf/ucloud.conf'])], diff --git a/ucloud/api/main.py b/ucloud/api/main.py index 92c73f5..91cbead 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -1,7 +1,6 @@ import json import pynetbox import logging -import urllib3 from uuid import uuid4 from os.path import join as join_path @@ -78,6 +77,7 @@ class CreateVM(Resource): "vnc_socket": "", "network": list(zip(data["network"], macs, tap_ids)), "metadata": {"ssh-keys": []}, + "in_migration": False } shared.etcd_client.put(vm_key, vm_entry, value_in_json=True) @@ -216,16 +216,13 @@ class VMMigration(Resource): if validator.is_valid(): vm = shared.vm_pool.get(data["uuid"]) + r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, + uuid=vm.uuid, + hostname=join_path( + settings['etcd']['host_prefix'], validator.destination.value + ), + request_prefix=settings['etcd']['request_prefix']) - r = RequestEntry.from_scratch( - type=RequestType.ScheduleVM, - uuid=vm.uuid, - destination=join_path( - settings['etcd']['host_prefix'], validator.destination.value - ), - migration=True, - request_prefix=settings['etcd']['request_prefix'] - ) shared.request_pool.put(r) return {"message": "VM Migration Initialization Queued"}, 200 else: diff --git a/ucloud/common/network.py b/ucloud/common/network.py index df7151b..629e92a 100644 --- a/ucloud/common/network.py +++ b/ucloud/common/network.py @@ -30,7 +30,7 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt=' def create_dev(script, _id, dev, ip=None): - command = [script, _id, dev] + command = [script, str(_id), dev] if ip: command.append(ip) try: diff --git a/ucloud/common/request.py b/ucloud/common/request.py index 1e4594d..2d9be44 100644 --- a/ucloud/common/request.py +++ b/ucloud/common/request.py @@ -19,6 +19,7 @@ class RequestType: class RequestEntry(SpecificEtcdEntryBase): def __init__(self, e): + self.destination_host_key = None self.type = None # type: str self.migration = None # type: bool self.destination = None # type: str diff --git a/ucloud/common/schemas.py b/ucloud/common/schemas.py new file mode 100644 index 0000000..a592ec2 --- /dev/null +++ b/ucloud/common/schemas.py @@ -0,0 +1,39 @@ +import bitmath + +from marshmallow import fields, Schema + + +class StorageUnit(fields.Field): + def _serialize(self, value, attr, obj, **kwargs): + return str(value) + + def _deserialize(self, value, attr, data, **kwargs): + return bitmath.parse_string_unsafe(value) + + +class SpecsSchema(Schema): + cpu = fields.Int() + ram = StorageUnit() + os_ssd = StorageUnit(data_key='os-ssd', attribute='os-ssd') + hdd = fields.List(StorageUnit()) + + +class VMSchema(Schema): + name = fields.Str() + owner = fields.Str() + owner_realm = fields.Str() + specs = fields.Nested(SpecsSchema) + status = fields.Str() + log = fields.List(fields.Str()) + vnc_socket = fields.Str() + image_uuid = fields.Str() + hostname = fields.Str() + metadata = fields.Dict() + network = fields.List(fields.Tuple((fields.Str(), fields.Str(), fields.Int()))) + in_migration = fields.Bool() + + +class NetworkSchema(Schema): + _id = fields.Int(data_key='id', attribute='id') + _type = fields.Str(data_key='type', attribute='type') + ipv6 = fields.Str() diff --git a/ucloud/common/storage_handlers.py b/ucloud/common/storage_handlers.py index 4a17ec7..d2190ba 100644 --- a/ucloud/common/storage_handlers.py +++ b/ucloud/common/storage_handlers.py @@ -19,8 +19,8 @@ class ImageStorageHandler(ABC): def import_image(self, image_src, image_dest, protect=False): """Put an image at the destination - :param src: An Image file - :param dest: A path where :param src: is to be put. + :param image_src: An Image file + :param image_dest: A path where :param src: is to be put. :param protect: If protect is true then the dest is protect (readonly etc) The obj must exist on filesystem. """ @@ -30,8 +30,8 @@ class ImageStorageHandler(ABC): def make_vm_image(self, image_path, path): """Copy image from src to dest - :param src: A path - :param dest: A path + :param image_path: A path + :param path: A path src and destination must be on same storage system i.e both on file system or both on CEPH etc. """ diff --git a/ucloud/common/vm.py b/ucloud/common/vm.py index 0fb5cea..238f19d 100644 --- a/ucloud/common/vm.py +++ b/ucloud/common/vm.py @@ -12,6 +12,12 @@ class VMStatus: error = "ERROR" # An error occurred that cannot be resolved automatically +def declare_stopped(vm): + vm['hostname'] = '' + vm['in_migration'] = False + vm['status'] = VMStatus.stopped + + class VMEntry(SpecificEtcdEntryBase): def __init__(self, e): diff --git a/ucloud/host/main.py b/ucloud/host/main.py index be4f501..8a7dbe7 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -1,17 +1,16 @@ import argparse import multiprocessing as mp import time -import sys from ucloud.common.request import RequestEntry, RequestType -from ucloud.common.host import HostPool from ucloud.shared import shared from ucloud.settings import settings +from ucloud.common.vm import VMStatus +from ucloud.vmm import VMM +from os.path import join as join_path from . import virtualmachine, logger -vmm = virtualmachine.VMM() - def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" @@ -25,6 +24,16 @@ def update_heartbeat(hostname): time.sleep(10) +def maintenance(): + vmm = VMM() + running_vms = vmm.discover() + for vm_uuid in running_vms: + if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': + vm = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], vm_uuid)) + vm.status = VMStatus.running + shared.vm_pool.put(vm) + + def main(hostname): host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) @@ -34,8 +43,7 @@ def main(hostname): heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() except Exception as e: - logger.exception(e) - sys.exit("No Need To Go Further. ucloud-host heartbeat updating mechanism is not working") + raise e.__class__('ucloud-host heartbeat updating mechanism is not working') from e for events_iterator in [ shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True), @@ -45,36 +53,37 @@ def main(hostname): request_event = RequestEntry(request_event) if request_event.type == "TIMEOUT": - vmm.maintenance(host) - continue + maintenance() - # If the event is directed toward me OR I am destination of a InitVMMigration - if request_event.hostname == host.key or request_event.destination == host.key: + if request_event.hostname == host.key: logger.debug("VM Request: %s", request_event) shared.request_pool.client.client.delete(request_event.key) - vm_entry = shared.vm_pool.get(request_event.uuid) + vm_entry = shared.etcd_client.get(join_path(settings['etcd']['vm_prefix'], request_event.uuid)) if vm_entry: + vm = virtualmachine.VM(vm_entry) if request_event.type == RequestType.StartVM: - vmm.start(vm_entry) + vm.start() elif request_event.type == RequestType.StopVM: - vmm.stop(vm_entry) + vm.stop() elif request_event.type == RequestType.DeleteVM: - vmm.delete(vm_entry) + vm.delete() elif request_event.type == RequestType.InitVMMigration: - vmm.start(vm_entry, host.key) + vm.start(destination_host_key=host.key) elif request_event.type == RequestType.TransferVM: - vmm.transfer(request_event) + host = host_pool.get(request_event.destination_host_key) + if host: + vm.migrate(destination=host.hostname) + else: + logger.error('Host %s not found!', request_event.destination_host_key) else: logger.info("VM Entry missing") - logger.info("Running VMs %s", vmm.running_vms) - if __name__ == "__main__": argparser = argparse.ArgumentParser() diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index dbd13de..6d25205 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -6,344 +6,189 @@ import os import subprocess as sp -import tempfile -import time import ipaddress -from functools import wraps from string import Template -from typing import Union from os.path import join as join_path -import bitmath -import sshtunnel - -from ucloud.common.helpers import get_ipv6_address from ucloud.common.request import RequestEntry, RequestType -from ucloud.common.vm import VMEntry, VMStatus -from ucloud.common.network import create_dev, delete_network_interface, find_free_port +from ucloud.common.vm import VMStatus, declare_stopped +from ucloud.common.network import create_dev, delete_network_interface +from ucloud.common.schemas import VMSchema, NetworkSchema from ucloud.host import logger from ucloud.shared import shared from ucloud.settings import settings +from ucloud.vmm import VMM -from . import qmp +from marshmallow import ValidationError + + +def maintenance(): + pass class VM: - def __init__(self, key, handle, vnc_socket_file): - self.key = key # type: str - self.handle = handle # type: qmp.QEMUMachine - self.vnc_socket_file = vnc_socket_file # type: tempfile.NamedTemporaryFile - - def __repr__(self): - return "VM({})".format(self.key) - - -def capture_all_exception(func): - @wraps(func) - def wrapper(*args, **kwargs): + def __init__(self, vm_entry): + self.schema = VMSchema() + self.vmm = VMM() + self.key = vm_entry.key try: - func(*args, **kwargs) - except Exception: - logger.exception('Unhandled exception occur in %s. For more details see Syslog.', __name__) - - return wrapper - - -class VMM: - def __init__(self): - self.etcd_client = shared.etcd_client - self.storage_handler = shared.storage_handler - self.running_vms = [] - - def get_start_command_args(self, vm_entry, vnc_sock_filename: str, migration=False, migration_port=None): - threads_per_core = 1 - vm_memory = int(bitmath.parse_string_unsafe(vm_entry.specs['ram']).to_MB()) - vm_cpus = int(vm_entry.specs['cpu']) - vm_uuid = vm_entry.uuid - vm_networks = vm_entry.network - - command = '-name {}_{}'.format(vm_entry.owner, vm_entry.name) - - command += ' -drive file={},format=raw,if=virtio,cache=none'.format( - self.storage_handler.qemu_path_string(vm_uuid) - ) - command += ' -device virtio-rng-pci -vnc unix:{}'.format(vnc_sock_filename) - command += ' -m {} -smp cores={},threads={}'.format( - vm_memory, vm_cpus, threads_per_core - ) - - if migration: - command += ' -incoming tcp:[::]:{}'.format(migration_port) - - for network_mac_and_tap in vm_networks: - network_name, mac, tap = network_mac_and_tap - - _key = os.path.join(settings['etcd']['network_prefix'], vm_entry.owner, network_name) - network = self.etcd_client.get(_key, value_in_json=True) - network_type = network.value["type"] - network_id = str(network.value["id"]) - network_ipv6 = network.value["ipv6"] - - if network_type == "vxlan": - tap = create_vxlan_br_tap(_id=network_id, - _dev=settings['network']['vxlan_phy_dev'], - tap_id=tap, - ip=network_ipv6) - all_networks = self.etcd_client.get_prefix('/v1/network/', value_in_json=True) - - if ipaddress.ip_network(network_ipv6).is_global: - update_radvd_conf(all_networks) - - command += " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" \ - " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" \ - .format(tap=tap, net_id=network_id, mac=mac) - - return command.split(" ") - - def create_vm_object(self, vm_entry, migration=False, migration_port=None): - vnc_sock_file = tempfile.NamedTemporaryFile() - - qemu_args = self.get_start_command_args( - vm_entry=vm_entry, - vnc_sock_filename=vnc_sock_file.name, - migration=migration, - migration_port=migration_port, - ) - qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64", args=qemu_args) - return VM(vm_entry.key, qemu_machine, vnc_sock_file) - - @staticmethod - def get_vm(vm_list: list, vm_key) -> Union[VM, None]: - return next((vm for vm in vm_list if vm.key == vm_key), None) - - @capture_all_exception - def create(self, vm_entry: VMEntry): - if self.storage_handler.is_vm_image_exists(vm_entry.uuid): - # File Already exists. No Problem Continue - logger.debug("Image for vm %s exists", vm_entry.uuid) - return None + self.vm = self.schema.loads(vm_entry.value) + except ValidationError: + logger.exception('Couldn\'t validate VM Entry', vm_entry.value) + self.vm = None else: - vm_hdd = int(bitmath.parse_string_unsafe(vm_entry.specs["os-ssd"]).to_MB()) - if self.storage_handler.make_vm_image(src=vm_entry.image_uuid, dest=vm_entry.uuid): - if not self.storage_handler.resize_vm_image(path=vm_entry.uuid, size=vm_hdd): - vm_entry.status = VMStatus.error - else: - logger.info("New VM Created") + self.uuid = vm_entry.key.split('/')[-1] + self.host_key = self.vm['hostname'] - @capture_all_exception - def start(self, vm_entry: VMEntry, destination_host_key=None): - _vm = self.get_vm(self.running_vms, vm_entry.key) + def get_qemu_args(self): + command = ( + '-name {owner}_{name}' + ' -drive file={file},format=raw,if=virtio,cache=none' + ' -device virtio-rng-pci' + ' -m {memory} -smp cores={cores},threads={threads}' + ).format(owner=self.vm['owner'], name=self.vm['name'], + memory=int(self.vm['specs']['ram'].to_MB()), cores=self.vm['specs']['cpu'], + threads=1, file=shared.storage_handler.qemu_path_string(self.uuid)) - # VM already running. No need to proceed further. - if _vm: - logger.info("VM %s already running" % vm_entry.uuid) - return - else: - logger.info("Trying to start %s" % vm_entry.uuid) - if destination_host_key: - migration_port = find_free_port() - self.launch_vm(vm_entry, migration=True, migration_port=migration_port, - destination_host_key=destination_host_key) - else: - self.create(vm_entry) - self.launch_vm(vm_entry) + return command.split(' ') - @capture_all_exception - def stop(self, vm_entry): - vm = self.get_vm(self.running_vms, vm_entry.key) - vm.handle.shutdown() - if not vm.handle.is_running(): - vm_entry.add_log("Shutdown successfully") - vm_entry.declare_stopped() - shared.vm_pool.put(vm_entry) - self.running_vms.remove(vm) - delete_vm_network(vm_entry) + def start(self, destination_host_key=None): + migration = False + if destination_host_key: + migration = True - @capture_all_exception - def delete(self, vm_entry): - logger.info("Deleting VM | %s", vm_entry) - self.stop(vm_entry) - - if self.storage_handler.is_vm_image_exists(vm_entry.uuid): - r_status = self.storage_handler.delete_vm_image(vm_entry.uuid) - if r_status: - shared.etcd_client.client.delete(vm_entry.key) - else: - shared.etcd_client.client.delete(vm_entry.key) - - @capture_all_exception - def transfer(self, request_event): - # This function would run on source host i.e host on which the vm - # is running initially. This host would be responsible for transferring - # vm state to destination host. - - _host, _port = request_event.parameters["host"], request_event.parameters["port"] - _uuid = request_event.uuid - _destination = request_event.destination_host_key - vm = self.get_vm(self.running_vms, join_path(settings['etcd']['vm_prefix'], _uuid)) - - if vm: - tunnel = sshtunnel.SSHTunnelForwarder( - _host, - ssh_username=settings['ssh']['username'], - ssh_pkey=settings['ssh']['private_key_path'], - remote_bind_address=("127.0.0.1", _port), - ssh_proxy_enabled=True, - ssh_proxy=(_host, 22) - ) - try: - tunnel.start() - except sshtunnel.BaseSSHTunnelForwarderError: - logger.exception("Couldn't establish connection to (%s, 22)", _host) - else: - vm.handle.command( - "migrate", uri="tcp:0.0.0.0:{}".format(tunnel.local_bind_port) - ) - - status = vm.handle.command("query-migrate")["status"] - while status not in ["failed", "completed"]: - time.sleep(2) - status = vm.handle.command("query-migrate")["status"] - - with shared.vm_pool.get_put(request_event.uuid) as source_vm: - if status == "failed": - source_vm.add_log("Migration Failed") - elif status == "completed": - # If VM is successfully migrated then shutdown the VM - # on this host and update hostname to destination host key - source_vm.add_log("Successfully migrated") - source_vm.hostname = _destination - self.running_vms.remove(vm) - vm.handle.shutdown() - source_vm.in_migration = False # VM transfer finished - finally: - tunnel.close() - - @capture_all_exception - def launch_vm(self, vm_entry, migration=False, migration_port=None, destination_host_key=None): - logger.info("Starting %s" % vm_entry.key) - - vm = self.create_vm_object(vm_entry, migration=migration, migration_port=migration_port) + self.create() try: - vm.handle.launch() - except Exception: - logger.exception("Error Occured while starting VM") - vm.handle.shutdown() - - if migration: - # We don't care whether MachineError or any other error occurred - pass - else: - # Error during typical launch of a vm - vm.handle.shutdown() - vm_entry.declare_killed() - shared.vm_pool.put(vm_entry) + network_args = self.create_network_dev() + except Exception as err: + declare_stopped(self.vm) + self.vm['log'].append('Cannot Setup Network Properly') + logger.error('Cannot Setup Network Properly for vm %s', self.uuid, exc_info=err) else: - vm_entry.vnc_socket = vm.vnc_socket_file.name - self.running_vms.append(vm) + self.vmm.start(uuid=self.uuid, migration=migration, + *self.get_qemu_args(), *network_args) - if migration: - vm_entry.in_migration = True + status = self.vmm.get_status(self.uuid) + if status == 'running': + self.vm['status'] = VMStatus.running + self.vm['vnc_socket'] = self.vmm.get_vnc(self.uuid) + elif status == 'inmigrate': r = RequestEntry.from_scratch( - type=RequestType.TransferVM, - hostname=vm_entry.hostname, - parameters={"host": get_ipv6_address(), "port": migration_port}, - uuid=vm_entry.uuid, - destination_host_key=destination_host_key, + type=RequestType.TransferVM, # Transfer VM + hostname=self.host_key, # Which VM should get this request. It is source host + uuid=self.uuid, # uuid of VM + destination_host_key=destination_host_key, # Where source host transfer VM request_prefix=settings['etcd']['request_prefix'] ) shared.request_pool.put(r) else: - # Typical launching of a vm - vm_entry.status = VMStatus.running - vm_entry.add_log("Started successfully") + self.stop() + declare_stopped(self.vm) - shared.vm_pool.put(vm_entry) + self.sync() - @capture_all_exception - def maintenance(self, host): - # To capture vm running according to running_vms list + def stop(self): + self.vmm.stop(self.uuid) + self.delete_network_dev() + declare_stopped(self.vm) + self.sync() - # This is to capture successful migration of a VM. - # Suppose, this host is running "vm1" and user initiated - # request to migrate this "vm1" to some other host. On, - # successful migration the destination host would set - # the vm hostname to itself. Thus, we are checking - # whether this host vm is successfully migrated. If yes - # then we shutdown "vm1" on this host. - logger.debug("Starting Maintenance!!") - to_be_removed = [] - for running_vm in self.running_vms: - with shared.vm_pool.get_put(running_vm.key) as vm_entry: - if vm_entry.hostname != host.key and not vm_entry.in_migration: - running_vm.handle.shutdown() - logger.info("VM migration not completed successfully.") - to_be_removed.append(running_vm) + def migrate(self, destination): + self.vmm.transfer(src_uuid=self.uuid, dest_uuid=self.uuid, host=destination) - for r in to_be_removed: - self.running_vms.remove(r) + def create_network_dev(self): + command = '' + for network_mac_and_tap in self.vm['network']: + network_name, mac, tap = network_mac_and_tap - # To check vm running according to etcd entries - alleged_running_vms = shared.vm_pool.by_status("RUNNING", shared.vm_pool.by_host(host.key)) + _key = os.path.join(settings['etcd']['network_prefix'], self.vm['owner'], network_name) + network = shared.etcd_client.get(_key, value_in_json=True) + network_schema = NetworkSchema() + try: + network = network_schema.load(network.value) + except ValidationError: + continue - for vm_entry in alleged_running_vms: - _vm = self.get_vm(self.running_vms, vm_entry.key) - # Whether, the allegedly running vm is in our - # running_vms list or not if it is said to be - # running on this host but it is not then we - # need to shut it down + if network['type'] == "vxlan": + tap = create_vxlan_br_tap(_id=network['id'], + _dev=settings['network']['vxlan_phy_dev'], + tap_id=tap, + ip=network['ipv6']) - # This is to capture poweroff/shutdown of a VM - # initiated by user inside VM. OR crash of VM by some - # user running process - if (_vm and not _vm.handle.is_running()) or not _vm: - logger.debug("_vm = %s, is_running() = %s" % (_vm, _vm.handle.is_running())) - vm_entry.add_log("""{} is not running but is said to be running. - So, shutting it down and declare it killed""".format(vm_entry.key)) - vm_entry.declare_killed() - shared.vm_pool.put(vm_entry) - if _vm: - self.running_vms.remove(_vm) + all_networks = shared.etcd_client.get_prefix(settings['etcd']['network_prefix'], + value_in_json=True) + + if ipaddress.ip_network(network['ipv6']).is_global: + update_radvd_conf(all_networks) + + command += '-netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no' \ + ' -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}' \ + .format(tap=tap, net_id=network['id'], mac=mac) + + return command.split(' ') + + def delete_network_dev(self): + try: + for network in self.vm['network']: + network_name = network[0] + _ = network[1] # tap_mac + tap_id = network[2] + + delete_network_interface('tap{}'.format(tap_id)) + + owners_vms = shared.vm_pool.by_owner(self.vm['owner']) + owners_running_vms = shared.vm_pool.by_status(VMStatus.running, + _vms=owners_vms) + + networks = map( + lambda n: n[0], map(lambda vm: vm.network, owners_running_vms) + ) + networks_in_use_by_user_vms = [vm[0] for vm in networks] + if network_name not in networks_in_use_by_user_vms: + network_entry = resolve_network(network[0], self.vm['owner']) + if network_entry: + network_type = network_entry.value["type"] + network_id = network_entry.value["id"] + if network_type == "vxlan": + delete_network_interface('br{}'.format(network_id)) + delete_network_interface('vxlan{}'.format(network_id)) + except Exception: + logger.exception("Exception in network interface deletion") + + def create(self): + if shared.storage_handler.is_vm_image_exists(self.uuid): + # File Already exists. No Problem Continue + logger.debug("Image for vm %s exists", self.uuid) + else: + if shared.storage_handler.make_vm_image(src=self.vm['image_uuid'], dest=self.uuid): + if not shared.storage_handler.resize_vm_image(path=self.uuid, + size=int(self.vm['specs']['os-ssd'].to_MB())): + self.vm['status'] = VMStatus.error + else: + logger.info("New VM Created") + + def sync(self): + shared.etcd_client.put(self.key, self.schema.dump(self.vm), value_in_json=True) + + def delete(self): + self.stop() + + if shared.storage_handler.is_vm_image_exists(self.uuid): + r_status = shared.storage_handler.delete_vm_image(self.uuid) + if r_status: + shared.etcd_client.client.delete(self.key) + else: + shared.etcd_client.client.delete(self.key) def resolve_network(network_name, network_owner): - network = shared.etcd_client.get(join_path(settings['etcd']['network_prefix'], - network_owner, - network_name), - value_in_json=True) + network = shared.etcd_client.get( + join_path(settings['etcd']['network_prefix'], network_owner, network_name), value_in_json=True + ) return network -def delete_vm_network(vm_entry): - try: - for network in vm_entry.network: - network_name = network[0] - tap_mac = network[1] - tap_id = network[2] - - delete_network_interface('tap{}'.format(tap_id)) - - owners_vms = shared.vm_pool.by_owner(vm_entry.owner) - owners_running_vms = shared.vm_pool.by_status(VMStatus.running, - _vms=owners_vms) - - networks = map( - lambda n: n[0], map(lambda vm: vm.network, owners_running_vms) - ) - networks_in_use_by_user_vms = [vm[0] for vm in networks] - if network_name not in networks_in_use_by_user_vms: - network_entry = resolve_network(network[0], vm_entry.owner) - if network_entry: - network_type = network_entry.value["type"] - network_id = network_entry.value["id"] - if network_type == "vxlan": - delete_network_interface('br{}'.format(network_id)) - delete_network_interface('vxlan{}'.format(network_id)) - except Exception: - logger.exception("Exception in network interface deletion") - - def create_vxlan_br_tap(_id, _dev, tap_id, ip=None): network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') vxlan = create_dev(script=os.path.join(network_script_base, 'create-vxlan.sh'), @@ -377,10 +222,12 @@ def update_radvd_conf(all_networks): ) for net in networks if networks.get(net) ] - with open('/etc/radvd.conf', 'w') as radvd_conf: radvd_conf.writelines(content) try: sp.check_output(['systemctl', 'restart', 'radvd']) - except Exception: - sp.check_output(['service', 'radvd', 'restart']) + except sp.CalledProcessError: + try: + sp.check_output(['service', 'radvd', 'restart']) + except sp.CalledProcessError as err: + raise err.__class__('Cannot start/restart radvd service', err.cmd) from err diff --git a/ucloud/scheduler/helper.py b/ucloud/scheduler/helper.py index 643e8e9..0e9ef73 100755 --- a/ucloud/scheduler/helper.py +++ b/ucloud/scheduler/helper.py @@ -95,7 +95,7 @@ def dead_host_mitigation(dead_hosts_keys): vms_hosted_on_dead_host = shared.vm_pool.by_host(host_key) for vm in vms_hosted_on_dead_host: - vm.declare_killed() + vm.status = 'UNKNOWN' shared.vm_pool.put(vm) shared.host_pool.put(host) diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index 3412545..d91979f 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -56,35 +56,14 @@ def main(): continue shared.etcd_client.client.delete(request_entry.key) # consume Request - # If the Request is about a VM which is labelled as "migration" - # and has a destination - if hasattr(request_entry, "migration") and request_entry.migration \ - and hasattr(request_entry, "destination") and request_entry.destination: - try: - get_suitable_host(vm_specs=vm_entry.specs, - hosts=[shared.host_pool.get(request_entry.destination)]) - except NoSuitableHostFound: - logger.info("Requested destination host doesn't have enough capacity" - "to hold %s" % vm_entry.uuid) - else: - r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, - uuid=request_entry.uuid, - destination=request_entry.destination, - request_prefix=settings['etcd']['request_prefix']) - shared.request_pool.put(r) + try: + assign_host(vm_entry) + except NoSuitableHostFound: + vm_entry.add_log("Can't schedule VM. No Resource Left.") + shared.vm_pool.put(vm_entry) - # If the Request is about a VM that just want to get started/created - else: - # assign_host only returns None when we couldn't be able to assign - # a host to a VM because of resource constraints - try: - assign_host(vm_entry) - except NoSuitableHostFound: - vm_entry.add_log("Can't schedule VM. No Resource Left.") - shared.vm_pool.put(vm_entry) - - pending_vms.append(vm_entry) - logger.info("No Resource Left. Emailing admin....") + pending_vms.append(vm_entry) + logger.info("No Resource Left. Emailing admin....") if __name__ == "__main__": diff --git a/ucloud/vmm/__init__.py b/ucloud/vmm/__init__.py new file mode 100644 index 0000000..1291da4 --- /dev/null +++ b/ucloud/vmm/__init__.py @@ -0,0 +1,181 @@ +import os +import subprocess as sp +import logging +import socket +import json +import tempfile +import time + +from contextlib import suppress +from multiprocessing import Process +from os.path import join as join_path +from os.path import isdir + +logger = logging.getLogger(__name__) + + +class VMQMPHandles: + def __init__(self, path): + self.path = path + self.sock = socket.socket(socket.AF_UNIX) + self.file = self.sock.makefile() + + def __enter__(self): + self.sock.connect(self.path) + + # eat qmp greetings + self.file.readline() + + # init qmp + self.sock.sendall(b'{ "execute": "qmp_capabilities" }') + self.file.readline() + + return self.sock, self.file + + def __exit__(self, exc_type, exc_val, exc_tb): + self.file.close() + self.sock.close() + + if exc_type: + logger.error('Couldn\'t get handle for VM.', exc_type, exc_val, exc_tb) + raise exc_type("Couldn't get handle for VM.") from exc_type + + +class TransferVM(Process): + def __init__(self, src_uuid, dest_uuid, host, socket_dir): + self.src_uuid = src_uuid + self.dest_uuid = dest_uuid + self.host = host + self.src_sock_path = os.path.join(socket_dir, self.src_uuid) + self.dest_sock_path = os.path.join(socket_dir, self.dest_uuid) + + super().__init__() + + def run(self): + with suppress(FileNotFoundError): + os.remove(self.src_sock_path) + + command = ['ssh', '-nNT', '-L', '{}:{}'.format(self.src_sock_path, self.dest_sock_path), + 'root@{}'.format(self.host)] + + try: + p = sp.Popen(command) + except Exception as e: + logger.error('Couldn\' forward unix socks over ssh.', exc_info=e) + else: + time.sleep(2) + vmm = VMM() + logger.debug('Executing: ssh forwarding command: %s', command) + vmm.execute_command(self.src_uuid, command='migrate', + arguments={'uri': 'unix:{}'.format(self.src_sock_path)}) + + while p.poll() is None: + success, output = vmm.execute_command(self.src_uuid, command='query-migrate') + if success: + status = output['return']['status'] + if status != 'active': + print('Migration Status: ', status) + return + else: + print('Migration Status: ', status) + else: + return + time.sleep(0.2) + + +class VMM: + # Virtual Machine Manager + def __init__(self, qemu_path='/usr/bin/qemu-system-x86_64', + vmm_backend=os.path.expanduser('~/ucloud/vmm/')): + self.qemu_path = qemu_path + self.vmm_backend = vmm_backend + self.socket_dir = os.path.join(self.vmm_backend, 'sock') + + def is_running(self, uuid): + sock_path = os.path.join(self.vmm_backend, uuid) + try: + sock = socket.socket(socket.AF_UNIX) + sock.connect(sock_path) + recv = sock.recv(4096) + except Exception as err: + # unix sock doesn't exists or it is closed + logger.info('VM %s sock either don\' exists or it is closed.', uuid, + 'It mean VM is stopped.', exc_info=err) + else: + # if we receive greetings from qmp it mean VM is running + if len(recv) > 0: + return True + + with suppress(FileNotFoundError): + os.remove(sock_path) + + return False + + def start(self, *args, uuid, migration=False): + # start --> sucess? + migration_args = () + if migration: + migration_args = ('-incoming', 'unix:{}'.format(os.path.join(self.socket_dir, uuid))) + + if self.is_running(uuid): + logger.warning('Cannot start VM. It is already running.') + else: + qmp_arg = ('-qmp', 'unix:{}/{},server,nowait'.format(self.vmm_backend, uuid)) + vnc_arg = ('-vnc', 'unix:{}'.format(tempfile.NamedTemporaryFile().name)) + + command = [self.qemu_path, *args, *qmp_arg, *migration_args, *vnc_arg, '-daemonize'] + try: + sp.check_output(command, stderr=sp.PIPE) + except sp.CalledProcessError as err: + logger.exception('Error occurred while starting VM.\nDetail %s', err.stderr.decode('utf-8')) + else: + time.sleep(2) + + def execute_command(self, uuid, command, **kwargs): + # execute_command -> sucess?, output + try: + with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as (sock_handle, file_handle): + command_to_execute = { + 'execute': command, + **kwargs + } + sock_handle.sendall(json.dumps(command_to_execute).encode('utf-8')) + output = file_handle.readline() + except Exception as err: + logger.exception('Error occurred while executing command and getting valid output from qmp') + else: + try: + output = json.loads(output) + except: + logger.exception('QMP Output isn\'t valid JSON. %s', output) + else: + return 'return' in output, output + return False, None + + def stop(self, uuid): + success, output = self.execute_command(command='quit', uuid=uuid) + return success + + def get_status(self, uuid): + success, output = self.execute_command(command='query-status', uuid=uuid) + if success: + return output['return']['status'] + else: + return 'STOPPED' + + def discover(self): + vms = [ + uuid for uuid in os.listdir(self.vmm_backend) + if not isdir(join_path(self.vmm_backend, uuid)) + ] + return vms + + def get_vnc(self, uuid): + success, output = self.execute_command(uuid, command='query-vnc') + if success: + return output['return']['service'] + return None + + def transfer(self, src_uuid, dest_uuid, host): + p = TransferVM(src_uuid, dest_uuid, socket_dir=self.socket_dir, host=host) + p.start() From 808271f3e07938e7886273e470b4a72e504848c5 Mon Sep 17 00:00:00 2001 From: meow Date: Sat, 28 Dec 2019 16:35:55 +0500 Subject: [PATCH 047/163] Return nice message when etcd section is missing --- ucloud/configure/main.py | 2 -- ucloud/settings/__init__.py | 9 ++++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ucloud/configure/main.py b/ucloud/configure/main.py index 71e07a1..e4770d9 100644 --- a/ucloud/configure/main.py +++ b/ucloud/configure/main.py @@ -1,5 +1,3 @@ -import argparse -import sys import os from ucloud.settings import settings diff --git a/ucloud/settings/__init__.py b/ucloud/settings/__init__.py index b651aa2..e589485 100644 --- a/ucloud/settings/__init__.py +++ b/ucloud/settings/__init__.py @@ -30,7 +30,10 @@ class Settings(object): self.config_key = config_key self.read_internal_values() - self.config_parser.read(self.config_file) + try: + self.config_parser.read(self.config_file) + except Exception as err: + logger.error('%s', err) def get_etcd_client(self): args = tuple() @@ -39,8 +42,8 @@ class Settings(object): 'host': self.config_parser.get('etcd', 'url'), 'port': self.config_parser.get('etcd', 'port'), 'ca_cert': self.config_parser.get('etcd', 'ca_cert'), - 'cert_cert': self.config_parser.get('etcd','cert_cert'), - 'cert_key': self.config_parser.get('etcd','cert_key') + 'cert_cert': self.config_parser.get('etcd', 'cert_cert'), + 'cert_key': self.config_parser.get('etcd', 'cert_key') } except configparser.Error as err: raise configparser.Error('{} in config file {}'.format(err.message, self.config_file)) from err From f980cdb4649f5d805bd939eb9596456e916f8768 Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 29 Dec 2019 23:14:39 +0500 Subject: [PATCH 048/163] Better error handling, Efforts to run non-root with occasional sudo --- scripts/ucloud | 39 +++++++++++++++++---------------- ucloud/common/etcd_wrapper.py | 14 +++++++----- ucloud/common/logging.py | 11 ++++++---- ucloud/common/network.py | 15 ++++++++----- ucloud/host/main.py | 6 ++--- ucloud/settings/__init__.py | 11 ++++++++-- ucloud/vmm/__init__.py | 41 +++++++++++++++++++++++++++++------ 7 files changed, 90 insertions(+), 47 deletions(-) diff --git a/scripts/ucloud b/scripts/ucloud index 3ddbb5a..9d05118 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -1,5 +1,4 @@ #!/usr/bin/env python3 - import argparse import logging import importlib @@ -8,13 +7,12 @@ import sys from logging.handlers import SysLogHandler -from ucloud.configure.main import configure_parser from ucloud.common.logging import NoTracebackStreamHandler +from ucloud.configure.main import configure_parser def exception_hook(exc_type, exc_value, exc_traceback): - logger = logging.getLogger(__name__) - logger.error( + logging.getLogger(__name__).error( 'Uncaught exception', exc_info=(exc_type, exc_value, exc_traceback) ) @@ -22,7 +20,25 @@ def exception_hook(exc_type, exc_value, exc_traceback): sys.excepthook = exception_hook + if __name__ == '__main__': + # Setting up root logger + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + syslog_handler = SysLogHandler(address='/dev/log') + syslog_handler.setLevel(logging.DEBUG) + syslog_formatter = logging.Formatter('%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s') + syslog_handler.setFormatter(syslog_formatter) + + stream_handler = NoTracebackStreamHandler() + stream_handler.setLevel(logging.INFO) + stream_formatter = logging.Formatter('%(message)s') + stream_handler.setFormatter(stream_formatter) + + logger.addHandler(syslog_handler) + logger.addHandler(stream_handler) + arg_parser = argparse.ArgumentParser() subparsers = arg_parser.add_subparsers(dest="command") @@ -46,21 +62,6 @@ if __name__ == '__main__': if not args.command: arg_parser.print_help() else: - # Setting up root logger - logger = logging.getLogger('ucloud') - - syslog_handler = SysLogHandler(address='/dev/log') - syslog_handler.setLevel(logging.DEBUG) - syslog_formatter = logging.Formatter('%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s') - syslog_handler.setFormatter(syslog_formatter) - - stream_handler = NoTracebackStreamHandler() - stream_handler.setLevel(logging.WARNING) - stream_formatter = logging.Formatter('%(message)s') - stream_handler.setFormatter(stream_formatter) - - logger.addHandler(syslog_handler) - logger.addHandler(stream_handler) # if we start etcd in seperate process with default settings # i.e inheriting few things from parent process etcd3 module diff --git a/ucloud/common/etcd_wrapper.py b/ucloud/common/etcd_wrapper.py index eecf4c7..5f464e1 100644 --- a/ucloud/common/etcd_wrapper.py +++ b/ucloud/common/etcd_wrapper.py @@ -29,15 +29,16 @@ def readable_errors(func): try: return func(*args, **kwargs) except etcd3.exceptions.ConnectionFailedError as err: - raise etcd3.exceptions.ConnectionFailedError('etcd connection failed') from err + raise etcd3.exceptions.ConnectionFailedError('etcd connection failed.') from err except etcd3.exceptions.ConnectionTimeoutError as err: - raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout') from err + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err except Exception: - logger.exception('Some etcd error occurred') + logger.exception('Some etcd error occured. See syslog for details.') return wrapper class Etcd3Wrapper: + @readable_errors def __init__(self, *args, **kwargs): self.client = etcd3.client(*args, **kwargs) @@ -77,9 +78,10 @@ class Etcd3Wrapper: event_queue = queue.Queue() def add_event_to_queue(event): - for e in event.events: - if e.value: - event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json)) + if hasattr(event, 'events'): + for e in event.events: + if e.value: + event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json)) self.client.add_watch_prefix_callback(key, add_event_to_queue) diff --git a/ucloud/common/logging.py b/ucloud/common/logging.py index 945f473..ba1e59d 100644 --- a/ucloud/common/logging.py +++ b/ucloud/common/logging.py @@ -7,14 +7,17 @@ class NoTracebackStreamHandler(logging.StreamHandler): info, cache = record.exc_info, record.exc_text record.exc_info, record.exc_text = None, None - if record.levelname == 'WARNING': - color = colorama.Fore.YELLOW - elif record.levelname in ['ERROR', 'EXCEPTION']: + if record.levelname in ['WARNING', 'WARN']: + color = colorama.Fore.LIGHTYELLOW_EX + elif record.levelname == 'ERROR': color = colorama.Fore.LIGHTRED_EX elif record.levelname == 'INFO': - color = colorama.Fore.LIGHTBLUE_EX + color = colorama.Fore.LIGHTGREEN_EX + elif record.levelname == 'CRITICAL': + color = colorama.Fore.LIGHTCYAN_EX else: color = colorama.Fore.WHITE + try: print(color, end='', flush=True) super().handle(record) diff --git a/ucloud/common/network.py b/ucloud/common/network.py index 629e92a..1503446 100644 --- a/ucloud/common/network.py +++ b/ucloud/common/network.py @@ -30,14 +30,14 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt=' def create_dev(script, _id, dev, ip=None): - command = [script, str(_id), dev] + command = ['sudo', '-p', 'Enter password to create network devices for vm: ', + script, str(_id), dev] if ip: command.append(ip) try: output = sp.check_output(command, stderr=sp.PIPE) - except Exception as e: + except Exception: logger.exception('Creation of interface %s failed.', dev) - print(e) return None else: return output.decode('utf-8').strip() @@ -45,9 +45,14 @@ def create_dev(script, _id, dev, ip=None): def delete_network_interface(iface): try: - sp.check_output(['ip', 'link', 'del', iface]) + sp.check_output( + [ + 'sudo', '-p', 'Enter password to remove {} network device: '.format(iface), + 'ip', 'link', 'del', iface + ], stderr=sp.PIPE + ) except Exception: - logger.exception('Interface Deletion failed') + logger.exception('Interface %s Deletion failed', iface) def find_free_port(): diff --git a/ucloud/host/main.py b/ucloud/host/main.py index 8a7dbe7..b5aeee3 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -14,10 +14,8 @@ from . import virtualmachine, logger def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" - host_pool = shared.host_pool this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) - while True: this_host.update_heartbeat() host_pool.put(this_host) @@ -43,7 +41,7 @@ def main(hostname): heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() except Exception as e: - raise e.__class__('ucloud-host heartbeat updating mechanism is not working') from e + raise Exception('ucloud-host heartbeat updating mechanism is not working') from e for events_iterator in [ shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True), @@ -87,7 +85,7 @@ def main(hostname): if __name__ == "__main__": argparser = argparse.ArgumentParser() - argparser.add_argument("hostname", help="Name of this host. e.g /v1/host/1") + argparser.add_argument("hostname", help="Name of this host. e.g uncloud1.ungleich.ch") args = argparser.parse_args() mp.set_start_method('spawn') main(args.hostname) diff --git a/ucloud/settings/__init__.py b/ucloud/settings/__init__.py index e589485..f9b358e 100644 --- a/ucloud/settings/__init__.py +++ b/ucloud/settings/__init__.py @@ -47,8 +47,15 @@ class Settings(object): } except configparser.Error as err: raise configparser.Error('{} in config file {}'.format(err.message, self.config_file)) from err - - return Etcd3Wrapper(*args, **kwargs) + else: + try: + wrapper = Etcd3Wrapper(*args, **kwargs) + except Exception as err: + logger.error('etcd connection not successfull. Please check your config file.' + '\nDetails: %s\netcd connection parameters: %s', err, kwargs) + sys.exit(1) + else: + return wrapper def read_internal_values(self): self.config_parser.read_dict({ diff --git a/ucloud/vmm/__init__.py b/ucloud/vmm/__init__.py index 1291da4..f85d7a3 100644 --- a/ucloud/vmm/__init__.py +++ b/ucloud/vmm/__init__.py @@ -91,6 +91,14 @@ class VMM: self.vmm_backend = vmm_backend self.socket_dir = os.path.join(self.vmm_backend, 'sock') + if not os.path.isdir(self.vmm_backend): + logger.info('{} does not exists. Creating it...'.format(self.vmm_backend)) + os.makedirs(self.vmm_backend, exist_ok=True) + + if not os.path.isdir(self.socket_dir): + logger.info('{} does not exists. Creating it...'.format(self.socket_dir)) + os.makedirs(self.socket_dir, exist_ok=True) + def is_running(self, uuid): sock_path = os.path.join(self.vmm_backend, uuid) try: @@ -99,8 +107,8 @@ class VMM: recv = sock.recv(4096) except Exception as err: # unix sock doesn't exists or it is closed - logger.info('VM %s sock either don\' exists or it is closed.', uuid, - 'It mean VM is stopped.', exc_info=err) + logger.debug('VM {} sock either don\' exists or it is closed. It mean VM is stopped.'.format(uuid), + exc_info=err) else: # if we receive greetings from qmp it mean VM is running if len(recv) > 0: @@ -120,16 +128,34 @@ class VMM: if self.is_running(uuid): logger.warning('Cannot start VM. It is already running.') else: - qmp_arg = ('-qmp', 'unix:{}/{},server,nowait'.format(self.vmm_backend, uuid)) + qmp_arg = ('-qmp', 'unix:{},server,nowait'.format(join_path(self.vmm_backend, uuid))) vnc_arg = ('-vnc', 'unix:{}'.format(tempfile.NamedTemporaryFile().name)) - command = [self.qemu_path, *args, *qmp_arg, *migration_args, *vnc_arg, '-daemonize'] + command = ['sudo', '-p', 'Enter password to start VM {}: '.format(uuid), + self.qemu_path, *args, *qmp_arg, *migration_args, *vnc_arg, '-daemonize'] try: sp.check_output(command, stderr=sp.PIPE) except sp.CalledProcessError as err: logger.exception('Error occurred while starting VM.\nDetail %s', err.stderr.decode('utf-8')) else: - time.sleep(2) + with suppress(sp.CalledProcessError): + sp.check_output([ + 'sudo', '-p', + 'Enter password to correct permission for uncloud-vmm\'s directory', + 'chmod', '-R', 'o=rwx,g=rwx', self.vmm_backend + ]) + + # TODO: Find some good way to check whether the virtual machine is up and + # running without relying on non-guarenteed ways. + for _ in range(10): + time.sleep(2) + status = self.get_status(uuid) + if status in ['running', 'inmigrate']: + return status + logger.warning('Timeout on VM\'s status. Shutting down VM %s', uuid) + self.stop(uuid) + # TODO: What should we do more. VM can still continue to run in background. + # If we have pid of vm we can kill it using OS. def execute_command(self, uuid, command, **kwargs): # execute_command -> sucess?, output @@ -141,12 +167,12 @@ class VMM: } sock_handle.sendall(json.dumps(command_to_execute).encode('utf-8')) output = file_handle.readline() - except Exception as err: + except Exception: logger.exception('Error occurred while executing command and getting valid output from qmp') else: try: output = json.loads(output) - except: + except Exception: logger.exception('QMP Output isn\'t valid JSON. %s', output) else: return 'return' in output, output @@ -161,6 +187,7 @@ class VMM: if success: return output['return']['status'] else: + # TODO: Think about this for a little more return 'STOPPED' def discover(self): From 29e938dc74bfea7edd0c69963efe25ec844527cb Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 29 Dec 2019 23:48:04 +0500 Subject: [PATCH 049/163] Destination Host of VM during migration now notify Source host of exact socket path --- ucloud/common/request.py | 1 + ucloud/host/main.py | 3 ++- ucloud/host/virtualmachine.py | 6 ++++-- ucloud/vmm/__init__.py | 15 ++++++++++----- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ucloud/common/request.py b/ucloud/common/request.py index 2d9be44..5705eed 100644 --- a/ucloud/common/request.py +++ b/ucloud/common/request.py @@ -19,6 +19,7 @@ class RequestType: class RequestEntry(SpecificEtcdEntryBase): def __init__(self, e): + self.destination_sock_path = None self.destination_host_key = None self.type = None # type: str self.migration = None # type: bool diff --git a/ucloud/host/main.py b/ucloud/host/main.py index b5aeee3..904f26c 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -76,7 +76,8 @@ def main(hostname): elif request_event.type == RequestType.TransferVM: host = host_pool.get(request_event.destination_host_key) if host: - vm.migrate(destination=host.hostname) + vm.migrate(destination_host=host.hostname, + destination_sock_path=request_event.destination_sock_path) else: logger.error('Host %s not found!', request_event.destination_host_key) else: diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index 6d25205..db0f7b8 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -78,6 +78,7 @@ class VM: type=RequestType.TransferVM, # Transfer VM hostname=self.host_key, # Which VM should get this request. It is source host uuid=self.uuid, # uuid of VM + destination_sock_path=join_path(self.vmm.socket_dir, self.uuid), destination_host_key=destination_host_key, # Where source host transfer VM request_prefix=settings['etcd']['request_prefix'] ) @@ -94,8 +95,9 @@ class VM: declare_stopped(self.vm) self.sync() - def migrate(self, destination): - self.vmm.transfer(src_uuid=self.uuid, dest_uuid=self.uuid, host=destination) + def migrate(self, destination_host, destination_sock_path): + self.vmm.transfer(src_uuid=self.uuid, destination_sock_path=destination_sock_path, + host=destination_host) def create_network_dev(self): command = '' diff --git a/ucloud/vmm/__init__.py b/ucloud/vmm/__init__.py index f85d7a3..9f9f5f9 100644 --- a/ucloud/vmm/__init__.py +++ b/ucloud/vmm/__init__.py @@ -42,12 +42,11 @@ class VMQMPHandles: class TransferVM(Process): - def __init__(self, src_uuid, dest_uuid, host, socket_dir): + def __init__(self, src_uuid, dest_sock_path, host, socket_dir): self.src_uuid = src_uuid - self.dest_uuid = dest_uuid self.host = host self.src_sock_path = os.path.join(socket_dir, self.src_uuid) - self.dest_sock_path = os.path.join(socket_dir, self.dest_uuid) + self.dest_sock_path = dest_sock_path super().__init__() @@ -203,6 +202,12 @@ class VMM: return output['return']['service'] return None - def transfer(self, src_uuid, dest_uuid, host): - p = TransferVM(src_uuid, dest_uuid, socket_dir=self.socket_dir, host=host) + def transfer(self, src_uuid, destination_sock_path, host): + p = TransferVM(src_uuid, destination_sock_path, socket_dir=self.socket_dir, host=host) p.start() + + # TODO: the following method should clean things that went wrong + # e.g If VM migration fails or didn't start for long time + # i.e 15 minutes we should stop the waiting VM. + def maintenace(self): + pass \ No newline at end of file From 9bdf4d2180a0748ec4ffd5cf83b104e99152ac08 Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 30 Dec 2019 14:35:07 +0500 Subject: [PATCH 050/163] Shutdown Source VM (PAUSED) on successfull migration + blackened all .py files --- setup.py | 71 +++-- ucloud/api/common_fields.py | 16 +- ucloud/api/create_image_store.py | 5 +- ucloud/api/helper.py | 44 ++- ucloud/api/main.py | 178 ++++++++---- ucloud/api/schemas.py | 275 ++++++++++++------ ucloud/common/etcd_wrapper.py | 46 ++- ucloud/common/host.py | 4 +- ucloud/common/logging.py | 12 +- ucloud/common/network.py | 40 ++- ucloud/common/request.py | 8 +- ucloud/common/schemas.py | 10 +- ucloud/common/storage_handlers.py | 53 ++-- ucloud/common/vm.py | 11 +- ucloud/configure/main.py | 66 +++-- ucloud/docs/source/conf.py | 14 +- ucloud/filescanner/main.py | 15 +- ucloud/host/main.py | 82 ++++-- ucloud/host/qmp/__init__.py | 173 ++++++----- ucloud/host/qmp/qmp.py | 22 +- ucloud/host/virtualmachine.py | 220 +++++++++----- ucloud/imagescanner/main.py | 83 ++++-- ucloud/metadata/main.py | 48 +-- ucloud/scheduler/__init__.py | 2 +- ucloud/scheduler/helper.py | 50 +++- ucloud/scheduler/main.py | 49 +++- ucloud/scheduler/tests/test_basics.py | 41 ++- .../tests/test_dead_host_mechanism.py | 30 +- ucloud/settings/__init__.py | 82 ++++-- ucloud/shared/__init__.py | 10 +- ucloud/vmm/__init__.py | 185 ++++++++---- 31 files changed, 1307 insertions(+), 638 deletions(-) diff --git a/setup.py b/setup.py index 51d21b8..956656b 100644 --- a/setup.py +++ b/setup.py @@ -7,41 +7,48 @@ with open("README.md", "r") as fh: try: import ucloud.version + version = ucloud.version.VERSION except: import subprocess - c = subprocess.check_output(['git', 'describe']) + + c = subprocess.check_output(["git", "describe"]) version = c.decode("utf-8").strip() -setup(name='ucloud', - version=version, - description='All ucloud server components.', - url='https://code.ungleich.ch/ucloud/ucloud', - long_description=long_description, - long_description_content_type='text/markdown', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', - 'Programming Language :: Python :: 3' - ], - author='ungleich', - author_email='technik@ungleich.ch', - packages=find_packages(), - install_requires=[ - 'requests', - 'Flask>=1.1.1', - 'flask-restful', - 'bitmath', - 'pyotp', - 'sshtunnel', - 'sphinx', - 'pynetbox', - 'colorama', - 'sphinx-rtd-theme', - 'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3', - 'werkzeug', 'marshmallow' - ], - scripts=['scripts/ucloud'], - data_files=[(os.path.expanduser('~/ucloud/'), ['conf/ucloud.conf'])], - zip_safe=False) +setup( + name="ucloud", + version=version, + description="All ucloud server components.", + url="https://code.ungleich.ch/ucloud/ucloud", + long_description=long_description, + long_description_content_type="text/markdown", + classifiers=[ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Programming Language :: Python :: 3", + ], + author="ungleich", + author_email="technik@ungleich.ch", + packages=find_packages(), + install_requires=[ + "requests", + "Flask>=1.1.1", + "flask-restful", + "bitmath", + "pyotp", + "sshtunnel", + "sphinx", + "pynetbox", + "colorama", + "sphinx-rtd-theme", + "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3", + "werkzeug", + "marshmallow", + ], + scripts=["scripts/ucloud"], + data_files=[ + (os.path.expanduser("~/ucloud/"), ["conf/ucloud.conf"]) + ], + zip_safe=False, +) diff --git a/ucloud/api/common_fields.py b/ucloud/api/common_fields.py index a793d26..93f9e06 100755 --- a/ucloud/api/common_fields.py +++ b/ucloud/api/common_fields.py @@ -20,12 +20,16 @@ class Field: def is_valid(self): if self.value == KeyError: - self.add_error("'{}' field is a required field".format(self.name)) + self.add_error( + "'{}' field is a required field".format(self.name) + ) else: if isinstance(self.value, Optional): pass elif not isinstance(self.value, self.type): - self.add_error("Incorrect Type for '{}' field".format(self.name)) + self.add_error( + "Incorrect Type for '{}' field".format(self.name) + ) else: self.validation() @@ -49,6 +53,10 @@ class VmUUIDField(Field): self.validation = self.vm_uuid_validation def vm_uuid_validation(self): - r = shared.etcd_client.get(os.path.join(settings['etcd']['vm_prefix'], self.uuid)) + r = shared.etcd_client.get( + os.path.join(settings["etcd"]["vm_prefix"], self.uuid) + ) if not r: - self.add_error("VM with uuid {} does not exists".format(self.uuid)) + self.add_error( + "VM with uuid {} does not exists".format(self.uuid) + ) diff --git a/ucloud/api/create_image_store.py b/ucloud/api/create_image_store.py index 978a182..a433ce3 100755 --- a/ucloud/api/create_image_store.py +++ b/ucloud/api/create_image_store.py @@ -14,4 +14,7 @@ data = { "attributes": {"list": [], "key": [], "pool": "images"}, } -shared.etcd_client.put(os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) +shared.etcd_client.put( + os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex), + json.dumps(data), +) diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index e275e46..a77a151 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -16,21 +16,23 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": settings['otp']['auth_name'], - "auth_token": TOTP(settings['otp']['auth_seed']).now(), - "auth_realm": settings['otp']['auth_realm'], + "auth_name": settings["otp"]["auth_name"], + "auth_token": TOTP(settings["otp"]["auth_seed"]).now(), + "auth_realm": settings["otp"]["auth_realm"], "name": name, "realm": realm, "token": token, } except binascii.Error as err: logger.error( - "Cannot compute OTP for seed: {}".format(settings['otp']['auth_seed']) + "Cannot compute OTP for seed: {}".format( + settings["otp"]["auth_seed"] + ) ) return 400 response = requests.post( - settings['otp']['verification_controller_url'], json=data + settings["otp"]["verification_controller_url"], json=data ) return response.status_code @@ -43,7 +45,8 @@ def resolve_vm_name(name, owner): """ result = next( filter( - lambda vm: vm.value["owner"] == owner and vm.value["name"] == name, + lambda vm: vm.value["owner"] == owner + and vm.value["name"] == name, shared.vm_pool.vms, ), None, @@ -80,18 +83,27 @@ def resolve_image_name(name, etcd_client): """ store_name, image_name = store_name_and_image_name except Exception: - raise ValueError("Image name not in correct format i.e {store_name}:{image_name}") + raise ValueError( + "Image name not in correct format i.e {store_name}:{image_name}" + ) - images = etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True) + images = etcd_client.get_prefix( + settings["etcd"]["image_prefix"], value_in_json=True + ) # Try to find image with name == image_name and store_name == store_name try: - image = next(filter(lambda im: im.value['name'] == image_name - and im.value['store_name'] == store_name, images)) + image = next( + filter( + lambda im: im.value["name"] == image_name + and im.value["store_name"] == store_name, + images, + ) + ) except StopIteration: raise KeyError("No image with name {} found.".format(name)) else: - image_uuid = image.key.split('/')[-1] + image_uuid = image.key.split("/")[-1] return image_uuid @@ -100,7 +112,9 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): +def generate_mac( + uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" +): mac = random_bytes() if oui: if type(oui) == str: @@ -131,7 +145,9 @@ def get_ip_addr(mac_address, device): and is connected/neighbor of arg:device """ try: - output = sp.check_output(['ip', '-6', 'neigh', 'show', 'dev', device], stderr=sp.PIPE) + output = sp.check_output( + ["ip", "-6", "neigh", "show", "dev", device], stderr=sp.PIPE + ) except sp.CalledProcessError: return None else: @@ -160,7 +176,7 @@ def mac2ipv6(mac, prefix): # format output ipv6_parts = [str(0)] * 4 for i in range(0, len(parts), 2): - ipv6_parts.append("".join(parts[i:i + 2])) + ipv6_parts.append("".join(parts[i : i + 2])) lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) prefix = ipaddress.IPv6Address(prefix) diff --git a/ucloud/api/main.py b/ucloud/api/main.py index 91cbead..c63babf 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -43,7 +43,7 @@ def handle_exception(e): return e # now you're handling non-HTTP exceptions only - return {'message': 'Server Error'}, 500 + return {"message": "Server Error"}, 500 class CreateVM(Resource): @@ -55,7 +55,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) + vm_key = join_path(settings["etcd"]["vm_prefix"], vm_uuid) specs = { "cpu": validator.specs["cpu"], "ram": validator.specs["ram"], @@ -63,8 +63,12 @@ class CreateVM(Resource): "hdd": validator.specs["hdd"], } macs = [generate_mac() for _ in range(len(data["network"]))] - tap_ids = [counters.increment_etcd_counter(shared.etcd_client, "/v1/counter/tap") - for _ in range(len(data["network"]))] + tap_ids = [ + counters.increment_etcd_counter( + shared.etcd_client, "/v1/counter/tap" + ) + for _ in range(len(data["network"])) + ] vm_entry = { "name": data["vm_name"], "owner": data["name"], @@ -77,14 +81,15 @@ class CreateVM(Resource): "vnc_socket": "", "network": list(zip(data["network"], macs, tap_ids)), "metadata": {"ssh-keys": []}, - "in_migration": False + "in_migration": False, } shared.etcd_client.put(vm_key, vm_entry, value_in_json=True) # Create ScheduleVM Request r = RequestEntry.from_scratch( - type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=settings['etcd']['request_prefix'] + type=RequestType.ScheduleVM, + uuid=vm_uuid, + request_prefix=settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) @@ -99,7 +104,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data["uuid"]) + join_path(settings["etcd"]["vm_prefix"], data["uuid"]) ) vm_value = vm.value.copy() vm_value["ip"] = [] @@ -107,13 +112,15 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( join_path( - settings['etcd']['network_prefix'], + settings["etcd"]["network_prefix"], data["name"], network_name, ), value_in_json=True, ) - ipv6_addr = network.value.get("ipv6").split("::")[0] + "::" + ipv6_addr = ( + network.value.get("ipv6").split("::")[0] + "::" + ) vm_value["ip"].append(mac2ipv6(mac, ipv6_addr)) vm.value = vm_value return vm.value @@ -128,7 +135,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = shared.etcd_client.get( - join_path(settings['etcd']['file_prefix'], data["uuid"]) + join_path(settings["etcd"]["file_prefix"], data["uuid"]) ) file_entry_value = json.loads(file_entry.value) @@ -141,7 +148,9 @@ class CreateImage(Resource): "visibility": "public", } shared.etcd_client.put( - join_path(settings['etcd']['image_prefix'], data["uuid"]), + join_path( + settings["etcd"]["image_prefix"], data["uuid"] + ), json.dumps(image_entry_json), ) @@ -153,11 +162,9 @@ class ListPublicImages(Resource): @staticmethod def get(): images = shared.etcd_client.get_prefix( - settings['etcd']['image_prefix'], value_in_json=True + settings["etcd"]["image_prefix"], value_in_json=True ) - r = { - "images": [] - } + r = {"images": []} for image in images: image_key = "{}:{}".format( image.value["store_name"], image.value["name"] @@ -176,7 +183,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data["uuid"]) + join_path(settings["etcd"]["vm_prefix"], data["uuid"]) ) action = data["action"] @@ -184,13 +191,19 @@ class VMAction(Resource): action = "schedule" if action == "delete" and vm_entry.hostname == "": - if shared.storage_handler.is_vm_image_exists(vm_entry.uuid): - r_status = shared.storage_handler.delete_vm_image(vm_entry.uuid) + if shared.storage_handler.is_vm_image_exists( + vm_entry.uuid + ): + r_status = shared.storage_handler.delete_vm_image( + vm_entry.uuid + ) if r_status: shared.etcd_client.client.delete(vm_entry.key) return {"message": "VM successfully deleted"} else: - logger.error("Some Error Occurred while deleting VM") + logger.error( + "Some Error Occurred while deleting VM" + ) return {"message": "VM deletion unsuccessfull"} else: shared.etcd_client.client.delete(vm_entry.key) @@ -200,10 +213,13 @@ class VMAction(Resource): type="{}VM".format(action.title()), uuid=data["uuid"], hostname=vm_entry.hostname, - request_prefix=settings['etcd']['request_prefix'] + request_prefix=settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) - return {"message": "VM {} Queued".format(action.title())}, 200 + return ( + {"message": "VM {} Queued".format(action.title())}, + 200, + ) else: return validator.get_errors(), 400 @@ -216,15 +232,21 @@ class VMMigration(Resource): if validator.is_valid(): vm = shared.vm_pool.get(data["uuid"]) - r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, - uuid=vm.uuid, - hostname=join_path( - settings['etcd']['host_prefix'], validator.destination.value - ), - request_prefix=settings['etcd']['request_prefix']) + r = RequestEntry.from_scratch( + type=RequestType.InitVMMigration, + uuid=vm.uuid, + hostname=join_path( + settings["etcd"]["host_prefix"], + validator.destination.value, + ), + request_prefix=settings["etcd"]["request_prefix"], + ) shared.request_pool.put(r) - return {"message": "VM Migration Initialization Queued"}, 200 + return ( + {"message": "VM Migration Initialization Queued"}, + 200, + ) else: return validator.get_errors(), 400 @@ -237,10 +259,12 @@ class ListUserVM(Resource): if validator.is_valid(): vms = shared.etcd_client.get_prefix( - settings['etcd']['vm_prefix'], value_in_json=True + settings["etcd"]["vm_prefix"], value_in_json=True ) return_vms = [] - user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) + user_vms = filter( + lambda v: v.value["owner"] == data["name"], vms + ) for vm in user_vms: return_vms.append( { @@ -249,9 +273,7 @@ class ListUserVM(Resource): "specs": vm.value["specs"], "status": vm.value["status"], "hostname": vm.value["hostname"], - "vnc_socket": None - if vm.value.get("vnc_socket", None) is None - else vm.value["vnc_socket"], + "vnc_socket": vm.value.get("vnc_socket", None), } ) if return_vms: @@ -270,11 +292,13 @@ class ListUserFiles(Resource): if validator.is_valid(): files = shared.etcd_client.get_prefix( - settings['etcd']['file_prefix'], value_in_json=True + settings["etcd"]["file_prefix"], value_in_json=True ) return_files = [] user_files = list( - filter(lambda f: f.value["owner"] == data["name"], files) + filter( + lambda f: f.value["owner"] == data["name"], files + ) ) for file in user_files: return_files.append( @@ -294,14 +318,18 @@ class CreateHost(Resource): data = request.json validator = schemas.CreateHostSchema(data) if validator.is_valid(): - host_key = join_path(settings['etcd']['host_prefix'], uuid4().hex) + host_key = join_path( + settings["etcd"]["host_prefix"], uuid4().hex + ) host_entry = { "specs": data["specs"], "hostname": data["hostname"], "status": "DEAD", "last_heartbeat": "", } - shared.etcd_client.put(host_key, host_entry, value_in_json=True) + shared.etcd_client.put( + host_key, host_entry, value_in_json=True + ) return {"message": "Host Created"}, 200 @@ -333,7 +361,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - settings['etcd']['user_prefix'], + settings["etcd"]["user_prefix"], data["realm"], data["name"], "key", @@ -343,25 +371,30 @@ class GetSSHKeys(Resource): ) keys = { - key.key.split("/")[-1]: key.value for key in etcd_entry + key.key.split("/")[-1]: key.value + for key in etcd_entry } return {"keys": keys} else: # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + settings["etcd"]["user_prefix"], data["realm"], data["name"], "key", data["key_name"], ) - etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) + etcd_entry = shared.etcd_client.get( + etcd_key, value_in_json=True + ) if etcd_entry: return { "keys": { - etcd_entry.key.split("/")[-1]: etcd_entry.value + etcd_entry.key.split("/")[ + -1 + ]: etcd_entry.value } } else: @@ -379,13 +412,15 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + settings["etcd"]["user_prefix"], data["realm"], data["name"], "key", data["key_name"], ) - etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) + etcd_entry = shared.etcd_client.get( + etcd_key, value_in_json=True + ) if etcd_entry: return { "message": "Key with name '{}' already exists".format( @@ -394,7 +429,9 @@ class AddSSHKey(Resource): } else: # Key Not Found. It implies user' haven't added any key yet. - shared.etcd_client.put(etcd_key, data["key"], value_in_json=True) + shared.etcd_client.put( + etcd_key, data["key"], value_in_json=True + ) return {"message": "Key added successfully"} else: return validator.get_errors(), 400 @@ -409,13 +446,15 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + settings["etcd"]["user_prefix"], data["realm"], data["name"], "key", data["key_name"], ) - etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) + etcd_entry = shared.etcd_client.get( + etcd_key, value_in_json=True + ) if etcd_entry: shared.etcd_client.client.delete(etcd_key) return {"message": "Key successfully removed."} @@ -446,15 +485,17 @@ class CreateNetwork(Resource): if validator.user.value: try: nb = pynetbox.api( - url=settings['netbox']['url'], - token=settings['netbox']['token'], + url=settings["netbox"]["url"], + token=settings["netbox"]["token"], ) nb_prefix = nb.ipam.prefixes.get( - prefix=settings['network']['prefix'] + prefix=settings["network"]["prefix"] ) prefix = nb_prefix.available_prefixes.create( data={ - "prefix_length": int(settings['network']['prefix_length']), + "prefix_length": int( + settings["network"]["prefix_length"] + ), "description": '{}\'s network "{}"'.format( data["name"], data["network_name"] ), @@ -463,18 +504,22 @@ class CreateNetwork(Resource): ) except Exception as err: app.logger.error(err) - return {"message": "Error occured while creating network."} + return { + "message": "Error occured while creating network." + } else: network_entry["ipv6"] = prefix["prefix"] else: network_entry["ipv6"] = "fd00::/64" network_key = join_path( - settings['etcd']['network_prefix'], - data['name'], - data['network_name'], + settings["etcd"]["network_prefix"], + data["name"], + data["network_name"], + ) + shared.etcd_client.put( + network_key, network_entry, value_in_json=True ) - shared.etcd_client.put(network_key, network_entry, value_in_json=True) return {"message": "Network successfully added."} else: return validator.get_errors(), 400 @@ -488,9 +533,11 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - settings['etcd']['network_prefix'], data["name"] + settings["etcd"]["network_prefix"], data["name"] + ) + networks = shared.etcd_client.get_prefix( + prefix, value_in_json=True ) - networks = shared.etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] for net in networks: net.value["name"] = net.key.split("/")[-1] @@ -524,7 +571,11 @@ api.add_resource(CreateNetwork, "/network/create") def main(): - image_stores = list(shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix'], value_in_json=True)) + image_stores = list( + shared.etcd_client.get_prefix( + settings["etcd"]["image_store_prefix"], value_in_json=True + ) + ) if len(image_stores) == 0: data = { "is_public": True, @@ -534,7 +585,12 @@ def main(): "attributes": {"list": [], "key": [], "pool": "images"}, } - shared.etcd_client.put(join_path(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) + shared.etcd_client.put( + join_path( + settings["etcd"]["image_store_prefix"], uuid4().hex + ), + json.dumps(data), + ) app.run(host="::", debug=True) diff --git a/ucloud/api/schemas.py b/ucloud/api/schemas.py index d639be4..a848a7d 100755 --- a/ucloud/api/schemas.py +++ b/ucloud/api/schemas.py @@ -80,7 +80,12 @@ class OTPSchema(BaseSchema): super().__init__(data=data, fields=_fields) def validation(self): - if check_otp(self.name.value, self.realm.value, self.token.value) != 200: + if ( + check_otp( + self.name.value, self.realm.value, self.token.value + ) + != 200 + ): self.add_error("Wrong Credentials") @@ -92,7 +97,9 @@ class CreateImageSchema(BaseSchema): # Fields self.uuid = Field("uuid", str, data.get("uuid", KeyError)) self.name = Field("name", str, data.get("name", KeyError)) - self.image_store = Field("image_store", str, data.get("image_store", KeyError)) + self.image_store = Field( + "image_store", str, data.get("image_store", KeyError) + ) # Validations self.uuid.validation = self.file_uuid_validation @@ -103,34 +110,52 @@ class CreateImageSchema(BaseSchema): super().__init__(data, fields) def file_uuid_validation(self): - file_entry = shared.etcd_client.get(os.path.join(settings['etcd']['file_prefix'], self.uuid.value)) + file_entry = shared.etcd_client.get( + os.path.join( + settings["etcd"]["file_prefix"], self.uuid.value + ) + ) if file_entry is None: self.add_error( - "Image File with uuid '{}' Not Found".format(self.uuid.value) + "Image File with uuid '{}' Not Found".format( + self.uuid.value + ) ) def image_store_name_validation(self): - image_stores = list(shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix'])) + image_stores = list( + shared.etcd_client.get_prefix( + settings["etcd"]["image_store_prefix"] + ) + ) image_store = next( filter( - lambda s: json.loads(s.value)["name"] == self.image_store.value, + lambda s: json.loads(s.value)["name"] + == self.image_store.value, image_stores, ), None, ) if not image_store: - self.add_error("Store '{}' does not exists".format(self.image_store.value)) + self.add_error( + "Store '{}' does not exists".format( + self.image_store.value + ) + ) # Host Operations + class CreateHostSchema(OTPSchema): def __init__(self, data): self.parsed_specs = {} # Fields self.specs = Field("specs", dict, data.get("specs", KeyError)) - self.hostname = Field("hostname", str, data.get("hostname", KeyError)) + self.hostname = Field( + "hostname", str, data.get("hostname", KeyError) + ) # Validation self.specs.validation = self.specs_validation @@ -142,22 +167,28 @@ class CreateHostSchema(OTPSchema): def specs_validation(self): ALLOWED_BASE = 10 - _cpu = self.specs.value.get('cpu', KeyError) - _ram = self.specs.value.get('ram', KeyError) - _os_ssd = self.specs.value.get('os-ssd', KeyError) - _hdd = self.specs.value.get('hdd', KeyError) + _cpu = self.specs.value.get("cpu", KeyError) + _ram = self.specs.value.get("ram", KeyError) + _os_ssd = self.specs.value.get("os-ssd", KeyError) + _hdd = self.specs.value.get("hdd", KeyError) if KeyError in [_cpu, _ram, _os_ssd, _hdd]: - self.add_error("You must specify CPU, RAM and OS-SSD in your specs") + self.add_error( + "You must specify CPU, RAM and OS-SSD in your specs" + ) return None try: parsed_ram = bitmath.parse_string_unsafe(_ram) parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) if parsed_ram.base != ALLOWED_BASE: - self.add_error("Your specified RAM is not in correct units") + self.add_error( + "Your specified RAM is not in correct units" + ) if parsed_os_ssd.base != ALLOWED_BASE: - self.add_error("Your specified OS-SSD is not in correct units") + self.add_error( + "Your specified OS-SSD is not in correct units" + ) if _cpu < 1: self.add_error("CPU must be atleast 1") @@ -172,7 +203,9 @@ class CreateHostSchema(OTPSchema): for hdd in _hdd: _parsed_hdd = bitmath.parse_string_unsafe(hdd) if _parsed_hdd.base != ALLOWED_BASE: - self.add_error("Your specified HDD is not in correct units") + self.add_error( + "Your specified HDD is not in correct units" + ) break else: parsed_hdd.append(str(_parsed_hdd)) @@ -183,15 +216,17 @@ class CreateHostSchema(OTPSchema): else: if self.get_errors(): self.specs = { - 'cpu': _cpu, - 'ram': str(parsed_ram), - 'os-ssd': str(parsed_os_ssd), - 'hdd': parsed_hdd + "cpu": _cpu, + "ram": str(parsed_ram), + "os-ssd": str(parsed_os_ssd), + "hdd": parsed_hdd, } def validation(self): if self.realm.value != "ungleich-admin": - self.add_error("Invalid Credentials/Insufficient Permission") + self.add_error( + "Invalid Credentials/Insufficient Permission" + ) # VM Operations @@ -203,9 +238,13 @@ class CreateVMSchema(OTPSchema): # Fields self.specs = Field("specs", dict, data.get("specs", KeyError)) - self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError)) + self.vm_name = Field( + "vm_name", str, data.get("vm_name", KeyError) + ) self.image = Field("image", str, data.get("image", KeyError)) - self.network = Field("network", list, data.get("network", KeyError)) + self.network = Field( + "network", list, data.get("network", KeyError) + ) # Validation self.image.validation = self.image_validation @@ -219,17 +258,25 @@ class CreateVMSchema(OTPSchema): def image_validation(self): try: - image_uuid = helper.resolve_image_name(self.image.value, shared.etcd_client) + image_uuid = helper.resolve_image_name( + self.image.value, shared.etcd_client + ) except Exception as e: - logger.exception("Cannot resolve image name = %s", self.image.value) + logger.exception( + "Cannot resolve image name = %s", self.image.value + ) self.add_error(str(e)) else: self.image_uuid = image_uuid def vm_name_validation(self): - if resolve_vm_name(name=self.vm_name.value, owner=self.name.value): + if resolve_vm_name( + name=self.vm_name.value, owner=self.name.value + ): self.add_error( - 'VM with same name "{}" already exists'.format(self.vm_name.value) + 'VM with same name "{}" already exists'.format( + self.vm_name.value + ) ) def network_validation(self): @@ -237,32 +284,46 @@ class CreateVMSchema(OTPSchema): if _network: for net in _network: - network = shared.etcd_client.get(os.path.join(settings['etcd']['network_prefix'], - self.name.value, - net), value_in_json=True) + network = shared.etcd_client.get( + os.path.join( + settings["etcd"]["network_prefix"], + self.name.value, + net, + ), + value_in_json=True, + ) if not network: - self.add_error("Network with name {} does not exists" \ - .format(net)) + self.add_error( + "Network with name {} does not exists".format( + net + ) + ) def specs_validation(self): ALLOWED_BASE = 10 - _cpu = self.specs.value.get('cpu', KeyError) - _ram = self.specs.value.get('ram', KeyError) - _os_ssd = self.specs.value.get('os-ssd', KeyError) - _hdd = self.specs.value.get('hdd', KeyError) + _cpu = self.specs.value.get("cpu", KeyError) + _ram = self.specs.value.get("ram", KeyError) + _os_ssd = self.specs.value.get("os-ssd", KeyError) + _hdd = self.specs.value.get("hdd", KeyError) if KeyError in [_cpu, _ram, _os_ssd, _hdd]: - self.add_error("You must specify CPU, RAM and OS-SSD in your specs") + self.add_error( + "You must specify CPU, RAM and OS-SSD in your specs" + ) return None try: parsed_ram = bitmath.parse_string_unsafe(_ram) parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) if parsed_ram.base != ALLOWED_BASE: - self.add_error("Your specified RAM is not in correct units") + self.add_error( + "Your specified RAM is not in correct units" + ) if parsed_os_ssd.base != ALLOWED_BASE: - self.add_error("Your specified OS-SSD is not in correct units") + self.add_error( + "Your specified OS-SSD is not in correct units" + ) if _cpu < 1: self.add_error("CPU must be atleast 1") @@ -277,7 +338,9 @@ class CreateVMSchema(OTPSchema): for hdd in _hdd: _parsed_hdd = bitmath.parse_string_unsafe(hdd) if _parsed_hdd.base != ALLOWED_BASE: - self.add_error("Your specified HDD is not in correct units") + self.add_error( + "Your specified HDD is not in correct units" + ) break else: parsed_hdd.append(str(_parsed_hdd)) @@ -288,21 +351,24 @@ class CreateVMSchema(OTPSchema): else: if self.get_errors(): self.specs = { - 'cpu': _cpu, - 'ram': str(parsed_ram), - 'os-ssd': str(parsed_os_ssd), - 'hdd': parsed_hdd + "cpu": _cpu, + "ram": str(parsed_ram), + "os-ssd": str(parsed_os_ssd), + "hdd": parsed_hdd, } class VMStatusSchema(OTPSchema): def __init__(self, data): data["uuid"] = ( - resolve_vm_name( - name=data.get("vm_name", None), - owner=(data.get("in_support_of", None) or data.get("name", None)), - ) - or KeyError + resolve_vm_name( + name=data.get("vm_name", None), + owner=( + data.get("in_support_of", None) + or data.get("name", None) + ), + ) + or KeyError ) self.uuid = VmUUIDField(data) @@ -313,7 +379,8 @@ class VMStatusSchema(OTPSchema): def validation(self): vm = shared.vm_pool.get(self.uuid.value) if not ( - vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + vm.value["owner"] == self.name.value + or self.realm.value == "ungleich-admin" ): self.add_error("Invalid User") @@ -321,11 +388,14 @@ class VMStatusSchema(OTPSchema): class VmActionSchema(OTPSchema): def __init__(self, data): data["uuid"] = ( - resolve_vm_name( - name=data.get("vm_name", None), - owner=(data.get("in_support_of", None) or data.get("name", None)), - ) - or KeyError + resolve_vm_name( + name=data.get("vm_name", None), + owner=( + data.get("in_support_of", None) + or data.get("name", None) + ), + ) + or KeyError ) self.uuid = VmUUIDField(data) self.action = Field("action", str, data.get("action", KeyError)) @@ -340,20 +410,23 @@ class VmActionSchema(OTPSchema): allowed_actions = ["start", "stop", "delete"] if self.action.value not in allowed_actions: self.add_error( - "Invalid Action. Allowed Actions are {}".format(allowed_actions) + "Invalid Action. Allowed Actions are {}".format( + allowed_actions + ) ) def validation(self): vm = shared.vm_pool.get(self.uuid.value) if not ( - vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + vm.value["owner"] == self.name.value + or self.realm.value == "ungleich-admin" ): self.add_error("Invalid User") if ( - self.action.value == "start" - and vm.status == VMStatus.running - and vm.hostname != "" + self.action.value == "start" + and vm.status == VMStatus.running + and vm.hostname != "" ): self.add_error("VM Already Running") @@ -367,15 +440,20 @@ class VmActionSchema(OTPSchema): class VmMigrationSchema(OTPSchema): def __init__(self, data): data["uuid"] = ( - resolve_vm_name( - name=data.get("vm_name", None), - owner=(data.get("in_support_of", None) or data.get("name", None)), - ) - or KeyError + resolve_vm_name( + name=data.get("vm_name", None), + owner=( + data.get("in_support_of", None) + or data.get("name", None) + ), + ) + or KeyError ) self.uuid = VmUUIDField(data) - self.destination = Field("destination", str, data.get("destination", KeyError)) + self.destination = Field( + "destination", str, data.get("destination", KeyError) + ) self.destination.validation = self.destination_validation @@ -384,9 +462,18 @@ class VmMigrationSchema(OTPSchema): def destination_validation(self): hostname = self.destination.value - host = next(filter(lambda h: h.hostname == hostname, shared.host_pool.hosts), None) + host = next( + filter( + lambda h: h.hostname == hostname, shared.host_pool.hosts + ), + None, + ) if not host: - self.add_error("No Such Host ({}) exists".format(self.destination.value)) + self.add_error( + "No Such Host ({}) exists".format( + self.destination.value + ) + ) elif host.status != HostStatus.alive: self.add_error("Destination Host is dead") else: @@ -395,20 +482,27 @@ class VmMigrationSchema(OTPSchema): def validation(self): vm = shared.vm_pool.get(self.uuid.value) if not ( - vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + vm.value["owner"] == self.name.value + or self.realm.value == "ungleich-admin" ): self.add_error("Invalid User") if vm.status != VMStatus.running: self.add_error("Can't migrate non-running VM") - if vm.hostname == os.path.join(settings['etcd']['host_prefix'], self.destination.value): - self.add_error("Destination host couldn't be same as Source Host") + if vm.hostname == os.path.join( + settings["etcd"]["host_prefix"], self.destination.value + ): + self.add_error( + "Destination host couldn't be same as Source Host" + ) class AddSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field("key_name", str, data.get("key_name", KeyError)) + self.key_name = Field( + "key_name", str, data.get("key_name", KeyError) + ) self.key = Field("key", str, data.get("key_name", KeyError)) fields = [self.key_name, self.key] @@ -417,7 +511,9 @@ class AddSSHSchema(OTPSchema): class RemoveSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field("key_name", str, data.get("key_name", KeyError)) + self.key_name = Field( + "key_name", str, data.get("key_name", KeyError) + ) fields = [self.key_name] super().__init__(data=data, fields=fields) @@ -425,7 +521,9 @@ class RemoveSSHSchema(OTPSchema): class GetSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field("key_name", str, data.get("key_name", None)) + self.key_name = Field( + "key_name", str, data.get("key_name", None) + ) fields = [self.key_name] super().__init__(data=data, fields=fields) @@ -433,7 +531,9 @@ class GetSSHSchema(OTPSchema): class CreateNetwork(OTPSchema): def __init__(self, data): - self.network_name = Field("network_name", str, data.get("network_name", KeyError)) + self.network_name = Field( + "network_name", str, data.get("network_name", KeyError) + ) self.type = Field("type", str, data.get("type", KeyError)) self.user = Field("user", bool, bool(data.get("user", False))) @@ -444,15 +544,26 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = shared.etcd_client.get(os.path.join(settings['etcd']['network_prefix'], - self.name.value, - self.network_name.value), - value_in_json=True) + network = shared.etcd_client.get( + os.path.join( + settings["etcd"]["network_prefix"], + self.name.value, + self.network_name.value, + ), + value_in_json=True, + ) if network: - self.add_error("Network with name {} already exists" \ - .format(self.network_name.value)) + self.add_error( + "Network with name {} already exists".format( + self.network_name.value + ) + ) def network_type_validation(self): supported_network_types = ["vxlan"] if self.type.value not in supported_network_types: - self.add_error("Unsupported Network Type. Supported network types are {}".format(supported_network_types)) + self.add_error( + "Unsupported Network Type. Supported network types are {}".format( + supported_network_types + ) + ) diff --git a/ucloud/common/etcd_wrapper.py b/ucloud/common/etcd_wrapper.py index 5f464e1..7367a6c 100644 --- a/ucloud/common/etcd_wrapper.py +++ b/ucloud/common/etcd_wrapper.py @@ -8,7 +8,7 @@ from functools import wraps from . import logger -PseudoEtcdMeta = namedtuple('PseudoEtcdMeta', ['key']) +PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) class EtcdEntry: @@ -16,8 +16,8 @@ class EtcdEntry: # value: str def __init__(self, meta, value, value_in_json=False): - self.key = meta.key.decode('utf-8') - self.value = value.decode('utf-8') + self.key = meta.key.decode("utf-8") + self.value = value.decode("utf-8") if value_in_json: self.value = json.loads(self.value) @@ -29,11 +29,18 @@ def readable_errors(func): try: return func(*args, **kwargs) except etcd3.exceptions.ConnectionFailedError as err: - raise etcd3.exceptions.ConnectionFailedError('etcd connection failed.') from err + raise etcd3.exceptions.ConnectionFailedError( + "etcd connection failed." + ) from err except etcd3.exceptions.ConnectionTimeoutError as err: - raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err + raise etcd3.exceptions.ConnectionTimeoutError( + "etcd connection timeout." + ) from err except Exception: - logger.exception('Some etcd error occured. See syslog for details.') + logger.exception( + "Some etcd error occured. See syslog for details." + ) + return wrapper @@ -56,7 +63,7 @@ class Etcd3Wrapper: _value = json.dumps(_value) if not isinstance(_key, str): - _key = _key.decode('utf-8') + _key = _key.decode("utf-8") return self.client.put(_key, _value, **kwargs) @@ -70,18 +77,25 @@ class Etcd3Wrapper: @readable_errors def watch_prefix(self, key, timeout=0, value_in_json=False): - timeout_event = EtcdEntry(PseudoEtcdMeta(key=b'TIMEOUT'), - value=str.encode(json.dumps({'status': 'TIMEOUT', - 'type': 'TIMEOUT'})), - value_in_json=value_in_json) + timeout_event = EtcdEntry( + PseudoEtcdMeta(key=b"TIMEOUT"), + value=str.encode( + json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"}) + ), + value_in_json=value_in_json, + ) event_queue = queue.Queue() def add_event_to_queue(event): - if hasattr(event, 'events'): + if hasattr(event, "events"): for e in event.events: if e.value: - event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json)) + event_queue.put( + EtcdEntry( + e, e.value, value_in_json=value_in_json + ) + ) self.client.add_watch_prefix_callback(key, add_event_to_queue) @@ -96,4 +110,8 @@ class Etcd3Wrapper: class PsuedoEtcdEntry(EtcdEntry): def __init__(self, key, value, value_in_json=False): - super().__init__(PseudoEtcdMeta(key=key.encode('utf-8')), value, value_in_json=value_in_json) + super().__init__( + PseudoEtcdMeta(key=key.encode("utf-8")), + value, + value_in_json=value_in_json, + ) diff --git a/ucloud/common/host.py b/ucloud/common/host.py index ccbf7a8..191a2c0 100644 --- a/ucloud/common/host.py +++ b/ucloud/common/host.py @@ -29,7 +29,9 @@ class HostEntry(SpecificEtcdEntryBase): self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S") def is_alive(self): - last_heartbeat = datetime.strptime(self.last_heartbeat, "%Y-%m-%d %H:%M:%S") + last_heartbeat = datetime.strptime( + self.last_heartbeat, "%Y-%m-%d %H:%M:%S" + ) delta = datetime.now() - last_heartbeat if delta.total_seconds() > 60: return False diff --git a/ucloud/common/logging.py b/ucloud/common/logging.py index ba1e59d..9e0d2be 100644 --- a/ucloud/common/logging.py +++ b/ucloud/common/logging.py @@ -7,21 +7,21 @@ class NoTracebackStreamHandler(logging.StreamHandler): info, cache = record.exc_info, record.exc_text record.exc_info, record.exc_text = None, None - if record.levelname in ['WARNING', 'WARN']: + if record.levelname in ["WARNING", "WARN"]: color = colorama.Fore.LIGHTYELLOW_EX - elif record.levelname == 'ERROR': + elif record.levelname == "ERROR": color = colorama.Fore.LIGHTRED_EX - elif record.levelname == 'INFO': + elif record.levelname == "INFO": color = colorama.Fore.LIGHTGREEN_EX - elif record.levelname == 'CRITICAL': + elif record.levelname == "CRITICAL": color = colorama.Fore.LIGHTCYAN_EX else: color = colorama.Fore.WHITE try: - print(color, end='', flush=True) + print(color, end="", flush=True) super().handle(record) finally: record.exc_info = info record.exc_text = cache - print(colorama.Style.RESET_ALL, end='', flush=True) + print(colorama.Style.RESET_ALL, end="", flush=True) diff --git a/ucloud/common/network.py b/ucloud/common/network.py index 1503446..61dbd64 100644 --- a/ucloud/common/network.py +++ b/ucloud/common/network.py @@ -11,7 +11,9 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): +def generate_mac( + uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" +): mac = random_bytes() if oui: if type(oui) == str: @@ -30,35 +32,51 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt=' def create_dev(script, _id, dev, ip=None): - command = ['sudo', '-p', 'Enter password to create network devices for vm: ', - script, str(_id), dev] + command = [ + "sudo", + "-p", + "Enter password to create network devices for vm: ", + script, + str(_id), + dev, + ] if ip: command.append(ip) try: output = sp.check_output(command, stderr=sp.PIPE) except Exception: - logger.exception('Creation of interface %s failed.', dev) + logger.exception("Creation of interface %s failed.", dev) return None else: - return output.decode('utf-8').strip() + return output.decode("utf-8").strip() def delete_network_interface(iface): try: sp.check_output( [ - 'sudo', '-p', 'Enter password to remove {} network device: '.format(iface), - 'ip', 'link', 'del', iface - ], stderr=sp.PIPE + "sudo", + "-p", + "Enter password to remove {} network device: ".format( + iface + ), + "ip", + "link", + "del", + iface, + ], + stderr=sp.PIPE, ) except Exception: - logger.exception('Interface %s Deletion failed', iface) + logger.exception("Interface %s Deletion failed", iface) def find_free_port(): - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + with closing( + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ) as s: try: - s.bind(('', 0)) + s.bind(("", 0)) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except Exception: return None diff --git a/ucloud/common/request.py b/ucloud/common/request.py index 5705eed..a8c2d0a 100644 --- a/ucloud/common/request.py +++ b/ucloud/common/request.py @@ -17,7 +17,6 @@ class RequestType: class RequestEntry(SpecificEtcdEntryBase): - def __init__(self, e): self.destination_sock_path = None self.destination_host_key = None @@ -30,8 +29,11 @@ class RequestEntry(SpecificEtcdEntryBase): @classmethod def from_scratch(cls, request_prefix, **kwargs): - e = PsuedoEtcdEntry(join(request_prefix, uuid4().hex), - value=json.dumps(kwargs).encode("utf-8"), value_in_json=True) + e = PsuedoEtcdEntry( + join(request_prefix, uuid4().hex), + value=json.dumps(kwargs).encode("utf-8"), + value_in_json=True, + ) return cls(e) diff --git a/ucloud/common/schemas.py b/ucloud/common/schemas.py index a592ec2..04978a5 100644 --- a/ucloud/common/schemas.py +++ b/ucloud/common/schemas.py @@ -14,7 +14,7 @@ class StorageUnit(fields.Field): class SpecsSchema(Schema): cpu = fields.Int() ram = StorageUnit() - os_ssd = StorageUnit(data_key='os-ssd', attribute='os-ssd') + os_ssd = StorageUnit(data_key="os-ssd", attribute="os-ssd") hdd = fields.List(StorageUnit()) @@ -29,11 +29,13 @@ class VMSchema(Schema): image_uuid = fields.Str() hostname = fields.Str() metadata = fields.Dict() - network = fields.List(fields.Tuple((fields.Str(), fields.Str(), fields.Int()))) + network = fields.List( + fields.Tuple((fields.Str(), fields.Str(), fields.Int())) + ) in_migration = fields.Bool() class NetworkSchema(Schema): - _id = fields.Int(data_key='id', attribute='id') - _type = fields.Str(data_key='type', attribute='type') + _id = fields.Int(data_key="id", attribute="id") + _type = fields.Str(data_key="type", attribute="type") ipv6 = fields.Str() diff --git a/ucloud/common/storage_handlers.py b/ucloud/common/storage_handlers.py index d2190ba..b337f23 100644 --- a/ucloud/common/storage_handlers.py +++ b/ucloud/common/storage_handlers.py @@ -11,7 +11,7 @@ from ucloud.settings import settings as config class ImageStorageHandler(ABC): - handler_name = 'base' + handler_name = "base" def __init__(self, image_base, vm_base): self.image_base = image_base @@ -55,9 +55,9 @@ class ImageStorageHandler(ABC): try: sp.check_output(command, stderr=sp.PIPE) except sp.CalledProcessError as e: - _stderr = e.stderr.decode('utf-8').strip() + _stderr = e.stderr.decode("utf-8").strip() if report: - logger.exception('%s:- %s', error_origin, _stderr) + logger.exception("%s:- %s", error_origin, _stderr) return False return True @@ -72,14 +72,16 @@ class ImageStorageHandler(ABC): class FileSystemBasedImageStorageHandler(ImageStorageHandler): - handler_name = 'Filesystem' + handler_name = "Filesystem" def import_image(self, src, dest, protect=False): dest = join_path(self.image_base, dest) try: shutil.copy(src, dest) if protect: - os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + os.chmod( + dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + ) except Exception as e: logger.exception(e) return False @@ -97,7 +99,14 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler): def resize_vm_image(self, path, size): path = join_path(self.vm_base, path) - command = ["qemu-img", "resize", "-f", "raw", path, "{}M".format(size)] + command = [ + "qemu-img", + "resize", + "-f", + "raw", + path, + "{}M".format(size), + ] if self.execute_command(command): return True else: @@ -126,15 +135,25 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler): class CEPHBasedImageStorageHandler(ImageStorageHandler): - handler_name = 'Ceph' + handler_name = "Ceph" def import_image(self, src, dest, protect=False): dest = join_path(self.image_base, dest) import_command = ["rbd", "import", src, dest] commands = [import_command] if protect: - snap_create_command = ["rbd", "snap", "create", "{}@protected".format(dest)] - snap_protect_command = ["rbd", "snap", "protect", "{}@protected".format(dest)] + snap_create_command = [ + "rbd", + "snap", + "create", + "{}@protected".format(dest), + ] + snap_protect_command = [ + "rbd", + "snap", + "protect", + "{}@protected".format(dest), + ] commands.append(snap_create_command) commands.append(snap_protect_command) @@ -174,16 +193,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): def get_storage_handler(): - __storage_backend = config['storage']['storage_backend'] - if __storage_backend == 'filesystem': + __storage_backend = config["storage"]["storage_backend"] + if __storage_backend == "filesystem": return FileSystemBasedImageStorageHandler( - vm_base=config['storage']['vm_dir'], - image_base=config['storage']['image_dir'] + vm_base=config["storage"]["vm_dir"], + image_base=config["storage"]["image_dir"], ) - elif __storage_backend == 'ceph': + elif __storage_backend == "ceph": return CEPHBasedImageStorageHandler( - vm_base=config['storage']['ceph_vm_pool'], - image_base=config['storage']['ceph_image_pool'] + vm_base=config["storage"]["ceph_vm_pool"], + image_base=config["storage"]["ceph_image_pool"], ) else: - raise Exception('Unknown Image Storage Handler') + raise Exception("Unknown Image Storage Handler") diff --git a/ucloud/common/vm.py b/ucloud/common/vm.py index 238f19d..d11046d 100644 --- a/ucloud/common/vm.py +++ b/ucloud/common/vm.py @@ -13,13 +13,12 @@ class VMStatus: def declare_stopped(vm): - vm['hostname'] = '' - vm['in_migration'] = False - vm['status'] = VMStatus.stopped + vm["hostname"] = "" + vm["in_migration"] = False + vm["status"] = VMStatus.stopped class VMEntry(SpecificEtcdEntryBase): - def __init__(self, e): self.owner = None # type: str self.specs = None # type: dict @@ -48,7 +47,9 @@ class VMEntry(SpecificEtcdEntryBase): def add_log(self, msg): self.log = self.log[:5] - self.log.append("{} - {}".format(datetime.now().isoformat(), msg)) + self.log.append( + "{} - {}".format(datetime.now().isoformat(), msg) + ) class VmPool: diff --git a/ucloud/configure/main.py b/ucloud/configure/main.py index e4770d9..31201f6 100644 --- a/ucloud/configure/main.py +++ b/ucloud/configure/main.py @@ -5,31 +5,41 @@ from ucloud.shared import shared def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) + uncloud_config = shared.etcd_client.get( + settings.config_key, value_in_json=True + ) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value - + uncloud_config[section] = kwargs - shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) + shared.etcd_client.put( + settings.config_key, uncloud_config, value_in_json=True + ) def configure_parser(parser): configure_subparsers = parser.add_subparsers(dest="subcommand") - + otp_parser = configure_subparsers.add_parser("otp") - otp_parser.add_argument("--verification-controller-url", - required=True, metavar="URL") - otp_parser.add_argument("--auth-name", required=True, - metavar="OTP-NAME") - otp_parser.add_argument("--auth-realm", required=True, - metavar="OTP-REALM") - otp_parser.add_argument("--auth-seed", required=True, - metavar="OTP-SEED") + otp_parser.add_argument( + "--verification-controller-url", required=True, metavar="URL" + ) + otp_parser.add_argument( + "--auth-name", required=True, metavar="OTP-NAME" + ) + otp_parser.add_argument( + "--auth-realm", required=True, metavar="OTP-REALM" + ) + otp_parser.add_argument( + "--auth-seed", required=True, metavar="OTP-SEED" + ) network_parser = configure_subparsers.add_parser("network") - network_parser.add_argument("--prefix-length", required=True, type=int) + network_parser.add_argument( + "--prefix-length", required=True, type=int + ) network_parser.add_argument("--prefix", required=True) network_parser.add_argument("--vxlan-phy-dev", required=True) @@ -38,25 +48,31 @@ def configure_parser(parser): netbox_parser.add_argument("--token", required=True) ssh_parser = configure_subparsers.add_parser("ssh") - ssh_parser.add_argument('--username', default="root") - ssh_parser.add_argument('--private-key-path', - default=os.path.expanduser("~/.ssh/id_rsa")) + ssh_parser.add_argument("--username", default="root") + ssh_parser.add_argument( + "--private-key-path", + default=os.path.expanduser("~/.ssh/id_rsa"), + ) storage_parser = configure_subparsers.add_parser("storage") - storage_parser.add_argument('--file-dir', required=True) - storage_parser_subparsers = storage_parser.add_subparsers(dest="storage_backend") - - filesystem_storage_parser = storage_parser_subparsers.add_parser("filesystem") - filesystem_storage_parser.add_argument('--vm-dir', required=True) - filesystem_storage_parser.add_argument('--image-dir', required=True) + storage_parser.add_argument("--file-dir", required=True) + storage_parser_subparsers = storage_parser.add_subparsers( + dest="storage_backend" + ) + + filesystem_storage_parser = storage_parser_subparsers.add_parser( + "filesystem" + ) + filesystem_storage_parser.add_argument("--vm-dir", required=True) + filesystem_storage_parser.add_argument("--image-dir", required=True) ceph_storage_parser = storage_parser_subparsers.add_parser("ceph") - ceph_storage_parser.add_argument('--ceph-vm-pool', required=True) - ceph_storage_parser.add_argument('--ceph-image-pool', required=True) + ceph_storage_parser.add_argument("--ceph-vm-pool", required=True) + ceph_storage_parser.add_argument("--ceph-image-pool", required=True) def main(**kwargs): - subcommand = kwargs.pop('subcommand') + subcommand = kwargs.pop("subcommand") if not subcommand: pass else: diff --git a/ucloud/docs/source/conf.py b/ucloud/docs/source/conf.py index 9b133f9..70307f8 100644 --- a/ucloud/docs/source/conf.py +++ b/ucloud/docs/source/conf.py @@ -17,9 +17,9 @@ # -- Project information ----------------------------------------------------- -project = 'ucloud' -copyright = '2019, ungleich' -author = 'ungleich' +project = "ucloud" +copyright = "2019, ungleich" +author = "ungleich" # -- General configuration --------------------------------------------------- @@ -27,12 +27,12 @@ author = 'ungleich' # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx_rtd_theme', + "sphinx.ext.autodoc", + "sphinx_rtd_theme", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -50,4 +50,4 @@ html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] diff --git a/ucloud/filescanner/main.py b/ucloud/filescanner/main.py index 778e942..b12797b 100755 --- a/ucloud/filescanner/main.py +++ b/ucloud/filescanner/main.py @@ -21,7 +21,8 @@ def sha512sum(file: str): ELSE: return None """ - if not isinstance(file, str): raise TypeError + if not isinstance(file, str): + raise TypeError try: output = sp.check_output(["sha512sum", file], stderr=sp.PIPE) except sp.CalledProcessError as e: @@ -49,23 +50,25 @@ def track_file(file, base_dir): file_path = pathlib.Path(file).parts[-1] # Create Entry - entry_key = os.path.join(settings['etcd']['file_prefix'], str(file_id)) + entry_key = os.path.join( + settings["etcd"]["file_prefix"], str(file_id) + ) entry_value = { "filename": file_path, "owner": owner, "sha512sum": sha512sum(file), "creation_date": creation_date, - "size": os.path.getsize(file) + "size": os.path.getsize(file), } logger.info("Tracking %s", file) shared.etcd_client.put(entry_key, entry_value, value_in_json=True) - os.setxattr(file, 'user.utracked', b'True') + os.setxattr(file, "user.utracked", b"True") def main(): - base_dir = settings['storage']['file_dir'] + base_dir = settings["storage"]["file_dir"] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob("{}/**".format(base_dir), recursive=True) @@ -76,7 +79,7 @@ def main(): untracked_files = [] for file in files: try: - os.getxattr(file, 'user.utracked') + os.getxattr(file, "user.utracked") except OSError: track_file(file, base_dir) untracked_files.append(file) diff --git a/ucloud/host/main.py b/ucloud/host/main.py index 904f26c..88dfb7c 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -15,49 +15,79 @@ from . import virtualmachine, logger def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" host_pool = shared.host_pool - this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + this_host = next( + filter(lambda h: h.hostname == hostname, host_pool.hosts), None + ) while True: this_host.update_heartbeat() host_pool.put(this_host) time.sleep(10) -def maintenance(): +def maintenance(host): vmm = VMM() running_vms = vmm.discover() for vm_uuid in running_vms: - if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': - vm = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], vm_uuid)) + if ( + vmm.is_running(vm_uuid) + and vmm.get_status(vm_uuid) == "running" + ): + vm = shared.vm_pool.get( + join_path(settings["etcd"]["vm_prefix"], vm_uuid) + ) vm.status = VMStatus.running + vm.vnc_socket = vmm.get_vnc(vm_uuid) + vm.hostname = host shared.vm_pool.put(vm) def main(hostname): host_pool = shared.host_pool - host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) - assert host is not None, "No such host with name = {}".format(hostname) + host = next( + filter(lambda h: h.hostname == hostname, host_pool.hosts), None + ) + assert host is not None, "No such host with name = {}".format( + hostname + ) try: - heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) + heartbeat_updating_process = mp.Process( + target=update_heartbeat, args=(hostname,) + ) heartbeat_updating_process.start() except Exception as e: - raise Exception('ucloud-host heartbeat updating mechanism is not working') from e + raise Exception( + "ucloud-host heartbeat updating mechanism is not working" + ) from e for events_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], timeout=10, value_in_json=True), + shared.etcd_client.get_prefix( + settings["etcd"]["request_prefix"], value_in_json=True + ), + shared.etcd_client.watch_prefix( + settings["etcd"]["request_prefix"], + timeout=10, + value_in_json=True, + ), ]: for request_event in events_iterator: request_event = RequestEntry(request_event) if request_event.type == "TIMEOUT": - maintenance() + maintenance(host.key) if request_event.hostname == host.key: logger.debug("VM Request: %s", request_event) - shared.request_pool.client.client.delete(request_event.key) - vm_entry = shared.etcd_client.get(join_path(settings['etcd']['vm_prefix'], request_event.uuid)) + shared.request_pool.client.client.delete( + request_event.key + ) + vm_entry = shared.etcd_client.get( + join_path( + settings["etcd"]["vm_prefix"], + request_event.uuid, + ) + ) if vm_entry: vm = virtualmachine.VM(vm_entry) @@ -70,23 +100,35 @@ def main(hostname): elif request_event.type == RequestType.DeleteVM: vm.delete() - elif request_event.type == RequestType.InitVMMigration: + elif ( + request_event.type + == RequestType.InitVMMigration + ): vm.start(destination_host_key=host.key) elif request_event.type == RequestType.TransferVM: - host = host_pool.get(request_event.destination_host_key) + host = host_pool.get( + request_event.destination_host_key + ) if host: - vm.migrate(destination_host=host.hostname, - destination_sock_path=request_event.destination_sock_path) + vm.migrate( + destination_host=host.hostname, + destination_sock_path=request_event.destination_sock_path, + ) else: - logger.error('Host %s not found!', request_event.destination_host_key) + logger.error( + "Host %s not found!", + request_event.destination_host_key, + ) else: logger.info("VM Entry missing") if __name__ == "__main__": argparser = argparse.ArgumentParser() - argparser.add_argument("hostname", help="Name of this host. e.g uncloud1.ungleich.ch") + argparser.add_argument( + "hostname", help="Name of this host. e.g uncloud1.ungleich.ch" + ) args = argparser.parse_args() - mp.set_start_method('spawn') + mp.set_start_method("spawn") main(args.hostname) diff --git a/ucloud/host/qmp/__init__.py b/ucloud/host/qmp/__init__.py index 775b397..40ac3a4 100755 --- a/ucloud/host/qmp/__init__.py +++ b/ucloud/host/qmp/__init__.py @@ -26,10 +26,7 @@ LOG = logging.getLogger(__name__) # Mapping host architecture to any additional architectures it can # support which often includes its 32 bit cousin. -ADDITIONAL_ARCHES = { - "x86_64": "i386", - "aarch64": "armhf" -} +ADDITIONAL_ARCHES = {"x86_64": "i386", "aarch64": "armhf"} def kvm_available(target_arch=None): @@ -81,10 +78,17 @@ class QEMUMachine(object): # vm is guaranteed to be shut down here """ - def __init__(self, binary, args=None, wrapper=None, name=None, - test_dir="/var/tmp", monitor_address=None, - socket_scm_helper=None): - ''' + def __init__( + self, + binary, + args=None, + wrapper=None, + name=None, + test_dir="/var/tmp", + monitor_address=None, + socket_scm_helper=None, + ): + """ Initialize a QEMUMachine @param binary: path to the qemu binary @@ -95,7 +99,7 @@ class QEMUMachine(object): @param monitor_address: address for QMP monitor @param socket_scm_helper: helper program, required for send_fd_scm() @note: Qemu process is not started until launch() is used. - ''' + """ if args is None: args = [] if wrapper is None: @@ -109,7 +113,9 @@ class QEMUMachine(object): self._qemu_log_file = None self._popen = None self._binary = binary - self._args = list(args) # Force copy args in case we modify them + self._args = list( + args + ) # Force copy args in case we modify them self._wrapper = wrapper self._events = [] self._iolog = None @@ -137,26 +143,24 @@ class QEMUMachine(object): # This can be used to add an unused monitor instance. def add_monitor_null(self): - self._args.append('-monitor') - self._args.append('null') + self._args.append("-monitor") + self._args.append("null") - def add_fd(self, fd, fdset, opaque, opts=''): + def add_fd(self, fd, fdset, opaque, opts=""): """ Pass a file descriptor to the VM """ - options = ['fd=%d' % fd, - 'set=%d' % fdset, - 'opaque=%s' % opaque] + options = ["fd=%d" % fd, "set=%d" % fdset, "opaque=%s" % opaque] if opts: options.append(opts) # This did not exist before 3.4, but since then it is # mandatory for our purpose - if hasattr(os, 'set_inheritable'): + if hasattr(os, "set_inheritable"): os.set_inheritable(fd, True) - self._args.append('-add-fd') - self._args.append(','.join(options)) + self._args.append("-add-fd") + self._args.append(",".join(options)) return self # Exactly one of fd and file_path must be given. @@ -168,18 +172,21 @@ class QEMUMachine(object): if self._socket_scm_helper is None: raise QEMUMachineError("No path to socket_scm_helper set") if not os.path.exists(self._socket_scm_helper): - raise QEMUMachineError("%s does not exist" % - self._socket_scm_helper) + raise QEMUMachineError( + "%s does not exist" % self._socket_scm_helper + ) # This did not exist before 3.4, but since then it is # mandatory for our purpose - if hasattr(os, 'set_inheritable'): + if hasattr(os, "set_inheritable"): os.set_inheritable(self._qmp.get_sock_fd(), True) if fd is not None: os.set_inheritable(fd, True) - fd_param = ["%s" % self._socket_scm_helper, - "%d" % self._qmp.get_sock_fd()] + fd_param = [ + "%s" % self._socket_scm_helper, + "%d" % self._qmp.get_sock_fd(), + ] if file_path is not None: assert fd is None @@ -188,9 +195,14 @@ class QEMUMachine(object): assert fd is not None fd_param.append(str(fd)) - devnull = open(os.path.devnull, 'rb') - proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, close_fds=False) + devnull = open(os.path.devnull, "rb") + proc = subprocess.Popen( + fd_param, + stdin=devnull, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=False, + ) output = proc.communicate()[0] if output: LOG.debug(output) @@ -231,24 +243,29 @@ class QEMUMachine(object): if isinstance(self._monitor_address, tuple): moncdev = "socket,id=mon,host=%s,port=%s" % ( self._monitor_address[0], - self._monitor_address[1]) + self._monitor_address[1], + ) else: - moncdev = 'socket,id=mon,path=%s' % self._vm_monitor - args = ['-chardev', moncdev, - '-mon', 'chardev=mon,mode=control'] + moncdev = "socket,id=mon,path=%s" % self._vm_monitor + args = ["-chardev", moncdev, "-mon", "chardev=mon,mode=control"] if self._machine is not None: - args.extend(['-machine', self._machine]) + args.extend(["-machine", self._machine]) if self._console_set: - self._console_address = os.path.join(self._temp_dir, - self._name + "-console.sock") - chardev = ('socket,id=console,path=%s,server,nowait' % - self._console_address) - args.extend(['-chardev', chardev]) + self._console_address = os.path.join( + self._temp_dir, self._name + "-console.sock" + ) + chardev = ( + "socket,id=console,path=%s,server,nowait" + % self._console_address + ) + args.extend(["-chardev", chardev]) if self._console_device_type is None: - args.extend(['-serial', 'chardev:console']) + args.extend(["-serial", "chardev:console"]) else: - device = '%s,chardev=console' % self._console_device_type - args.extend(['-device', device]) + device = ( + "%s,chardev=console" % self._console_device_type + ) + args.extend(["-device", device]) return args def _pre_launch(self): @@ -256,13 +273,17 @@ class QEMUMachine(object): if self._monitor_address is not None: self._vm_monitor = self._monitor_address else: - self._vm_monitor = os.path.join(self._temp_dir, - self._name + "-monitor.sock") - self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log") - self._qemu_log_file = open(self._qemu_log_path, 'wb') + self._vm_monitor = os.path.join( + self._temp_dir, self._name + "-monitor.sock" + ) + self._qemu_log_path = os.path.join( + self._temp_dir, self._name + ".log" + ) + self._qemu_log_file = open(self._qemu_log_path, "wb") - self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor, - server=True) + self._qmp = qmp.QEMUMonitorProtocol( + self._vm_monitor, server=True + ) def _post_launch(self): self._qmp.accept() @@ -289,7 +310,7 @@ class QEMUMachine(object): """ if self._launched: - raise QEMUMachineError('VM already launched') + raise QEMUMachineError("VM already launched") self._iolog = None self._qemu_full_args = None @@ -299,11 +320,11 @@ class QEMUMachine(object): except: self.shutdown() - LOG.debug('Error launching VM') + LOG.debug("Error launching VM") if self._qemu_full_args: - LOG.debug('Command: %r', ' '.join(self._qemu_full_args)) + LOG.debug("Command: %r", " ".join(self._qemu_full_args)) if self._iolog: - LOG.debug('Output: %r', self._iolog) + LOG.debug("Output: %r", self._iolog) raise Exception(self._iolog) raise @@ -311,17 +332,25 @@ class QEMUMachine(object): """ Launch the VM and establish a QMP connection """ - devnull = open(os.path.devnull, 'rb') + devnull = open(os.path.devnull, "rb") self._pre_launch() - self._qemu_full_args = (self._wrapper + [self._binary] + - self._base_args() + self._args) - LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args)) - self._popen = subprocess.Popen(self._qemu_full_args, - stdin=devnull, - stdout=self._qemu_log_file, - stderr=subprocess.STDOUT, - shell=False, - close_fds=False) + self._qemu_full_args = ( + self._wrapper + + [self._binary] + + self._base_args() + + self._args + ) + LOG.debug( + "VM launch command: %r", " ".join(self._qemu_full_args) + ) + self._popen = subprocess.Popen( + self._qemu_full_args, + stdin=devnull, + stdout=self._qemu_log_file, + stderr=subprocess.STDOUT, + shell=False, + close_fds=False, + ) self._post_launch() def wait(self): @@ -339,7 +368,7 @@ class QEMUMachine(object): """ if self.is_running(): try: - self._qmp.cmd('quit') + self._qmp.cmd("quit") self._qmp.close() except: self._popen.kill() @@ -350,11 +379,11 @@ class QEMUMachine(object): exitcode = self.exitcode() if exitcode is not None and exitcode < 0: - msg = 'qemu received signal %i: %s' + msg = "qemu received signal %i: %s" if self._qemu_full_args: - command = ' '.join(self._qemu_full_args) + command = " ".join(self._qemu_full_args) else: - command = '' + command = "" LOG.warn(msg, -exitcode, command) self._launched = False @@ -366,7 +395,7 @@ class QEMUMachine(object): qmp_args = dict() for key, value in args.items(): if conv_keys: - qmp_args[key.replace('_', '-')] = value + qmp_args[key.replace("_", "-")] = value else: qmp_args[key] = value @@ -427,7 +456,9 @@ class QEMUMachine(object): try: for key in match: if key in event: - if not QEMUMachine.event_match(event[key], match[key]): + if not QEMUMachine.event_match( + event[key], match[key] + ): return False else: return False @@ -458,8 +489,9 @@ class QEMUMachine(object): def _match(event): for name, match in events: - if (event['event'] == name and - self.event_match(event, match)): + if event["event"] == name and self.event_match( + event, match + ): return True return False @@ -531,7 +563,8 @@ class QEMUMachine(object): Returns a socket connected to the console """ if self._console_socket is None: - self._console_socket = socket.socket(socket.AF_UNIX, - socket.SOCK_STREAM) + self._console_socket = socket.socket( + socket.AF_UNIX, socket.SOCK_STREAM + ) self._console_socket.connect(self._console_address) return self._console_socket diff --git a/ucloud/host/qmp/qmp.py b/ucloud/host/qmp/qmp.py index bf35d71..ad187eb 100755 --- a/ucloud/host/qmp/qmp.py +++ b/ucloud/host/qmp/qmp.py @@ -32,7 +32,7 @@ class QMPTimeoutError(QMPError): class QEMUMonitorProtocol(object): #: Logger object for debugging messages - logger = logging.getLogger('QMP') + logger = logging.getLogger("QMP") #: Socket's error class error = socket.error #: Socket's timeout @@ -55,7 +55,9 @@ class QEMUMonitorProtocol(object): self.__sock = self.__get_sock() self.__sockfile = None if server: - self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.__sock.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 + ) self.__sock.bind(self.__address) self.__sock.listen(1) @@ -71,7 +73,7 @@ class QEMUMonitorProtocol(object): if greeting is None or "QMP" not in greeting: raise QMPConnectError # Greeting seems ok, negotiate capabilities - resp = self.cmd('qmp_capabilities') + resp = self.cmd("qmp_capabilities") if "return" in resp: return greeting raise QMPCapabilitiesError @@ -82,7 +84,7 @@ class QEMUMonitorProtocol(object): if not data: return resp = json.loads(data) - if 'event' in resp: + if "event" in resp: self.logger.debug("<<< %s", resp) self.__events.append(resp) if not only_event: @@ -165,7 +167,7 @@ class QEMUMonitorProtocol(object): """ self.logger.debug(">>> %s", qmp_cmd) try: - self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8')) + self.__sock.sendall(json.dumps(qmp_cmd).encode("utf-8")) except socket.error as err: if err[0] == errno.EPIPE: return @@ -182,11 +184,11 @@ class QEMUMonitorProtocol(object): @param args: command arguments (dict) @param cmd_id: command id (dict, list, string or int) """ - qmp_cmd = {'execute': name} + qmp_cmd = {"execute": name} if args: - qmp_cmd['arguments'] = args + qmp_cmd["arguments"] = args if cmd_id: - qmp_cmd['id'] = cmd_id + qmp_cmd["id"] = cmd_id return self.cmd_obj(qmp_cmd) def command(self, cmd, **kwds): @@ -195,8 +197,8 @@ class QEMUMonitorProtocol(object): """ ret = self.cmd(cmd, kwds) if "error" in ret: - raise Exception(ret['error']['desc']) - return ret['return'] + raise Exception(ret["error"]["desc"]) + return ret["return"] def pull_event(self, wait=False): """ diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index db0f7b8..d795b3f 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -23,10 +23,6 @@ from ucloud.vmm import VMM from marshmallow import ValidationError -def maintenance(): - pass - - class VM: def __init__(self, vm_entry): self.schema = VMSchema() @@ -35,23 +31,30 @@ class VM: try: self.vm = self.schema.loads(vm_entry.value) except ValidationError: - logger.exception('Couldn\'t validate VM Entry', vm_entry.value) + logger.exception( + "Couldn't validate VM Entry", vm_entry.value + ) self.vm = None else: - self.uuid = vm_entry.key.split('/')[-1] - self.host_key = self.vm['hostname'] + self.uuid = vm_entry.key.split("/")[-1] + self.host_key = self.vm["hostname"] def get_qemu_args(self): command = ( - '-name {owner}_{name}' - ' -drive file={file},format=raw,if=virtio,cache=none' - ' -device virtio-rng-pci' - ' -m {memory} -smp cores={cores},threads={threads}' - ).format(owner=self.vm['owner'], name=self.vm['name'], - memory=int(self.vm['specs']['ram'].to_MB()), cores=self.vm['specs']['cpu'], - threads=1, file=shared.storage_handler.qemu_path_string(self.uuid)) + "-name {owner}_{name}" + " -drive file={file},format=raw,if=virtio,cache=none" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + ).format( + owner=self.vm["owner"], + name=self.vm["name"], + memory=int(self.vm["specs"]["ram"].to_MB()), + cores=self.vm["specs"]["cpu"], + threads=1, + file=shared.storage_handler.qemu_path_string(self.uuid), + ) - return command.split(' ') + return command.split(" ") def start(self, destination_host_key=None): migration = False @@ -63,24 +66,34 @@ class VM: network_args = self.create_network_dev() except Exception as err: declare_stopped(self.vm) - self.vm['log'].append('Cannot Setup Network Properly') - logger.error('Cannot Setup Network Properly for vm %s', self.uuid, exc_info=err) + self.vm["log"].append("Cannot Setup Network Properly") + logger.error( + "Cannot Setup Network Properly for vm %s", + self.uuid, + exc_info=err, + ) else: - self.vmm.start(uuid=self.uuid, migration=migration, - *self.get_qemu_args(), *network_args) + self.vmm.start( + uuid=self.uuid, + migration=migration, + *self.get_qemu_args(), + *network_args + ) status = self.vmm.get_status(self.uuid) - if status == 'running': - self.vm['status'] = VMStatus.running - self.vm['vnc_socket'] = self.vmm.get_vnc(self.uuid) - elif status == 'inmigrate': + if status == "running": + self.vm["status"] = VMStatus.running + self.vm["vnc_socket"] = self.vmm.get_vnc(self.uuid) + elif status == "inmigrate": r = RequestEntry.from_scratch( type=RequestType.TransferVM, # Transfer VM hostname=self.host_key, # Which VM should get this request. It is source host uuid=self.uuid, # uuid of VM - destination_sock_path=join_path(self.vmm.socket_dir, self.uuid), + destination_sock_path=join_path( + self.vmm.socket_dir, self.uuid + ), destination_host_key=destination_host_key, # Where source host transfer VM - request_prefix=settings['etcd']['request_prefix'] + request_prefix=settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) else: @@ -96,15 +109,22 @@ class VM: self.sync() def migrate(self, destination_host, destination_sock_path): - self.vmm.transfer(src_uuid=self.uuid, destination_sock_path=destination_sock_path, - host=destination_host) + self.vmm.transfer( + src_uuid=self.uuid, + destination_sock_path=destination_sock_path, + host=destination_host, + ) def create_network_dev(self): - command = '' - for network_mac_and_tap in self.vm['network']: + command = "" + for network_mac_and_tap in self.vm["network"]: network_name, mac, tap = network_mac_and_tap - _key = os.path.join(settings['etcd']['network_prefix'], self.vm['owner'], network_name) + _key = os.path.join( + settings["etcd"]["network_prefix"], + self.vm["owner"], + network_name, + ) network = shared.etcd_client.get(_key, value_in_json=True) network_schema = NetworkSchema() try: @@ -112,49 +132,64 @@ class VM: except ValidationError: continue - if network['type'] == "vxlan": - tap = create_vxlan_br_tap(_id=network['id'], - _dev=settings['network']['vxlan_phy_dev'], - tap_id=tap, - ip=network['ipv6']) + if network["type"] == "vxlan": + tap = create_vxlan_br_tap( + _id=network["id"], + _dev=settings["network"]["vxlan_phy_dev"], + tap_id=tap, + ip=network["ipv6"], + ) - all_networks = shared.etcd_client.get_prefix(settings['etcd']['network_prefix'], - value_in_json=True) + all_networks = shared.etcd_client.get_prefix( + settings["etcd"]["network_prefix"], + value_in_json=True, + ) - if ipaddress.ip_network(network['ipv6']).is_global: + if ipaddress.ip_network(network["ipv6"]).is_global: update_radvd_conf(all_networks) - command += '-netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no' \ - ' -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}' \ - .format(tap=tap, net_id=network['id'], mac=mac) + command += ( + "-netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}".format( + tap=tap, net_id=network["id"], mac=mac + ) + ) - return command.split(' ') + return command.split(" ") def delete_network_dev(self): try: - for network in self.vm['network']: + for network in self.vm["network"]: network_name = network[0] _ = network[1] # tap_mac tap_id = network[2] - delete_network_interface('tap{}'.format(tap_id)) + delete_network_interface("tap{}".format(tap_id)) - owners_vms = shared.vm_pool.by_owner(self.vm['owner']) - owners_running_vms = shared.vm_pool.by_status(VMStatus.running, - _vms=owners_vms) + owners_vms = shared.vm_pool.by_owner(self.vm["owner"]) + owners_running_vms = shared.vm_pool.by_status( + VMStatus.running, _vms=owners_vms + ) networks = map( - lambda n: n[0], map(lambda vm: vm.network, owners_running_vms) + lambda n: n[0], + map(lambda vm: vm.network, owners_running_vms), ) networks_in_use_by_user_vms = [vm[0] for vm in networks] if network_name not in networks_in_use_by_user_vms: - network_entry = resolve_network(network[0], self.vm['owner']) + network_entry = resolve_network( + network[0], self.vm["owner"] + ) if network_entry: network_type = network_entry.value["type"] network_id = network_entry.value["id"] if network_type == "vxlan": - delete_network_interface('br{}'.format(network_id)) - delete_network_interface('vxlan{}'.format(network_id)) + delete_network_interface( + "br{}".format(network_id) + ) + delete_network_interface( + "vxlan{}".format(network_id) + ) except Exception: logger.exception("Exception in network interface deletion") @@ -163,15 +198,21 @@ class VM: # File Already exists. No Problem Continue logger.debug("Image for vm %s exists", self.uuid) else: - if shared.storage_handler.make_vm_image(src=self.vm['image_uuid'], dest=self.uuid): - if not shared.storage_handler.resize_vm_image(path=self.uuid, - size=int(self.vm['specs']['os-ssd'].to_MB())): - self.vm['status'] = VMStatus.error + if shared.storage_handler.make_vm_image( + src=self.vm["image_uuid"], dest=self.uuid + ): + if not shared.storage_handler.resize_vm_image( + path=self.uuid, + size=int(self.vm["specs"]["os-ssd"].to_MB()), + ): + self.vm["status"] = VMStatus.error else: logger.info("New VM Created") def sync(self): - shared.etcd_client.put(self.key, self.schema.dump(self.vm), value_in_json=True) + shared.etcd_client.put( + self.key, self.schema.dump(self.vm), value_in_json=True + ) def delete(self): self.stop() @@ -186,50 +227,77 @@ class VM: def resolve_network(network_name, network_owner): network = shared.etcd_client.get( - join_path(settings['etcd']['network_prefix'], network_owner, network_name), value_in_json=True + join_path( + settings["etcd"]["network_prefix"], + network_owner, + network_name, + ), + value_in_json=True, ) return network def create_vxlan_br_tap(_id, _dev, tap_id, ip=None): - network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') - vxlan = create_dev(script=os.path.join(network_script_base, 'create-vxlan.sh'), - _id=_id, dev=_dev) + network_script_base = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "network" + ) + vxlan = create_dev( + script=os.path.join(network_script_base, "create-vxlan.sh"), + _id=_id, + dev=_dev, + ) if vxlan: - bridge = create_dev(script=os.path.join(network_script_base, 'create-bridge.sh'), - _id=_id, dev=vxlan, ip=ip) + bridge = create_dev( + script=os.path.join( + network_script_base, "create-bridge.sh" + ), + _id=_id, + dev=vxlan, + ip=ip, + ) if bridge: - tap = create_dev(script=os.path.join(network_script_base, 'create-tap.sh'), - _id=str(tap_id), dev=bridge) + tap = create_dev( + script=os.path.join( + network_script_base, "create-tap.sh" + ), + _id=str(tap_id), + dev=bridge, + ) if tap: return tap def update_radvd_conf(all_networks): - network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') + network_script_base = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "network" + ) networks = { - net.value['ipv6']: net.value['id'] + net.value["ipv6"]: net.value["id"] for net in all_networks - if net.value.get('ipv6') and ipaddress.ip_network(net.value.get('ipv6')).is_global + if net.value.get("ipv6") + and ipaddress.ip_network(net.value.get("ipv6")).is_global } - radvd_template = open(os.path.join(network_script_base, - 'radvd-template.conf'), 'r').read() + radvd_template = open( + os.path.join(network_script_base, "radvd-template.conf"), "r" + ).read() radvd_template = Template(radvd_template) content = [ radvd_template.safe_substitute( - bridge='br{}'.format(networks[net]), - prefix=net + bridge="br{}".format(networks[net]), prefix=net ) - for net in networks if networks.get(net) + for net in networks + if networks.get(net) ] - with open('/etc/radvd.conf', 'w') as radvd_conf: + with open("/etc/radvd.conf", "w") as radvd_conf: radvd_conf.writelines(content) try: - sp.check_output(['systemctl', 'restart', 'radvd']) + sp.check_output(["systemctl", "restart", "radvd"]) except sp.CalledProcessError: try: - sp.check_output(['service', 'radvd', 'restart']) + sp.check_output(["service", "radvd", "restart"]) except sp.CalledProcessError as err: - raise err.__class__('Cannot start/restart radvd service', err.cmd) from err + raise err.__class__( + "Cannot start/restart radvd service", err.cmd + ) from err diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py index e215c88..e1960bc 100755 --- a/ucloud/imagescanner/main.py +++ b/ucloud/imagescanner/main.py @@ -9,7 +9,13 @@ from ucloud.imagescanner import logger def qemu_img_type(path): - qemu_img_info_command = ["qemu-img", "info", "--output", "json", path] + qemu_img_info_command = [ + "qemu-img", + "info", + "--output", + "json", + path, + ] try: qemu_img_info = sp.check_output(qemu_img_info_command) except Exception as e: @@ -22,32 +28,57 @@ def qemu_img_type(path): def main(): # We want to get images entries that requests images to be created - images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True) - images_to_be_created = list(filter(lambda im: im.value['status'] == 'TO_BE_CREATED', images)) + images = shared.etcd_client.get_prefix( + settings["etcd"]["image_prefix"], value_in_json=True + ) + images_to_be_created = list( + filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) + ) for image in images_to_be_created: try: - image_uuid = image.key.split('/')[-1] - image_owner = image.value['owner'] - image_filename = image.value['filename'] - image_store_name = image.value['store_name'] - image_full_path = join_path(settings['storage']['file_dir'], image_owner, image_filename) + image_uuid = image.key.split("/")[-1] + image_owner = image.value["owner"] + image_filename = image.value["filename"] + image_store_name = image.value["store_name"] + image_full_path = join_path( + settings["storage"]["file_dir"], + image_owner, + image_filename, + ) - image_stores = shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix'], - value_in_json=True) - user_image_store = next(filter( - lambda s, store_name=image_store_name: s.value["name"] == store_name, - image_stores - )) + image_stores = shared.etcd_client.get_prefix( + settings["etcd"]["image_store_prefix"], + value_in_json=True, + ) + user_image_store = next( + filter( + lambda s, store_name=image_store_name: s.value[ + "name" + ] + == store_name, + image_stores, + ) + ) - image_store_pool = user_image_store.value['attributes']['pool'] + image_store_pool = user_image_store.value["attributes"][ + "pool" + ] except Exception as e: logger.exception(e) else: # At least our basic data is available - qemu_img_convert_command = ["qemu-img", "convert", "-f", "qcow2", - "-O", "raw", image_full_path, "image.raw"] + qemu_img_convert_command = [ + "qemu-img", + "convert", + "-f", + "qcow2", + "-O", + "raw", + image_full_path, + "image.raw", + ] if qemu_img_type(image_full_path) == "qcow2": try: @@ -55,16 +86,20 @@ def main(): sp.check_output(qemu_img_convert_command,) except sp.CalledProcessError: - logger.exception('Image convertion from .qcow2 to .raw failed.') + logger.exception( + "Image convertion from .qcow2 to .raw failed." + ) else: # Import and Protect - r_status = shared.storage_handler.import_image(src="image.raw", - dest=image_uuid, - protect=True) + r_status = shared.storage_handler.import_image( + src="image.raw", dest=image_uuid, protect=True + ) if r_status: # Everything is successfully done image.value["status"] = "CREATED" - shared.etcd_client.put(image.key, json.dumps(image.value)) + shared.etcd_client.put( + image.key, json.dumps(image.value) + ) finally: try: os.remove("image.raw") @@ -74,7 +109,9 @@ def main(): else: # The user provided image is either not found or of invalid format image.value["status"] = "INVALID_IMAGE" - shared.etcd_client.put(image.key, json.dumps(image.value)) + shared.etcd_client.put( + image.key, json.dumps(image.value) + ) if __name__ == "__main__": diff --git a/ucloud/metadata/main.py b/ucloud/metadata/main.py index adec9e7..2974e33 100644 --- a/ucloud/metadata/main.py +++ b/ucloud/metadata/main.py @@ -21,33 +21,39 @@ def handle_exception(e): return e # now you're handling non-HTTP exceptions only - return {'message': 'Server Error'}, 500 + return {"message": "Server Error"}, 500 def get_vm_entry(mac_addr): - return next(filter(lambda vm: mac_addr in list(zip(*vm.network))[1], shared.vm_pool.vms), None) + return next( + filter( + lambda vm: mac_addr in list(zip(*vm.network))[1], + shared.vm_pool.vms, + ), + None, + ) # https://stackoverflow.com/questions/37140846/how-to-convert-ipv6-link-local-address-to-mac-address-in-python def ipv62mac(ipv6): # remove subnet info if given - subnet_index = ipv6.find('/') + subnet_index = ipv6.find("/") if subnet_index != -1: ipv6 = ipv6[:subnet_index] - ipv6_parts = ipv6.split(':') + ipv6_parts = ipv6.split(":") mac_parts = list() for ipv6_part in ipv6_parts[-4:]: while len(ipv6_part) < 4: - ipv6_part = '0' + ipv6_part + ipv6_part = "0" + ipv6_part mac_parts.append(ipv6_part[:2]) mac_parts.append(ipv6_part[-2:]) # modify parts to match MAC value - mac_parts[0] = '%02x' % (int(mac_parts[0], 16) ^ 2) + mac_parts[0] = "%02x" % (int(mac_parts[0], 16) ^ 2) del mac_parts[4] del mac_parts[3] - return ':'.join(mac_parts) + return ":".join(mac_parts) class Root(Resource): @@ -56,19 +62,27 @@ class Root(Resource): data = get_vm_entry(ipv62mac(request.remote_addr)) if not data: - return {'message': 'Metadata for such VM does not exists.'}, 404 + return ( + {"message": "Metadata for such VM does not exists."}, + 404, + ) else: - etcd_key = os.path.join(settings['etcd']['user_prefix'], - data.value['owner_realm'], - data.value['owner'], 'key') - etcd_entry = shared.etcd_client.get_prefix(etcd_key, value_in_json=True) + etcd_key = os.path.join( + settings["etcd"]["user_prefix"], + data.value["owner_realm"], + data.value["owner"], + "key", + ) + etcd_entry = shared.etcd_client.get_prefix( + etcd_key, value_in_json=True + ) user_personal_ssh_keys = [key.value for key in etcd_entry] - data.value['metadata']['ssh-keys'] += user_personal_ssh_keys - return data.value['metadata'], 200 + data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys + return data.value["metadata"], 200 @staticmethod def post(): - return {'message': 'Previous Implementation is deprecated.'} + return {"message": "Previous Implementation is deprecated."} # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True) # print(data) # if data: @@ -94,12 +108,12 @@ class Root(Resource): # data, value_in_json=True) -api.add_resource(Root, '/') +api.add_resource(Root, "/") def main(): app.run(debug=True, host="::", port="80") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/ucloud/scheduler/__init__.py b/ucloud/scheduler/__init__.py index 95e1be0..eea436a 100644 --- a/ucloud/scheduler/__init__.py +++ b/ucloud/scheduler/__init__.py @@ -1,3 +1,3 @@ import logging -logger = logging.getLogger(__name__) \ No newline at end of file +logger = logging.getLogger(__name__) diff --git a/ucloud/scheduler/helper.py b/ucloud/scheduler/helper.py index 0e9ef73..2fb7a22 100755 --- a/ucloud/scheduler/helper.py +++ b/ucloud/scheduler/helper.py @@ -24,17 +24,35 @@ def remaining_resources(host_specs, vms_specs): for component in _vms_specs: if isinstance(_vms_specs[component], str): - _vms_specs[component] = int(bitmath.parse_string_unsafe(_vms_specs[component]).to_MB()) + _vms_specs[component] = int( + bitmath.parse_string_unsafe( + _vms_specs[component] + ).to_MB() + ) elif isinstance(_vms_specs[component], list): - _vms_specs[component] = map(lambda x: int(bitmath.parse_string_unsafe(x).to_MB()), _vms_specs[component]) - _vms_specs[component] = reduce(lambda x, y: x + y, _vms_specs[component], 0) + _vms_specs[component] = map( + lambda x: int(bitmath.parse_string_unsafe(x).to_MB()), + _vms_specs[component], + ) + _vms_specs[component] = reduce( + lambda x, y: x + y, _vms_specs[component], 0 + ) for component in _remaining: if isinstance(_remaining[component], str): - _remaining[component] = int(bitmath.parse_string_unsafe(_remaining[component]).to_MB()) + _remaining[component] = int( + bitmath.parse_string_unsafe( + _remaining[component] + ).to_MB() + ) elif isinstance(_remaining[component], list): - _remaining[component] = map(lambda x: int(bitmath.parse_string_unsafe(x).to_MB()), _remaining[component]) - _remaining[component] = reduce(lambda x, y: x + y, _remaining[component], 0) + _remaining[component] = map( + lambda x: int(bitmath.parse_string_unsafe(x).to_MB()), + _remaining[component], + ) + _remaining[component] = reduce( + lambda x, y: x + y, _remaining[component], 0 + ) _remaining.subtract(_vms_specs) @@ -59,11 +77,15 @@ def get_suitable_host(vm_specs, hosts=None): running_vms_specs = [vm.specs for vm in vms] # Accumulate all of their combined specs - running_vms_accumulated_specs = accumulated_specs(running_vms_specs) + running_vms_accumulated_specs = accumulated_specs( + running_vms_specs + ) # Find out remaining resources after # host_specs - already running vm_specs - remaining = remaining_resources(host.specs, running_vms_accumulated_specs) + remaining = remaining_resources( + host.specs, running_vms_accumulated_specs + ) # Find out remaining - new_vm_specs remaining = remaining_resources(remaining, vm_specs) @@ -95,7 +117,7 @@ def dead_host_mitigation(dead_hosts_keys): vms_hosted_on_dead_host = shared.vm_pool.by_host(host_key) for vm in vms_hosted_on_dead_host: - vm.status = 'UNKNOWN' + vm.status = "UNKNOWN" shared.vm_pool.put(vm) shared.host_pool.put(host) @@ -104,10 +126,12 @@ def assign_host(vm): vm.hostname = get_suitable_host(vm.specs) shared.vm_pool.put(vm) - r = RequestEntry.from_scratch(type=RequestType.StartVM, - uuid=vm.uuid, - hostname=vm.hostname, - request_prefix=settings['etcd']['request_prefix']) + r = RequestEntry.from_scratch( + type=RequestType.StartVM, + uuid=vm.uuid, + hostname=vm.hostname, + request_prefix=settings["etcd"]["request_prefix"], + ) shared.request_pool.put(r) vm.log.append("VM scheduled for starting") diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index d91979f..7ee75e0 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -7,8 +7,13 @@ from ucloud.common.request import RequestEntry, RequestType from ucloud.shared import shared from ucloud.settings import settings -from .helper import (get_suitable_host, dead_host_mitigation, dead_host_detection, - assign_host, NoSuitableHostFound) +from .helper import ( + get_suitable_host, + dead_host_mitigation, + dead_host_detection, + assign_host, + NoSuitableHostFound, +) from . import logger @@ -16,8 +21,14 @@ def main(): pending_vms = [] for request_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], timeout=5, value_in_json=True), + shared.etcd_client.get_prefix( + settings["etcd"]["request_prefix"], value_in_json=True + ), + shared.etcd_client.watch_prefix( + settings["etcd"]["request_prefix"], + timeout=5, + value_in_json=True, + ), ]: for request_event in request_iterator: request_entry = RequestEntry(request_event) @@ -41,25 +52,39 @@ def main(): # on our behalf. while pending_vms: pending_vm_entry = pending_vms.pop() - r = RequestEntry.from_scratch(type="ScheduleVM", - uuid=pending_vm_entry.uuid, - hostname=pending_vm_entry.hostname, - request_prefix=settings['etcd']['request_prefix']) + r = RequestEntry.from_scratch( + type="ScheduleVM", + uuid=pending_vm_entry.uuid, + hostname=pending_vm_entry.hostname, + request_prefix=settings["etcd"][ + "request_prefix" + ], + ) shared.request_pool.put(r) elif request_entry.type == RequestType.ScheduleVM: - logger.debug("%s, %s", request_entry.key, request_entry.value) + logger.debug( + "%s, %s", request_entry.key, request_entry.value + ) vm_entry = shared.vm_pool.get(request_entry.uuid) if vm_entry is None: - logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) + logger.info( + "Trying to act on {} but it is deleted".format( + request_entry.uuid + ) + ) continue - shared.etcd_client.client.delete(request_entry.key) # consume Request + shared.etcd_client.client.delete( + request_entry.key + ) # consume Request try: assign_host(vm_entry) except NoSuitableHostFound: - vm_entry.add_log("Can't schedule VM. No Resource Left.") + vm_entry.add_log( + "Can't schedule VM. No Resource Left." + ) shared.vm_pool.put(vm_entry) pending_vms.append(vm_entry) diff --git a/ucloud/scheduler/tests/test_basics.py b/ucloud/scheduler/tests/test_basics.py index 92b3a83..68bd8ec 100755 --- a/ucloud/scheduler/tests/test_basics.py +++ b/ucloud/scheduler/tests/test_basics.py @@ -70,9 +70,15 @@ class TestFunctions(unittest.TestCase): "last_heartbeat": datetime.utcnow().isoformat(), } with self.client.client.lock("lock"): - self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True) - self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True) - self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True) + self.client.put( + f"{self.host_prefix}/1", host1, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/2", host2, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/3", host3, value_in_json=True + ) def create_vms(self): vm1 = json.dumps( @@ -146,15 +152,17 @@ class TestFunctions(unittest.TestCase): {"cpu": 8, "ram": 32}, ] self.assertEqual( - accumulated_specs(vms), {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10} + accumulated_specs(vms), + {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10}, ) def test_remaining_resources(self): host_specs = {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10} vms_specs = {"ssd": 10, "cpu": 32, "ram": 12, "hdd": 0} resultant_specs = {"ssd": 0, "cpu": -16, "ram": 36, "hdd": 10} - self.assertEqual(remaining_resources(host_specs, vms_specs), - resultant_specs) + self.assertEqual( + remaining_resources(host_specs, vms_specs), resultant_specs + ) def test_vmpool(self): self.p.join(1) @@ -167,7 +175,12 @@ class TestFunctions(unittest.TestCase): f"{self.vm_prefix}/1", { "owner": "meow", - "specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256}, + "specs": { + "cpu": 4, + "ram": 8, + "hdd": 100, + "sdd": 256, + }, "hostname": f"{self.host_prefix}/3", "status": "SCHEDULED_DEPLOY", }, @@ -182,7 +195,12 @@ class TestFunctions(unittest.TestCase): f"{self.vm_prefix}/7", { "owner": "meow", - "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "specs": { + "cpu": 10, + "ram": 22, + "hdd": 146, + "sdd": 0, + }, "hostname": "", "status": "REQUESTED_NEW", }, @@ -197,7 +215,12 @@ class TestFunctions(unittest.TestCase): f"{self.vm_prefix}/7", { "owner": "meow", - "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "specs": { + "cpu": 10, + "ram": 22, + "hdd": 146, + "sdd": 0, + }, "hostname": "", "status": "REQUESTED_NEW", }, diff --git a/ucloud/scheduler/tests/test_dead_host_mechanism.py b/ucloud/scheduler/tests/test_dead_host_mechanism.py index 0b403ef..466b9ee 100755 --- a/ucloud/scheduler/tests/test_dead_host_mechanism.py +++ b/ucloud/scheduler/tests/test_dead_host_mechanism.py @@ -6,11 +6,7 @@ from os.path import dirname BASE_DIR = dirname(dirname(__file__)) sys.path.insert(0, BASE_DIR) -from main import ( - dead_host_detection, - dead_host_mitigation, - config -) +from main import dead_host_detection, dead_host_mitigation, config class TestDeadHostMechanism(unittest.TestCase): @@ -52,13 +48,23 @@ class TestDeadHostMechanism(unittest.TestCase): "last_heartbeat": datetime(2011, 1, 1).isoformat(), } with self.client.client.lock("lock"): - self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True) - self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True) - self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True) - self.client.put(f"{self.host_prefix}/4", host4, value_in_json=True) + self.client.put( + f"{self.host_prefix}/1", host1, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/2", host2, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/3", host3, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/4", host4, value_in_json=True + ) def test_dead_host_detection(self): - hosts = self.client.get_prefix(self.host_prefix, value_in_json=True) + hosts = self.client.get_prefix( + self.host_prefix, value_in_json=True + ) deads = dead_host_detection(hosts) self.assertEqual(deads, ["/test/host/2", "/test/host/3"]) return deads @@ -66,7 +72,9 @@ class TestDeadHostMechanism(unittest.TestCase): def test_dead_host_mitigation(self): deads = self.test_dead_host_detection() dead_host_mitigation(self.client, deads) - hosts = self.client.get_prefix(self.host_prefix, value_in_json=True) + hosts = self.client.get_prefix( + self.host_prefix, value_in_json=True + ) deads = dead_host_detection(hosts) self.assertEqual(deads, []) diff --git a/ucloud/settings/__init__.py b/ucloud/settings/__init__.py index f9b358e..906e857 100644 --- a/ucloud/settings/__init__.py +++ b/ucloud/settings/__init__.py @@ -14,18 +14,22 @@ class CustomConfigParser(configparser.RawConfigParser): result = super().__getitem__(key) except KeyError as err: raise KeyError( - 'Key \'{}\' not found in configuration. Make sure you configure ucloud.'.format(key) + "Key '{}' not found in configuration. Make sure you configure ucloud.".format( + key + ) ) from err else: return result class Settings(object): - def __init__(self, config_key='/uncloud/config/'): - conf_name = 'ucloud.conf' - conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/ucloud/')) + def __init__(self, config_key="/uncloud/config/"): + conf_name = "ucloud.conf" + conf_dir = os.environ.get( + "UCLOUD_CONF_DIR", os.path.expanduser("~/ucloud/") + ) self.config_file = os.path.join(conf_dir, conf_name) - + self.config_parser = CustomConfigParser(allow_no_value=True) self.config_key = config_key @@ -33,43 +37,55 @@ class Settings(object): try: self.config_parser.read(self.config_file) except Exception as err: - logger.error('%s', err) + logger.error("%s", err) def get_etcd_client(self): args = tuple() try: kwargs = { - 'host': self.config_parser.get('etcd', 'url'), - 'port': self.config_parser.get('etcd', 'port'), - 'ca_cert': self.config_parser.get('etcd', 'ca_cert'), - 'cert_cert': self.config_parser.get('etcd', 'cert_cert'), - 'cert_key': self.config_parser.get('etcd', 'cert_key') + "host": self.config_parser.get("etcd", "url"), + "port": self.config_parser.get("etcd", "port"), + "ca_cert": self.config_parser.get("etcd", "ca_cert"), + "cert_cert": self.config_parser.get( + "etcd", "cert_cert" + ), + "cert_key": self.config_parser.get("etcd", "cert_key"), } except configparser.Error as err: - raise configparser.Error('{} in config file {}'.format(err.message, self.config_file)) from err + raise configparser.Error( + "{} in config file {}".format( + err.message, self.config_file + ) + ) from err else: try: wrapper = Etcd3Wrapper(*args, **kwargs) except Exception as err: - logger.error('etcd connection not successfull. Please check your config file.' - '\nDetails: %s\netcd connection parameters: %s', err, kwargs) + logger.error( + "etcd connection not successfull. Please check your config file." + "\nDetails: %s\netcd connection parameters: %s", + err, + kwargs, + ) sys.exit(1) else: return wrapper - + def read_internal_values(self): - self.config_parser.read_dict({ - 'etcd': { - 'file_prefix': '/files/', - 'host_prefix': '/hosts/', - 'image_prefix': '/images/', - 'image_store_prefix': '/imagestore/', - 'network_prefix': '/networks/', - 'request_prefix': '/requests/', - 'user_prefix': '/users/', - 'vm_prefix': '/vms/', + self.config_parser.read_dict( + { + "etcd": { + "file_prefix": "/files/", + "host_prefix": "/hosts/", + "image_prefix": "/images/", + "image_store_prefix": "/imagestore/", + "network_prefix": "/networks/", + "request_prefix": "/requests/", + "user_prefix": "/users/", + "vm_prefix": "/vms/", + } } - }) + ) def read_config_file_values(self, config_file): try: @@ -77,18 +93,26 @@ class Settings(object): with open(config_file, "r") as config_file_handle: self.config_parser.read_file(config_file_handle) except FileNotFoundError: - sys.exit('Configuration file {} not found!'.format(config_file)) + sys.exit( + "Configuration file {} not found!".format(config_file) + ) except Exception as err: logger.exception(err) sys.exit("Error occurred while reading configuration file") def read_values_from_etcd(self): etcd_client = self.get_etcd_client() - config_from_etcd = etcd_client.get(self.config_key, value_in_json=True) + config_from_etcd = etcd_client.get( + self.config_key, value_in_json=True + ) if config_from_etcd: self.config_parser.read_dict(config_from_etcd.value) else: - raise KeyError("Key '{}' not found in etcd. Please configure ucloud.".format(self.config_key)) + raise KeyError( + "Key '{}' not found in etcd. Please configure ucloud.".format( + self.config_key + ) + ) def __getitem__(self, key): self.read_values_from_etcd() diff --git a/ucloud/shared/__init__.py b/ucloud/shared/__init__.py index 7a296e9..294e34a 100644 --- a/ucloud/shared/__init__.py +++ b/ucloud/shared/__init__.py @@ -12,15 +12,19 @@ class Shared: @property def host_pool(self): - return HostPool(self.etcd_client, settings['etcd']['host_prefix']) + return HostPool( + self.etcd_client, settings["etcd"]["host_prefix"] + ) @property def vm_pool(self): - return VmPool(self.etcd_client, settings['etcd']['vm_prefix']) + return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) @property def request_pool(self): - return RequestPool(self.etcd_client, settings['etcd']['request_prefix']) + return RequestPool( + self.etcd_client, settings["etcd"]["request_prefix"] + ) @property def storage_handler(self): diff --git a/ucloud/vmm/__init__.py b/ucloud/vmm/__init__.py index 9f9f5f9..3d3c304 100644 --- a/ucloud/vmm/__init__.py +++ b/ucloud/vmm/__init__.py @@ -37,7 +37,9 @@ class VMQMPHandles: self.sock.close() if exc_type: - logger.error('Couldn\'t get handle for VM.', exc_type, exc_val, exc_tb) + logger.error( + "Couldn't get handle for VM.", exc_type, exc_val, exc_tb + ) raise exc_type("Couldn't get handle for VM.") from exc_type @@ -54,29 +56,46 @@ class TransferVM(Process): with suppress(FileNotFoundError): os.remove(self.src_sock_path) - command = ['ssh', '-nNT', '-L', '{}:{}'.format(self.src_sock_path, self.dest_sock_path), - 'root@{}'.format(self.host)] + command = [ + "ssh", + "-nNT", + "-L", + "{}:{}".format(self.src_sock_path, self.dest_sock_path), + "root@{}".format(self.host), + ] try: p = sp.Popen(command) except Exception as e: - logger.error('Couldn\' forward unix socks over ssh.', exc_info=e) + logger.error( + "Couldn' forward unix socks over ssh.", exc_info=e + ) else: time.sleep(2) vmm = VMM() - logger.debug('Executing: ssh forwarding command: %s', command) - vmm.execute_command(self.src_uuid, command='migrate', - arguments={'uri': 'unix:{}'.format(self.src_sock_path)}) + logger.debug( + "Executing: ssh forwarding command: %s", command + ) + vmm.execute_command( + self.src_uuid, + command="migrate", + arguments={"uri": "unix:{}".format(self.src_sock_path)}, + ) while p.poll() is None: - success, output = vmm.execute_command(self.src_uuid, command='query-migrate') + success, output = vmm.execute_command( + self.src_uuid, command="query-migrate" + ) if success: - status = output['return']['status'] - if status != 'active': - print('Migration Status: ', status) + status = output["return"]["status"] + + if status != "active": + print("Migration Status: ", status) + if status == "completed": + vmm.stop(self.src_uuid) return else: - print('Migration Status: ', status) + print("Migration Status: ", status) else: return time.sleep(0.2) @@ -84,18 +103,29 @@ class TransferVM(Process): class VMM: # Virtual Machine Manager - def __init__(self, qemu_path='/usr/bin/qemu-system-x86_64', - vmm_backend=os.path.expanduser('~/ucloud/vmm/')): + def __init__( + self, + qemu_path="/usr/bin/qemu-system-x86_64", + vmm_backend=os.path.expanduser("~/ucloud/vmm/"), + ): self.qemu_path = qemu_path self.vmm_backend = vmm_backend - self.socket_dir = os.path.join(self.vmm_backend, 'sock') + self.socket_dir = os.path.join(self.vmm_backend, "sock") if not os.path.isdir(self.vmm_backend): - logger.info('{} does not exists. Creating it...'.format(self.vmm_backend)) + logger.info( + "{} does not exists. Creating it...".format( + self.vmm_backend + ) + ) os.makedirs(self.vmm_backend, exist_ok=True) if not os.path.isdir(self.socket_dir): - logger.info('{} does not exists. Creating it...'.format(self.socket_dir)) + logger.info( + "{} does not exists. Creating it...".format( + self.socket_dir + ) + ) os.makedirs(self.socket_dir, exist_ok=True) def is_running(self, uuid): @@ -106,8 +136,12 @@ class VMM: recv = sock.recv(4096) except Exception as err: # unix sock doesn't exists or it is closed - logger.debug('VM {} sock either don\' exists or it is closed. It mean VM is stopped.'.format(uuid), - exc_info=err) + logger.debug( + "VM {} sock either don' exists or it is closed. It mean VM is stopped.".format( + uuid + ), + exc_info=err, + ) else: # if we receive greetings from qmp it mean VM is running if len(recv) > 0: @@ -122,36 +156,67 @@ class VMM: # start --> sucess? migration_args = () if migration: - migration_args = ('-incoming', 'unix:{}'.format(os.path.join(self.socket_dir, uuid))) + migration_args = ( + "-incoming", + "unix:{}".format(os.path.join(self.socket_dir, uuid)), + ) if self.is_running(uuid): - logger.warning('Cannot start VM. It is already running.') + logger.warning("Cannot start VM. It is already running.") else: - qmp_arg = ('-qmp', 'unix:{},server,nowait'.format(join_path(self.vmm_backend, uuid))) - vnc_arg = ('-vnc', 'unix:{}'.format(tempfile.NamedTemporaryFile().name)) + qmp_arg = ( + "-qmp", + "unix:{},server,nowait".format( + join_path(self.vmm_backend, uuid) + ), + ) + vnc_arg = ( + "-vnc", + "unix:{}".format(tempfile.NamedTemporaryFile().name), + ) - command = ['sudo', '-p', 'Enter password to start VM {}: '.format(uuid), - self.qemu_path, *args, *qmp_arg, *migration_args, *vnc_arg, '-daemonize'] + command = [ + "sudo", + "-p", + "Enter password to start VM {}: ".format(uuid), + self.qemu_path, + *args, + *qmp_arg, + *migration_args, + *vnc_arg, + "-daemonize", + ] try: sp.check_output(command, stderr=sp.PIPE) except sp.CalledProcessError as err: - logger.exception('Error occurred while starting VM.\nDetail %s', err.stderr.decode('utf-8')) + logger.exception( + "Error occurred while starting VM.\nDetail %s", + err.stderr.decode("utf-8"), + ) else: with suppress(sp.CalledProcessError): - sp.check_output([ - 'sudo', '-p', - 'Enter password to correct permission for uncloud-vmm\'s directory', - 'chmod', '-R', 'o=rwx,g=rwx', self.vmm_backend - ]) + sp.check_output( + [ + "sudo", + "-p", + "Enter password to correct permission for uncloud-vmm's directory", + "chmod", + "-R", + "o=rwx,g=rwx", + self.vmm_backend, + ] + ) # TODO: Find some good way to check whether the virtual machine is up and # running without relying on non-guarenteed ways. for _ in range(10): time.sleep(2) status = self.get_status(uuid) - if status in ['running', 'inmigrate']: + if status in ["running", "inmigrate"]: return status - logger.warning('Timeout on VM\'s status. Shutting down VM %s', uuid) + logger.warning( + "Timeout on VM's status. Shutting down VM %s", uuid + ) self.stop(uuid) # TODO: What should we do more. VM can still continue to run in background. # If we have pid of vm we can kill it using OS. @@ -159,55 +224,73 @@ class VMM: def execute_command(self, uuid, command, **kwargs): # execute_command -> sucess?, output try: - with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as (sock_handle, file_handle): - command_to_execute = { - 'execute': command, - **kwargs - } - sock_handle.sendall(json.dumps(command_to_execute).encode('utf-8')) + with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as ( + sock_handle, + file_handle, + ): + command_to_execute = {"execute": command, **kwargs} + sock_handle.sendall( + json.dumps(command_to_execute).encode("utf-8") + ) output = file_handle.readline() except Exception: - logger.exception('Error occurred while executing command and getting valid output from qmp') + logger.exception( + "Error occurred while executing command and getting valid output from qmp" + ) else: try: output = json.loads(output) except Exception: - logger.exception('QMP Output isn\'t valid JSON. %s', output) + logger.exception( + "QMP Output isn't valid JSON. %s", output + ) else: - return 'return' in output, output + return "return" in output, output return False, None def stop(self, uuid): - success, output = self.execute_command(command='quit', uuid=uuid) + success, output = self.execute_command( + command="quit", uuid=uuid + ) return success def get_status(self, uuid): - success, output = self.execute_command(command='query-status', uuid=uuid) + success, output = self.execute_command( + command="query-status", uuid=uuid + ) if success: - return output['return']['status'] + return output["return"]["status"] else: # TODO: Think about this for a little more - return 'STOPPED' + return "STOPPED" def discover(self): vms = [ - uuid for uuid in os.listdir(self.vmm_backend) + uuid + for uuid in os.listdir(self.vmm_backend) if not isdir(join_path(self.vmm_backend, uuid)) ] return vms def get_vnc(self, uuid): - success, output = self.execute_command(uuid, command='query-vnc') + success, output = self.execute_command( + uuid, command="query-vnc" + ) if success: - return output['return']['service'] + return output["return"]["service"] return None def transfer(self, src_uuid, destination_sock_path, host): - p = TransferVM(src_uuid, destination_sock_path, socket_dir=self.socket_dir, host=host) + p = TransferVM( + src_uuid, + destination_sock_path, + socket_dir=self.socket_dir, + host=host, + ) p.start() # TODO: the following method should clean things that went wrong # e.g If VM migration fails or didn't start for long time # i.e 15 minutes we should stop the waiting VM. def maintenace(self): - pass \ No newline at end of file + pass From 52867614df4923fc06bd94a8295636390d67af20 Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 30 Dec 2019 14:58:05 +0500 Subject: [PATCH 051/163] Remove unused code + Increase frequeuncy of host heartbeat update --- ucloud/api/helper.py | 32 -- ucloud/api/main.py | 9 - ucloud/api/schemas.py | 3 - ucloud/common/helpers.py | 40 --- ucloud/common/network.py | 12 - ucloud/host/main.py | 2 +- ucloud/host/qmp/__init__.py | 570 ------------------------------------ ucloud/host/qmp/qmp.py | 257 ---------------- 8 files changed, 1 insertion(+), 924 deletions(-) delete mode 100644 ucloud/common/helpers.py delete mode 100755 ucloud/host/qmp/__init__.py delete mode 100755 ucloud/host/qmp/qmp.py diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index a77a151..6fdeb30 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -132,38 +132,6 @@ def generate_mac( return separator.join(byte_fmt % b for b in mac) -def get_ip_addr(mac_address, device): - """Return IP address of a device provided its mac address / link local address - and the device with which it is connected. - - For Example, if we call get_ip_addr(mac_address="52:54:00:12:34:56", device="br0") - the following two scenarios can happen - 1. It would return None if we can't be able to find device whose mac_address is equal - to the arg:mac_address or the mentioned arg:device does not exists or the ip address - we found is local. - 2. It would return ip_address of device whose mac_address is equal to arg:mac_address - and is connected/neighbor of arg:device - """ - try: - output = sp.check_output( - ["ip", "-6", "neigh", "show", "dev", device], stderr=sp.PIPE - ) - except sp.CalledProcessError: - return None - else: - result = [] - output = output.strip().decode("utf-8") - output = output.split("\n") - for entry in output: - entry = entry.split() - if entry: - ip = ipaddress.ip_address(entry[0]) - mac = entry[2] - if ip.is_global and mac_address == mac: - result.append(ip) - return result - - def mac2ipv6(mac, prefix): # only accept MACs separated by a colon parts = mac.split(":") diff --git a/ucloud/api/main.py b/ucloud/api/main.py index c63babf..d4cdbe9 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -19,15 +19,6 @@ from . import schemas from .helper import generate_mac, mac2ipv6 -def get_parent(obj, attr): - parent = getattr(obj, attr) - child = parent - while parent is not None: - child = parent - parent = getattr(parent, attr) - return child - - logger = logging.getLogger(__name__) app = Flask(__name__) diff --git a/ucloud/api/schemas.py b/ucloud/api/schemas.py index a848a7d..91289b0 100755 --- a/ucloud/api/schemas.py +++ b/ucloud/api/schemas.py @@ -150,7 +150,6 @@ class CreateImageSchema(BaseSchema): class CreateHostSchema(OTPSchema): def __init__(self, data): - self.parsed_specs = {} # Fields self.specs = Field("specs", dict, data.get("specs", KeyError)) self.hostname = Field( @@ -234,8 +233,6 @@ class CreateHostSchema(OTPSchema): class CreateVMSchema(OTPSchema): def __init__(self, data): - self.parsed_specs = {} - # Fields self.specs = Field("specs", dict, data.get("specs", KeyError)) self.vm_name = Field( diff --git a/ucloud/common/helpers.py b/ucloud/common/helpers.py deleted file mode 100644 index 501aa90..0000000 --- a/ucloud/common/helpers.py +++ /dev/null @@ -1,40 +0,0 @@ -import logging -import socket -import requests -import json - -from ipaddress import ip_address - -from os.path import join as join_path -from . import logger - - -# TODO: Should be removed as soon as migration -# mechanism is finalized inside ucloud -def get_ipv4_address(): - # If host is connected to internet - # Return IPv4 address of machine - # Otherwise, return 127.0.0.1 - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: - try: - s.connect(("8.8.8.8", 80)) - except socket.timeout: - address = "127.0.0.1" - except Exception as e: - logger.exception(e) - address = "127.0.0.1" - else: - address = s.getsockname()[0] - - return address - - -def get_ipv6_address(): - try: - r = requests.get("https://api6.ipify.org?format=json") - content = json.loads(r.content.decode("utf-8")) - ip = ip_address(content["ip"]).exploded - except Exception as e: - logger.exception(e) - else: - return ip diff --git a/ucloud/common/network.py b/ucloud/common/network.py index 61dbd64..adba108 100644 --- a/ucloud/common/network.py +++ b/ucloud/common/network.py @@ -70,15 +70,3 @@ def delete_network_interface(iface): except Exception: logger.exception("Interface %s Deletion failed", iface) - -def find_free_port(): - with closing( - socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ) as s: - try: - s.bind(("", 0)) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except Exception: - return None - else: - return s.getsockname()[1] diff --git a/ucloud/host/main.py b/ucloud/host/main.py index 88dfb7c..f25a984 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -21,7 +21,7 @@ def update_heartbeat(hostname): while True: this_host.update_heartbeat() host_pool.put(this_host) - time.sleep(10) + time.sleep(3) def maintenance(host): diff --git a/ucloud/host/qmp/__init__.py b/ucloud/host/qmp/__init__.py deleted file mode 100755 index 40ac3a4..0000000 --- a/ucloud/host/qmp/__init__.py +++ /dev/null @@ -1,570 +0,0 @@ -# QEMU library -# -# Copyright (C) 2015-2016 Red Hat Inc. -# Copyright (C) 2012 IBM Corp. -# -# Authors: -# Fam Zheng -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. -# -# Based on qmp.py. -# - -import errno -import logging -import os -import shutil -import socket -import subprocess -import tempfile - -from . import qmp - -LOG = logging.getLogger(__name__) - -# Mapping host architecture to any additional architectures it can -# support which often includes its 32 bit cousin. -ADDITIONAL_ARCHES = {"x86_64": "i386", "aarch64": "armhf"} - - -def kvm_available(target_arch=None): - host_arch = os.uname()[4] - if target_arch and target_arch != host_arch: - if target_arch != ADDITIONAL_ARCHES.get(host_arch): - return False - return os.access("/dev/kvm", os.R_OK | os.W_OK) - - -class QEMUMachineError(Exception): - """ - Exception called when an error in QEMUMachine happens. - """ - - -class QEMUMachineAddDeviceError(QEMUMachineError): - """ - Exception raised when a request to add a device can not be fulfilled - - The failures are caused by limitations, lack of information or conflicting - requests on the QEMUMachine methods. This exception does not represent - failures reported by the QEMU binary itself. - """ - - -class MonitorResponseError(qmp.QMPError): - """ - Represents erroneous QMP monitor reply - """ - - def __init__(self, reply): - try: - desc = reply["error"]["desc"] - except KeyError: - desc = reply - super(MonitorResponseError, self).__init__(desc) - self.reply = reply - - -class QEMUMachine(object): - """ - A QEMU VM - - Use this object as a context manager to ensure the QEMU process terminates:: - - with VM(binary) as vm: - ... - # vm is guaranteed to be shut down here - """ - - def __init__( - self, - binary, - args=None, - wrapper=None, - name=None, - test_dir="/var/tmp", - monitor_address=None, - socket_scm_helper=None, - ): - """ - Initialize a QEMUMachine - - @param binary: path to the qemu binary - @param args: list of extra arguments - @param wrapper: list of arguments used as prefix to qemu binary - @param name: prefix for socket and log file names (default: qemu-PID) - @param test_dir: where to create socket and log file - @param monitor_address: address for QMP monitor - @param socket_scm_helper: helper program, required for send_fd_scm() - @note: Qemu process is not started until launch() is used. - """ - if args is None: - args = [] - if wrapper is None: - wrapper = [] - if name is None: - name = "qemu-%d" % os.getpid() - self._name = name - self._monitor_address = monitor_address - self._vm_monitor = None - self._qemu_log_path = None - self._qemu_log_file = None - self._popen = None - self._binary = binary - self._args = list( - args - ) # Force copy args in case we modify them - self._wrapper = wrapper - self._events = [] - self._iolog = None - self._socket_scm_helper = socket_scm_helper - self._qmp = None - self._qemu_full_args = None - self._test_dir = test_dir - self._temp_dir = None - self._launched = False - self._machine = None - self._console_set = False - self._console_device_type = None - self._console_address = None - self._console_socket = None - - # just in case logging wasn't configured by the main script: - logging.basicConfig(level=logging.DEBUG) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.shutdown() - return False - - # This can be used to add an unused monitor instance. - def add_monitor_null(self): - self._args.append("-monitor") - self._args.append("null") - - def add_fd(self, fd, fdset, opaque, opts=""): - """ - Pass a file descriptor to the VM - """ - options = ["fd=%d" % fd, "set=%d" % fdset, "opaque=%s" % opaque] - if opts: - options.append(opts) - - # This did not exist before 3.4, but since then it is - # mandatory for our purpose - if hasattr(os, "set_inheritable"): - os.set_inheritable(fd, True) - - self._args.append("-add-fd") - self._args.append(",".join(options)) - return self - - # Exactly one of fd and file_path must be given. - # (If it is file_path, the helper will open that file and pass its - # own fd) - def send_fd_scm(self, fd=None, file_path=None): - # In iotest.py, the qmp should always use unix socket. - assert self._qmp.is_scm_available() - if self._socket_scm_helper is None: - raise QEMUMachineError("No path to socket_scm_helper set") - if not os.path.exists(self._socket_scm_helper): - raise QEMUMachineError( - "%s does not exist" % self._socket_scm_helper - ) - - # This did not exist before 3.4, but since then it is - # mandatory for our purpose - if hasattr(os, "set_inheritable"): - os.set_inheritable(self._qmp.get_sock_fd(), True) - if fd is not None: - os.set_inheritable(fd, True) - - fd_param = [ - "%s" % self._socket_scm_helper, - "%d" % self._qmp.get_sock_fd(), - ] - - if file_path is not None: - assert fd is None - fd_param.append(file_path) - else: - assert fd is not None - fd_param.append(str(fd)) - - devnull = open(os.path.devnull, "rb") - proc = subprocess.Popen( - fd_param, - stdin=devnull, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - close_fds=False, - ) - output = proc.communicate()[0] - if output: - LOG.debug(output) - - return proc.returncode - - @staticmethod - def _remove_if_exists(path): - """ - Remove file object at path if it exists - """ - try: - os.remove(path) - except OSError as exception: - if exception.errno == errno.ENOENT: - return - raise - - def is_running(self): - return self._popen is not None and self._popen.poll() is None - - def exitcode(self): - if self._popen is None: - return None - return self._popen.poll() - - def get_pid(self): - if not self.is_running(): - return None - return self._popen.pid - - def _load_io_log(self): - if self._qemu_log_path is not None: - with open(self._qemu_log_path, "r") as iolog: - self._iolog = iolog.read() - - def _base_args(self): - if isinstance(self._monitor_address, tuple): - moncdev = "socket,id=mon,host=%s,port=%s" % ( - self._monitor_address[0], - self._monitor_address[1], - ) - else: - moncdev = "socket,id=mon,path=%s" % self._vm_monitor - args = ["-chardev", moncdev, "-mon", "chardev=mon,mode=control"] - if self._machine is not None: - args.extend(["-machine", self._machine]) - if self._console_set: - self._console_address = os.path.join( - self._temp_dir, self._name + "-console.sock" - ) - chardev = ( - "socket,id=console,path=%s,server,nowait" - % self._console_address - ) - args.extend(["-chardev", chardev]) - if self._console_device_type is None: - args.extend(["-serial", "chardev:console"]) - else: - device = ( - "%s,chardev=console" % self._console_device_type - ) - args.extend(["-device", device]) - return args - - def _pre_launch(self): - self._temp_dir = tempfile.mkdtemp(dir=self._test_dir) - if self._monitor_address is not None: - self._vm_monitor = self._monitor_address - else: - self._vm_monitor = os.path.join( - self._temp_dir, self._name + "-monitor.sock" - ) - self._qemu_log_path = os.path.join( - self._temp_dir, self._name + ".log" - ) - self._qemu_log_file = open(self._qemu_log_path, "wb") - - self._qmp = qmp.QEMUMonitorProtocol( - self._vm_monitor, server=True - ) - - def _post_launch(self): - self._qmp.accept() - - def _post_shutdown(self): - if self._qemu_log_file is not None: - self._qemu_log_file.close() - self._qemu_log_file = None - - self._qemu_log_path = None - - if self._console_socket is not None: - self._console_socket.close() - self._console_socket = None - - if self._temp_dir is not None: - shutil.rmtree(self._temp_dir) - self._temp_dir = None - - def launch(self): - """ - Launch the VM and make sure we cleanup and expose the - command line/output in case of exception - """ - - if self._launched: - raise QEMUMachineError("VM already launched") - - self._iolog = None - self._qemu_full_args = None - try: - self._launch() - self._launched = True - except: - self.shutdown() - - LOG.debug("Error launching VM") - if self._qemu_full_args: - LOG.debug("Command: %r", " ".join(self._qemu_full_args)) - if self._iolog: - LOG.debug("Output: %r", self._iolog) - raise Exception(self._iolog) - raise - - def _launch(self): - """ - Launch the VM and establish a QMP connection - """ - devnull = open(os.path.devnull, "rb") - self._pre_launch() - self._qemu_full_args = ( - self._wrapper - + [self._binary] - + self._base_args() - + self._args - ) - LOG.debug( - "VM launch command: %r", " ".join(self._qemu_full_args) - ) - self._popen = subprocess.Popen( - self._qemu_full_args, - stdin=devnull, - stdout=self._qemu_log_file, - stderr=subprocess.STDOUT, - shell=False, - close_fds=False, - ) - self._post_launch() - - def wait(self): - """ - Wait for the VM to power off - """ - self._popen.wait() - self._qmp.close() - self._load_io_log() - self._post_shutdown() - - def shutdown(self): - """ - Terminate the VM and clean up - """ - if self.is_running(): - try: - self._qmp.cmd("quit") - self._qmp.close() - except: - self._popen.kill() - self._popen.wait() - - self._load_io_log() - self._post_shutdown() - - exitcode = self.exitcode() - if exitcode is not None and exitcode < 0: - msg = "qemu received signal %i: %s" - if self._qemu_full_args: - command = " ".join(self._qemu_full_args) - else: - command = "" - LOG.warn(msg, -exitcode, command) - - self._launched = False - - def qmp(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command and return the response dict - """ - qmp_args = dict() - for key, value in args.items(): - if conv_keys: - qmp_args[key.replace("_", "-")] = value - else: - qmp_args[key] = value - - return self._qmp.cmd(cmd, args=qmp_args) - - def command(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command. - On success return the response dict. - On failure raise an exception. - """ - reply = self.qmp(cmd, conv_keys, **args) - if reply is None: - raise qmp.QMPError("Monitor is closed") - if "error" in reply: - raise MonitorResponseError(reply) - return reply["return"] - - def get_qmp_event(self, wait=False): - """ - Poll for one queued QMP events and return it - """ - if len(self._events) > 0: - return self._events.pop(0) - return self._qmp.pull_event(wait=wait) - - def get_qmp_events(self, wait=False): - """ - Poll for queued QMP events and return a list of dicts - """ - events = self._qmp.get_events(wait=wait) - events.extend(self._events) - del self._events[:] - self._qmp.clear_events() - return events - - @staticmethod - def event_match(event, match=None): - """ - Check if an event matches optional match criteria. - - The match criteria takes the form of a matching subdict. The event is - checked to be a superset of the subdict, recursively, with matching - values whenever the subdict values are not None. - - This has a limitation that you cannot explicitly check for None values. - - Examples, with the subdict queries on the left: - - None matches any object. - - {"foo": None} matches {"foo": {"bar": 1}} - - {"foo": None} matches {"foo": 5} - - {"foo": {"abc": None}} does not match {"foo": {"bar": 1}} - - {"foo": {"rab": 2}} matches {"foo": {"bar": 1, "rab": 2}} - """ - if match is None: - return True - - try: - for key in match: - if key in event: - if not QEMUMachine.event_match( - event[key], match[key] - ): - return False - else: - return False - return True - except TypeError: - # either match or event wasn't iterable (not a dict) - return match == event - - def event_wait(self, name, timeout=60.0, match=None): - """ - event_wait waits for and returns a named event from QMP with a timeout. - - name: The event to wait for. - timeout: QEMUMonitorProtocol.pull_event timeout parameter. - match: Optional match criteria. See event_match for details. - """ - return self.events_wait([(name, match)], timeout) - - def events_wait(self, events, timeout=60.0): - """ - events_wait waits for and returns a named event from QMP with a timeout. - - events: a sequence of (name, match_criteria) tuples. - The match criteria are optional and may be None. - See event_match for details. - timeout: QEMUMonitorProtocol.pull_event timeout parameter. - """ - - def _match(event): - for name, match in events: - if event["event"] == name and self.event_match( - event, match - ): - return True - return False - - # Search cached events - for event in self._events: - if _match(event): - self._events.remove(event) - return event - - # Poll for new events - while True: - event = self._qmp.pull_event(wait=timeout) - if _match(event): - return event - self._events.append(event) - - return None - - def get_log(self): - """ - After self.shutdown or failed qemu execution, this returns the output - of the qemu process. - """ - return self._iolog - - def add_args(self, *args): - """ - Adds to the list of extra arguments to be given to the QEMU binary - """ - self._args.extend(args) - - def set_machine(self, machine_type): - """ - Sets the machine type - - If set, the machine type will be added to the base arguments - of the resulting QEMU command line. - """ - self._machine = machine_type - - def set_console(self, device_type=None): - """ - Sets the device type for a console device - - If set, the console device and a backing character device will - be added to the base arguments of the resulting QEMU command - line. - - This is a convenience method that will either use the provided - device type, or default to a "-serial chardev:console" command - line argument. - - The actual setting of command line arguments will be be done at - machine launch time, as it depends on the temporary directory - to be created. - - @param device_type: the device type, such as "isa-serial". If - None is given (the default value) a "-serial - chardev:console" command line argument will - be used instead, resorting to the machine's - default device type. - """ - self._console_set = True - self._console_device_type = device_type - - @property - def console_socket(self): - """ - Returns a socket connected to the console - """ - if self._console_socket is None: - self._console_socket = socket.socket( - socket.AF_UNIX, socket.SOCK_STREAM - ) - self._console_socket.connect(self._console_address) - return self._console_socket diff --git a/ucloud/host/qmp/qmp.py b/ucloud/host/qmp/qmp.py deleted file mode 100755 index ad187eb..0000000 --- a/ucloud/host/qmp/qmp.py +++ /dev/null @@ -1,257 +0,0 @@ -# QEMU Monitor Protocol Python class -# -# Copyright (C) 2009, 2010 Red Hat Inc. -# -# Authors: -# Luiz Capitulino -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. - -import errno -import json -import logging -import socket - - -class QMPError(Exception): - pass - - -class QMPConnectError(QMPError): - pass - - -class QMPCapabilitiesError(QMPError): - pass - - -class QMPTimeoutError(QMPError): - pass - - -class QEMUMonitorProtocol(object): - #: Logger object for debugging messages - logger = logging.getLogger("QMP") - #: Socket's error class - error = socket.error - #: Socket's timeout - timeout = socket.timeout - - def __init__(self, address, server=False): - """ - Create a QEMUMonitorProtocol class. - - @param address: QEMU address, can be either a unix socket path (string) - or a tuple in the form ( address, port ) for a TCP - connection - @param server: server mode listens on the socket (bool) - @raise socket.error on socket connection errors - @note No connection is established, this is done by the connect() or - accept() methods - """ - self.__events = [] - self.__address = address - self.__sock = self.__get_sock() - self.__sockfile = None - if server: - self.__sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 - ) - self.__sock.bind(self.__address) - self.__sock.listen(1) - - def __get_sock(self): - if isinstance(self.__address, tuple): - family = socket.AF_INET - else: - family = socket.AF_UNIX - return socket.socket(family, socket.SOCK_STREAM) - - def __negotiate_capabilities(self): - greeting = self.__json_read() - if greeting is None or "QMP" not in greeting: - raise QMPConnectError - # Greeting seems ok, negotiate capabilities - resp = self.cmd("qmp_capabilities") - if "return" in resp: - return greeting - raise QMPCapabilitiesError - - def __json_read(self, only_event=False): - while True: - data = self.__sockfile.readline() - if not data: - return - resp = json.loads(data) - if "event" in resp: - self.logger.debug("<<< %s", resp) - self.__events.append(resp) - if not only_event: - continue - return resp - - def __get_events(self, wait=False): - """ - Check for new events in the stream and cache them in __events. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - """ - - # Check for new events regardless and pull them into the cache: - self.__sock.setblocking(0) - try: - self.__json_read() - except socket.error as err: - if err[0] == errno.EAGAIN: - # No data available - pass - self.__sock.setblocking(1) - - # Wait for new events, if needed. - # if wait is 0.0, this means "no wait" and is also implicitly false. - if not self.__events and wait: - if isinstance(wait, float): - self.__sock.settimeout(wait) - try: - ret = self.__json_read(only_event=True) - except socket.timeout: - raise QMPTimeoutError("Timeout waiting for event") - except: - raise QMPConnectError("Error while reading from socket") - if ret is None: - raise QMPConnectError("Error while reading from socket") - self.__sock.settimeout(None) - - def connect(self, negotiate=True): - """ - Connect to the QMP Monitor and perform capabilities negotiation. - - @return QMP greeting dict - @raise socket.error on socket connection errors - @raise QMPConnectError if the greeting is not received - @raise QMPCapabilitiesError if fails to negotiate capabilities - """ - self.__sock.connect(self.__address) - self.__sockfile = self.__sock.makefile() - if negotiate: - return self.__negotiate_capabilities() - - def accept(self): - """ - Await connection from QMP Monitor and perform capabilities negotiation. - - @return QMP greeting dict - @raise socket.error on socket connection errors - @raise QMPConnectError if the greeting is not received - @raise QMPCapabilitiesError if fails to negotiate capabilities - """ - self.__sock.settimeout(15) - self.__sock, _ = self.__sock.accept() - self.__sockfile = self.__sock.makefile() - return self.__negotiate_capabilities() - - def cmd_obj(self, qmp_cmd): - """ - Send a QMP command to the QMP Monitor. - - @param qmp_cmd: QMP command to be sent as a Python dict - @return QMP response as a Python dict or None if the connection has - been closed - """ - self.logger.debug(">>> %s", qmp_cmd) - try: - self.__sock.sendall(json.dumps(qmp_cmd).encode("utf-8")) - except socket.error as err: - if err[0] == errno.EPIPE: - return - raise socket.error(err) - resp = self.__json_read() - self.logger.debug("<<< %s", resp) - return resp - - def cmd(self, name, args=None, cmd_id=None): - """ - Build a QMP command and send it to the QMP Monitor. - - @param name: command name (string) - @param args: command arguments (dict) - @param cmd_id: command id (dict, list, string or int) - """ - qmp_cmd = {"execute": name} - if args: - qmp_cmd["arguments"] = args - if cmd_id: - qmp_cmd["id"] = cmd_id - return self.cmd_obj(qmp_cmd) - - def command(self, cmd, **kwds): - """ - Build and send a QMP command to the monitor, report errors if any - """ - ret = self.cmd(cmd, kwds) - if "error" in ret: - raise Exception(ret["error"]["desc"]) - return ret["return"] - - def pull_event(self, wait=False): - """ - Pulls a single event. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - - @return The first available QMP event, or None. - """ - self.__get_events(wait) - - if self.__events: - return self.__events.pop(0) - return None - - def get_events(self, wait=False): - """ - Get a list of available QMP events. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - - @return The list of available QMP events. - """ - self.__get_events(wait) - return self.__events - - def clear_events(self): - """ - Clear current list of pending events. - """ - self.__events = [] - - def close(self): - self.__sock.close() - self.__sockfile.close() - - def settimeout(self, timeout): - self.__sock.settimeout(timeout) - - def get_sock_fd(self): - return self.__sock.fileno() - - def is_scm_available(self): - return self.__sock.family == socket.AF_UNIX From 9963e9c62d0028495e2ba03162c56b87ea437acb Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 30 Dec 2019 15:18:25 +0500 Subject: [PATCH 052/163] Slow down heartbeat update --- ucloud/host/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ucloud/host/main.py b/ucloud/host/main.py index f25a984..88dfb7c 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -21,7 +21,7 @@ def update_heartbeat(hostname): while True: this_host.update_heartbeat() host_pool.put(this_host) - time.sleep(3) + time.sleep(10) def maintenance(host): From d2d6c6bf5cb39621ae4f843501a5c01a020e0bd6 Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 30 Dec 2019 15:30:26 +0500 Subject: [PATCH 053/163] Use UTC time for heartbeat mechanism --- ucloud/common/host.py | 4 ++-- ucloud/scheduler/main.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ucloud/common/host.py b/ucloud/common/host.py index 191a2c0..01e2091 100644 --- a/ucloud/common/host.py +++ b/ucloud/common/host.py @@ -26,13 +26,13 @@ class HostEntry(SpecificEtcdEntryBase): def update_heartbeat(self): self.status = HostStatus.alive - self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S") + self.last_heartbeat = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") def is_alive(self): last_heartbeat = datetime.strptime( self.last_heartbeat, "%Y-%m-%d %H:%M:%S" ) - delta = datetime.now() - last_heartbeat + delta = datetime.utcnow() - last_heartbeat if delta.total_seconds() > 60: return False return True diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index 7ee75e0..051b338 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -8,7 +8,6 @@ from ucloud.common.request import RequestEntry, RequestType from ucloud.shared import shared from ucloud.settings import settings from .helper import ( - get_suitable_host, dead_host_mitigation, dead_host_detection, assign_host, From d13a4bcc3778b5032f117857587f2b7857c56096 Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 30 Dec 2019 20:05:12 +0500 Subject: [PATCH 054/163] Remove pending vm handling mechanism from scheduler + fixed issue that update VM's hostname even on migration failure --- scripts/ucloud | 2 +- ucloud/api/main.py | 4 +- ucloud/host/main.py | 93 +++++++++++------------------------ ucloud/host/virtualmachine.py | 14 +++--- ucloud/scheduler/main.py | 21 +------- 5 files changed, 40 insertions(+), 94 deletions(-) diff --git a/scripts/ucloud b/scripts/ucloud index 9d05118..05e47a5 100755 --- a/scripts/ucloud +++ b/scripts/ucloud @@ -24,7 +24,7 @@ sys.excepthook = exception_hook if __name__ == '__main__': # Setting up root logger logger = logging.getLogger() - logger.setLevel(logging.INFO) + logger.setLevel(logging.DEBUG) syslog_handler = SysLogHandler(address='/dev/log') syslog_handler.setLevel(logging.DEBUG) diff --git a/ucloud/api/main.py b/ucloud/api/main.py index d4cdbe9..85133df 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -567,7 +567,7 @@ def main(): settings["etcd"]["image_store_prefix"], value_in_json=True ) ) - if len(image_stores) == 0: + if not image_stores: data = { "is_public": True, "type": "ceph", @@ -583,7 +583,7 @@ def main(): json.dumps(data), ) - app.run(host="::", debug=True) + app.run(host="::", debug=False) if __name__ == "__main__": diff --git a/ucloud/host/main.py b/ucloud/host/main.py index 88dfb7c..ed734b5 100755 --- a/ucloud/host/main.py +++ b/ucloud/host/main.py @@ -28,10 +28,8 @@ def maintenance(host): vmm = VMM() running_vms = vmm.discover() for vm_uuid in running_vms: - if ( - vmm.is_running(vm_uuid) - and vmm.get_status(vm_uuid) == "running" - ): + if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running": + logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( join_path(settings["etcd"]["vm_prefix"], vm_uuid) ) @@ -43,32 +41,18 @@ def maintenance(host): def main(hostname): host_pool = shared.host_pool - host = next( - filter(lambda h: h.hostname == hostname, host_pool.hosts), None - ) - assert host is not None, "No such host with name = {}".format( - hostname - ) + host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + assert host is not None, "No such host with name = {}".format(hostname) try: - heartbeat_updating_process = mp.Process( - target=update_heartbeat, args=(hostname,) - ) + heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() except Exception as e: - raise Exception( - "ucloud-host heartbeat updating mechanism is not working" - ) from e + raise Exception("ucloud-host heartbeat updating mechanism is not working") from e for events_iterator in [ - shared.etcd_client.get_prefix( - settings["etcd"]["request_prefix"], value_in_json=True - ), - shared.etcd_client.watch_prefix( - settings["etcd"]["request_prefix"], - timeout=10, - value_in_json=True, - ), + shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True), + shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True) ]: for request_event in events_iterator: request_event = RequestEntry(request_event) @@ -76,52 +60,35 @@ def main(hostname): if request_event.type == "TIMEOUT": maintenance(host.key) - if request_event.hostname == host.key: - logger.debug("VM Request: %s", request_event) - - shared.request_pool.client.client.delete( - request_event.key - ) + elif request_event.hostname == host.key: + logger.debug("VM Request: %s on Host %s", request_event, host.hostname) + shared.request_pool.client.client.delete(request_event.key) vm_entry = shared.etcd_client.get( - join_path( - settings["etcd"]["vm_prefix"], - request_event.uuid, - ) + join_path(settings["etcd"]["vm_prefix"], request_event.uuid) ) + logger.debug("VM hostname: {}".format(vm_entry.value)) + vm = virtualmachine.VM(vm_entry) + if request_event.type == RequestType.StartVM: + vm.start() - if vm_entry: - vm = virtualmachine.VM(vm_entry) - if request_event.type == RequestType.StartVM: - vm.start() + elif request_event.type == RequestType.StopVM: + vm.stop() - elif request_event.type == RequestType.StopVM: - vm.stop() + elif request_event.type == RequestType.DeleteVM: + vm.delete() - elif request_event.type == RequestType.DeleteVM: - vm.delete() + elif request_event.type == RequestType.InitVMMigration: + vm.start(destination_host_key=host.key) - elif ( - request_event.type - == RequestType.InitVMMigration - ): - vm.start(destination_host_key=host.key) - - elif request_event.type == RequestType.TransferVM: - host = host_pool.get( - request_event.destination_host_key + elif request_event.type == RequestType.TransferVM: + destination_host = host_pool.get(request_event.destination_host_key) + if destination_host: + vm.migrate( + destination_host=destination_host.hostname, + destination_sock_path=request_event.destination_sock_path, ) - if host: - vm.migrate( - destination_host=host.hostname, - destination_sock_path=request_event.destination_sock_path, - ) - else: - logger.error( - "Host %s not found!", - request_event.destination_host_key, - ) - else: - logger.info("VM Entry missing") + else: + logger.error("Host %s not found!", request_event.destination_host_key) if __name__ == "__main__": diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py index d795b3f..8f6c79e 100755 --- a/ucloud/host/virtualmachine.py +++ b/ucloud/host/virtualmachine.py @@ -38,13 +38,14 @@ class VM: else: self.uuid = vm_entry.key.split("/")[-1] self.host_key = self.vm["hostname"] + logger.debug('VM Hostname {}'.format(self.host_key)) def get_qemu_args(self): command = ( - "-name {owner}_{name}" - " -drive file={file},format=raw,if=virtio,cache=none" + "-drive file={file},format=raw,if=virtio,cache=none" " -device virtio-rng-pci" " -m {memory} -smp cores={cores},threads={threads}" + " -name {owner}_{name}" ).format( owner=self.vm["owner"], name=self.vm["name"], @@ -67,11 +68,7 @@ class VM: except Exception as err: declare_stopped(self.vm) self.vm["log"].append("Cannot Setup Network Properly") - logger.error( - "Cannot Setup Network Properly for vm %s", - self.uuid, - exc_info=err, - ) + logger.error("Cannot Setup Network Properly for vm %s", self.uuid, exc_info=err) else: self.vmm.start( uuid=self.uuid, @@ -81,6 +78,7 @@ class VM: ) status = self.vmm.get_status(self.uuid) + logger.debug('VM {} status is {}'.format(self.uuid, status)) if status == "running": self.vm["status"] = VMStatus.running self.vm["vnc_socket"] = self.vmm.get_vnc(self.uuid) @@ -99,7 +97,7 @@ class VM: else: self.stop() declare_stopped(self.vm) - + logger.debug('VM {} has hostname {}'.format(self.uuid, self.vm['hostname'])) self.sync() def stop(self): diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py index 051b338..d64017a 100755 --- a/ucloud/scheduler/main.py +++ b/ucloud/scheduler/main.py @@ -17,8 +17,6 @@ from . import logger def main(): - pending_vms = [] - for request_iterator in [ shared.etcd_client.get_prefix( settings["etcd"]["request_prefix"], value_in_json=True @@ -44,24 +42,8 @@ def main(): logger.debug("Dead hosts: %s", dead_hosts) dead_host_mitigation(dead_hosts) - # If there are VMs that weren't assigned a host - # because there wasn't a host available which - # meets requirement of that VM then we would - # create a new ScheduleVM request for that VM - # on our behalf. - while pending_vms: - pending_vm_entry = pending_vms.pop() - r = RequestEntry.from_scratch( - type="ScheduleVM", - uuid=pending_vm_entry.uuid, - hostname=pending_vm_entry.hostname, - request_prefix=settings["etcd"][ - "request_prefix" - ], - ) - shared.request_pool.put(r) - elif request_entry.type == RequestType.ScheduleVM: + print(request_event.value) logger.debug( "%s, %s", request_entry.key, request_entry.value ) @@ -86,7 +68,6 @@ def main(): ) shared.vm_pool.put(vm_entry) - pending_vms.append(vm_entry) logger.info("No Resource Left. Emailing admin....") From 4b7d6d5099ea7d944a03c4cb0e61b00a3bed3db8 Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 30 Dec 2019 21:14:08 +0500 Subject: [PATCH 055/163] Bug fixed in migration code --- ucloud/vmm/__init__.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/ucloud/vmm/__init__.py b/ucloud/vmm/__init__.py index 3d3c304..d64473b 100644 --- a/ucloud/vmm/__init__.py +++ b/ucloud/vmm/__init__.py @@ -57,6 +57,7 @@ class TransferVM(Process): os.remove(self.src_sock_path) command = [ + "sudo" "ssh", "-nNT", "-L", @@ -73,9 +74,7 @@ class TransferVM(Process): else: time.sleep(2) vmm = VMM() - logger.debug( - "Executing: ssh forwarding command: %s", command - ) + logger.debug("Executing: ssh forwarding command: %s", command) vmm.execute_command( self.src_uuid, command="migrate", @@ -83,22 +82,20 @@ class TransferVM(Process): ) while p.poll() is None: - success, output = vmm.execute_command( - self.src_uuid, command="query-migrate" - ) + success, output = vmm.execute_command(self.src_uuid, command="query-migrate") if success: status = output["return"]["status"] - - if status != "active": - print("Migration Status: ", status) - if status == "completed": - vmm.stop(self.src_uuid) + logger.info('Migration Status: {}'.format(status)) + if status == "completed": + vmm.stop(self.src_uuid) + return + elif status in ['failed', 'cancelled']: return - else: - print("Migration Status: ", status) else: + logger.error("Couldn't be able to query VM {} that was in migration".format(self.src_uuid)) return - time.sleep(0.2) + + time.sleep(2) class VMM: From 27e780b359e1f12f06debae1ecf7c76620d56481 Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 30 Dec 2019 21:30:59 +0500 Subject: [PATCH 056/163] Remove unneccassary sudo from ssh forwarding command --- ucloud/vmm/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ucloud/vmm/__init__.py b/ucloud/vmm/__init__.py index d64473b..4a7fc2f 100644 --- a/ucloud/vmm/__init__.py +++ b/ucloud/vmm/__init__.py @@ -57,7 +57,6 @@ class TransferVM(Process): os.remove(self.src_sock_path) command = [ - "sudo" "ssh", "-nNT", "-L", From 6a40a7f12fe8f900b05c8dab82fb9c71d533e3bc Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 30 Dec 2019 23:22:00 +0500 Subject: [PATCH 057/163] sshtunnel, sphinx, sphinx-rtd-theme, werkzeug removed from dependencies --- setup.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup.py b/setup.py index 956656b..3cf10a0 100644 --- a/setup.py +++ b/setup.py @@ -37,13 +37,9 @@ setup( "flask-restful", "bitmath", "pyotp", - "sshtunnel", - "sphinx", "pynetbox", "colorama", - "sphinx-rtd-theme", "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3", - "werkzeug", "marshmallow", ], scripts=["scripts/ucloud"], From 70c8da544e464c5411a03f8a0f0e8390cdc15c86 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 30 Dec 2019 20:06:15 +0100 Subject: [PATCH 058/163] [refactor] rename scripts to uncloud --- bin/gen-version | 2 +- bin/{ucloud => uncloud} | 2 +- bin/{ucloud-run-reinstall => uncloud-run-reinstall} | 4 ++-- scripts/{ucloud => uncloud} | 0 4 files changed, 4 insertions(+), 4 deletions(-) rename bin/{ucloud => uncloud} (97%) rename bin/{ucloud-run-reinstall => uncloud-run-reinstall} (95%) rename scripts/{ucloud => uncloud} (100%) diff --git a/bin/gen-version b/bin/gen-version index 8f622b8..a2e2882 100755 --- a/bin/gen-version +++ b/bin/gen-version @@ -26,4 +26,4 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../ucloud/version.py +printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py diff --git a/bin/ucloud b/bin/uncloud similarity index 97% rename from bin/ucloud rename to bin/uncloud index ba337fd..1c572d5 100755 --- a/bin/ucloud +++ b/bin/uncloud @@ -30,4 +30,4 @@ ${dir}/gen-version libdir=$(cd "${dir}/../" && pwd -P) export PYTHONPATH="${libdir}" -"$dir/../scripts/ucloud" "$@" +"$dir/../scripts/uncloud" "$@" diff --git a/bin/ucloud-run-reinstall b/bin/uncloud-run-reinstall similarity index 95% rename from bin/ucloud-run-reinstall rename to bin/uncloud-run-reinstall index b189bbc..18e95c0 100755 --- a/bin/ucloud-run-reinstall +++ b/bin/uncloud-run-reinstall @@ -24,6 +24,6 @@ dir=${0%/*} ${dir}/gen-version; -pip uninstall -y ucloud +pip uninstall -y uncloud python setup.py install -${dir}/ucloud "$@" +${dir}/uncloud "$@" diff --git a/scripts/ucloud b/scripts/uncloud similarity index 100% rename from scripts/ucloud rename to scripts/uncloud From 7b6c02b3ab2ade5a7b725e4e5e47e742f592508f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 11:29:08 +0100 Subject: [PATCH 059/163] find ucloud -name \*.py -exec sed -i "s/ucloud/uncloud/g" {} \; --- {ucloud => uncloud}/__init__.py | 0 {ucloud => uncloud}/api/README.md | 0 {ucloud => uncloud}/api/__init__.py | 0 {ucloud => uncloud}/api/common_fields.py | 0 {ucloud => uncloud}/api/create_image_store.py | 0 {ucloud => uncloud}/api/helper.py | 0 {ucloud => uncloud}/api/main.py | 0 {ucloud => uncloud}/api/schemas.py | 0 {ucloud => uncloud}/common/__init__.py | 0 {ucloud => uncloud}/common/classes.py | 0 {ucloud => uncloud}/common/counters.py | 0 {ucloud => uncloud}/common/etcd_wrapper.py | 0 {ucloud => uncloud}/common/host.py | 0 {ucloud => uncloud}/common/logging.py | 0 {ucloud => uncloud}/common/network.py | 0 {ucloud => uncloud}/common/request.py | 0 {ucloud => uncloud}/common/schemas.py | 0 {ucloud => uncloud}/common/storage_handlers.py | 0 {ucloud => uncloud}/common/vm.py | 0 {ucloud => uncloud}/configure/__init__.py | 0 {ucloud => uncloud}/configure/main.py | 0 {ucloud => uncloud}/docs/Makefile | 0 {ucloud => uncloud}/docs/__init__.py | 0 {ucloud => uncloud}/docs/source/__init__.py | 0 {ucloud => uncloud}/docs/source/admin-guide | 0 {ucloud => uncloud}/docs/source/conf.py | 0 {ucloud => uncloud}/docs/source/diagram-code/ucloud | 0 {ucloud => uncloud}/docs/source/hacking.rst | 0 {ucloud => uncloud}/docs/source/images/ucloud.svg | 0 {ucloud => uncloud}/docs/source/index.rst | 0 {ucloud => uncloud}/docs/source/introduction.rst | 0 {ucloud => uncloud}/docs/source/misc/todo.rst | 0 {ucloud => uncloud}/docs/source/setup-install.rst | 0 {ucloud => uncloud}/docs/source/theory/summary.rst | 0 {ucloud => uncloud}/docs/source/troubleshooting.rst | 0 {ucloud => uncloud}/docs/source/user-guide.rst | 0 .../source/user-guide/how-to-create-an-os-image-for-ucloud.rst | 0 {ucloud => uncloud}/filescanner/__init__.py | 0 {ucloud => uncloud}/filescanner/main.py | 0 {ucloud => uncloud}/hack/README.org | 0 {ucloud => uncloud}/hack/conf.d/ucloud-host | 0 {ucloud => uncloud}/hack/nftables.conf | 0 {ucloud => uncloud}/hack/rc-scripts/ucloud-api | 0 {ucloud => uncloud}/hack/rc-scripts/ucloud-host | 0 {ucloud => uncloud}/hack/rc-scripts/ucloud-metadata | 0 {ucloud => uncloud}/hack/rc-scripts/ucloud-scheduler | 0 {ucloud => uncloud}/host/__init__.py | 0 {ucloud => uncloud}/host/main.py | 0 {ucloud => uncloud}/host/virtualmachine.py | 0 {ucloud => uncloud}/imagescanner/__init__.py | 0 {ucloud => uncloud}/imagescanner/main.py | 0 {ucloud => uncloud}/metadata/__init__.py | 0 {ucloud => uncloud}/metadata/main.py | 0 {ucloud => uncloud}/network/README | 0 {ucloud => uncloud}/network/__init__.py | 0 {ucloud => uncloud}/network/create-bridge.sh | 0 {ucloud => uncloud}/network/create-tap.sh | 0 {ucloud => uncloud}/network/create-vxlan.sh | 0 {ucloud => uncloud}/network/radvd-template.conf | 0 {ucloud => uncloud}/scheduler/__init__.py | 0 {ucloud => uncloud}/scheduler/helper.py | 0 {ucloud => uncloud}/scheduler/main.py | 0 {ucloud => uncloud}/scheduler/tests/__init__.py | 0 {ucloud => uncloud}/scheduler/tests/test_basics.py | 0 {ucloud => uncloud}/scheduler/tests/test_dead_host_mechanism.py | 0 {ucloud => uncloud}/settings/__init__.py | 0 {ucloud => uncloud}/shared/__init__.py | 0 {ucloud => uncloud}/vmm/__init__.py | 0 68 files changed, 0 insertions(+), 0 deletions(-) rename {ucloud => uncloud}/__init__.py (100%) rename {ucloud => uncloud}/api/README.md (100%) rename {ucloud => uncloud}/api/__init__.py (100%) rename {ucloud => uncloud}/api/common_fields.py (100%) rename {ucloud => uncloud}/api/create_image_store.py (100%) rename {ucloud => uncloud}/api/helper.py (100%) rename {ucloud => uncloud}/api/main.py (100%) rename {ucloud => uncloud}/api/schemas.py (100%) rename {ucloud => uncloud}/common/__init__.py (100%) rename {ucloud => uncloud}/common/classes.py (100%) rename {ucloud => uncloud}/common/counters.py (100%) rename {ucloud => uncloud}/common/etcd_wrapper.py (100%) rename {ucloud => uncloud}/common/host.py (100%) rename {ucloud => uncloud}/common/logging.py (100%) rename {ucloud => uncloud}/common/network.py (100%) rename {ucloud => uncloud}/common/request.py (100%) rename {ucloud => uncloud}/common/schemas.py (100%) rename {ucloud => uncloud}/common/storage_handlers.py (100%) rename {ucloud => uncloud}/common/vm.py (100%) rename {ucloud => uncloud}/configure/__init__.py (100%) rename {ucloud => uncloud}/configure/main.py (100%) rename {ucloud => uncloud}/docs/Makefile (100%) rename {ucloud => uncloud}/docs/__init__.py (100%) rename {ucloud => uncloud}/docs/source/__init__.py (100%) rename {ucloud => uncloud}/docs/source/admin-guide (100%) rename {ucloud => uncloud}/docs/source/conf.py (100%) rename {ucloud => uncloud}/docs/source/diagram-code/ucloud (100%) rename {ucloud => uncloud}/docs/source/hacking.rst (100%) rename {ucloud => uncloud}/docs/source/images/ucloud.svg (100%) rename {ucloud => uncloud}/docs/source/index.rst (100%) rename {ucloud => uncloud}/docs/source/introduction.rst (100%) rename {ucloud => uncloud}/docs/source/misc/todo.rst (100%) rename {ucloud => uncloud}/docs/source/setup-install.rst (100%) rename {ucloud => uncloud}/docs/source/theory/summary.rst (100%) rename {ucloud => uncloud}/docs/source/troubleshooting.rst (100%) rename {ucloud => uncloud}/docs/source/user-guide.rst (100%) rename {ucloud => uncloud}/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst (100%) rename {ucloud => uncloud}/filescanner/__init__.py (100%) rename {ucloud => uncloud}/filescanner/main.py (100%) rename {ucloud => uncloud}/hack/README.org (100%) rename {ucloud => uncloud}/hack/conf.d/ucloud-host (100%) rename {ucloud => uncloud}/hack/nftables.conf (100%) rename {ucloud => uncloud}/hack/rc-scripts/ucloud-api (100%) rename {ucloud => uncloud}/hack/rc-scripts/ucloud-host (100%) rename {ucloud => uncloud}/hack/rc-scripts/ucloud-metadata (100%) rename {ucloud => uncloud}/hack/rc-scripts/ucloud-scheduler (100%) rename {ucloud => uncloud}/host/__init__.py (100%) rename {ucloud => uncloud}/host/main.py (100%) rename {ucloud => uncloud}/host/virtualmachine.py (100%) rename {ucloud => uncloud}/imagescanner/__init__.py (100%) rename {ucloud => uncloud}/imagescanner/main.py (100%) rename {ucloud => uncloud}/metadata/__init__.py (100%) rename {ucloud => uncloud}/metadata/main.py (100%) rename {ucloud => uncloud}/network/README (100%) rename {ucloud => uncloud}/network/__init__.py (100%) rename {ucloud => uncloud}/network/create-bridge.sh (100%) rename {ucloud => uncloud}/network/create-tap.sh (100%) rename {ucloud => uncloud}/network/create-vxlan.sh (100%) rename {ucloud => uncloud}/network/radvd-template.conf (100%) rename {ucloud => uncloud}/scheduler/__init__.py (100%) rename {ucloud => uncloud}/scheduler/helper.py (100%) rename {ucloud => uncloud}/scheduler/main.py (100%) rename {ucloud => uncloud}/scheduler/tests/__init__.py (100%) rename {ucloud => uncloud}/scheduler/tests/test_basics.py (100%) rename {ucloud => uncloud}/scheduler/tests/test_dead_host_mechanism.py (100%) rename {ucloud => uncloud}/settings/__init__.py (100%) rename {ucloud => uncloud}/shared/__init__.py (100%) rename {ucloud => uncloud}/vmm/__init__.py (100%) diff --git a/ucloud/__init__.py b/uncloud/__init__.py similarity index 100% rename from ucloud/__init__.py rename to uncloud/__init__.py diff --git a/ucloud/api/README.md b/uncloud/api/README.md similarity index 100% rename from ucloud/api/README.md rename to uncloud/api/README.md diff --git a/ucloud/api/__init__.py b/uncloud/api/__init__.py similarity index 100% rename from ucloud/api/__init__.py rename to uncloud/api/__init__.py diff --git a/ucloud/api/common_fields.py b/uncloud/api/common_fields.py similarity index 100% rename from ucloud/api/common_fields.py rename to uncloud/api/common_fields.py diff --git a/ucloud/api/create_image_store.py b/uncloud/api/create_image_store.py similarity index 100% rename from ucloud/api/create_image_store.py rename to uncloud/api/create_image_store.py diff --git a/ucloud/api/helper.py b/uncloud/api/helper.py similarity index 100% rename from ucloud/api/helper.py rename to uncloud/api/helper.py diff --git a/ucloud/api/main.py b/uncloud/api/main.py similarity index 100% rename from ucloud/api/main.py rename to uncloud/api/main.py diff --git a/ucloud/api/schemas.py b/uncloud/api/schemas.py similarity index 100% rename from ucloud/api/schemas.py rename to uncloud/api/schemas.py diff --git a/ucloud/common/__init__.py b/uncloud/common/__init__.py similarity index 100% rename from ucloud/common/__init__.py rename to uncloud/common/__init__.py diff --git a/ucloud/common/classes.py b/uncloud/common/classes.py similarity index 100% rename from ucloud/common/classes.py rename to uncloud/common/classes.py diff --git a/ucloud/common/counters.py b/uncloud/common/counters.py similarity index 100% rename from ucloud/common/counters.py rename to uncloud/common/counters.py diff --git a/ucloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py similarity index 100% rename from ucloud/common/etcd_wrapper.py rename to uncloud/common/etcd_wrapper.py diff --git a/ucloud/common/host.py b/uncloud/common/host.py similarity index 100% rename from ucloud/common/host.py rename to uncloud/common/host.py diff --git a/ucloud/common/logging.py b/uncloud/common/logging.py similarity index 100% rename from ucloud/common/logging.py rename to uncloud/common/logging.py diff --git a/ucloud/common/network.py b/uncloud/common/network.py similarity index 100% rename from ucloud/common/network.py rename to uncloud/common/network.py diff --git a/ucloud/common/request.py b/uncloud/common/request.py similarity index 100% rename from ucloud/common/request.py rename to uncloud/common/request.py diff --git a/ucloud/common/schemas.py b/uncloud/common/schemas.py similarity index 100% rename from ucloud/common/schemas.py rename to uncloud/common/schemas.py diff --git a/ucloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py similarity index 100% rename from ucloud/common/storage_handlers.py rename to uncloud/common/storage_handlers.py diff --git a/ucloud/common/vm.py b/uncloud/common/vm.py similarity index 100% rename from ucloud/common/vm.py rename to uncloud/common/vm.py diff --git a/ucloud/configure/__init__.py b/uncloud/configure/__init__.py similarity index 100% rename from ucloud/configure/__init__.py rename to uncloud/configure/__init__.py diff --git a/ucloud/configure/main.py b/uncloud/configure/main.py similarity index 100% rename from ucloud/configure/main.py rename to uncloud/configure/main.py diff --git a/ucloud/docs/Makefile b/uncloud/docs/Makefile similarity index 100% rename from ucloud/docs/Makefile rename to uncloud/docs/Makefile diff --git a/ucloud/docs/__init__.py b/uncloud/docs/__init__.py similarity index 100% rename from ucloud/docs/__init__.py rename to uncloud/docs/__init__.py diff --git a/ucloud/docs/source/__init__.py b/uncloud/docs/source/__init__.py similarity index 100% rename from ucloud/docs/source/__init__.py rename to uncloud/docs/source/__init__.py diff --git a/ucloud/docs/source/admin-guide b/uncloud/docs/source/admin-guide similarity index 100% rename from ucloud/docs/source/admin-guide rename to uncloud/docs/source/admin-guide diff --git a/ucloud/docs/source/conf.py b/uncloud/docs/source/conf.py similarity index 100% rename from ucloud/docs/source/conf.py rename to uncloud/docs/source/conf.py diff --git a/ucloud/docs/source/diagram-code/ucloud b/uncloud/docs/source/diagram-code/ucloud similarity index 100% rename from ucloud/docs/source/diagram-code/ucloud rename to uncloud/docs/source/diagram-code/ucloud diff --git a/ucloud/docs/source/hacking.rst b/uncloud/docs/source/hacking.rst similarity index 100% rename from ucloud/docs/source/hacking.rst rename to uncloud/docs/source/hacking.rst diff --git a/ucloud/docs/source/images/ucloud.svg b/uncloud/docs/source/images/ucloud.svg similarity index 100% rename from ucloud/docs/source/images/ucloud.svg rename to uncloud/docs/source/images/ucloud.svg diff --git a/ucloud/docs/source/index.rst b/uncloud/docs/source/index.rst similarity index 100% rename from ucloud/docs/source/index.rst rename to uncloud/docs/source/index.rst diff --git a/ucloud/docs/source/introduction.rst b/uncloud/docs/source/introduction.rst similarity index 100% rename from ucloud/docs/source/introduction.rst rename to uncloud/docs/source/introduction.rst diff --git a/ucloud/docs/source/misc/todo.rst b/uncloud/docs/source/misc/todo.rst similarity index 100% rename from ucloud/docs/source/misc/todo.rst rename to uncloud/docs/source/misc/todo.rst diff --git a/ucloud/docs/source/setup-install.rst b/uncloud/docs/source/setup-install.rst similarity index 100% rename from ucloud/docs/source/setup-install.rst rename to uncloud/docs/source/setup-install.rst diff --git a/ucloud/docs/source/theory/summary.rst b/uncloud/docs/source/theory/summary.rst similarity index 100% rename from ucloud/docs/source/theory/summary.rst rename to uncloud/docs/source/theory/summary.rst diff --git a/ucloud/docs/source/troubleshooting.rst b/uncloud/docs/source/troubleshooting.rst similarity index 100% rename from ucloud/docs/source/troubleshooting.rst rename to uncloud/docs/source/troubleshooting.rst diff --git a/ucloud/docs/source/user-guide.rst b/uncloud/docs/source/user-guide.rst similarity index 100% rename from ucloud/docs/source/user-guide.rst rename to uncloud/docs/source/user-guide.rst diff --git a/ucloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst b/uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst similarity index 100% rename from ucloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst rename to uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst diff --git a/ucloud/filescanner/__init__.py b/uncloud/filescanner/__init__.py similarity index 100% rename from ucloud/filescanner/__init__.py rename to uncloud/filescanner/__init__.py diff --git a/ucloud/filescanner/main.py b/uncloud/filescanner/main.py similarity index 100% rename from ucloud/filescanner/main.py rename to uncloud/filescanner/main.py diff --git a/ucloud/hack/README.org b/uncloud/hack/README.org similarity index 100% rename from ucloud/hack/README.org rename to uncloud/hack/README.org diff --git a/ucloud/hack/conf.d/ucloud-host b/uncloud/hack/conf.d/ucloud-host similarity index 100% rename from ucloud/hack/conf.d/ucloud-host rename to uncloud/hack/conf.d/ucloud-host diff --git a/ucloud/hack/nftables.conf b/uncloud/hack/nftables.conf similarity index 100% rename from ucloud/hack/nftables.conf rename to uncloud/hack/nftables.conf diff --git a/ucloud/hack/rc-scripts/ucloud-api b/uncloud/hack/rc-scripts/ucloud-api similarity index 100% rename from ucloud/hack/rc-scripts/ucloud-api rename to uncloud/hack/rc-scripts/ucloud-api diff --git a/ucloud/hack/rc-scripts/ucloud-host b/uncloud/hack/rc-scripts/ucloud-host similarity index 100% rename from ucloud/hack/rc-scripts/ucloud-host rename to uncloud/hack/rc-scripts/ucloud-host diff --git a/ucloud/hack/rc-scripts/ucloud-metadata b/uncloud/hack/rc-scripts/ucloud-metadata similarity index 100% rename from ucloud/hack/rc-scripts/ucloud-metadata rename to uncloud/hack/rc-scripts/ucloud-metadata diff --git a/ucloud/hack/rc-scripts/ucloud-scheduler b/uncloud/hack/rc-scripts/ucloud-scheduler similarity index 100% rename from ucloud/hack/rc-scripts/ucloud-scheduler rename to uncloud/hack/rc-scripts/ucloud-scheduler diff --git a/ucloud/host/__init__.py b/uncloud/host/__init__.py similarity index 100% rename from ucloud/host/__init__.py rename to uncloud/host/__init__.py diff --git a/ucloud/host/main.py b/uncloud/host/main.py similarity index 100% rename from ucloud/host/main.py rename to uncloud/host/main.py diff --git a/ucloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py similarity index 100% rename from ucloud/host/virtualmachine.py rename to uncloud/host/virtualmachine.py diff --git a/ucloud/imagescanner/__init__.py b/uncloud/imagescanner/__init__.py similarity index 100% rename from ucloud/imagescanner/__init__.py rename to uncloud/imagescanner/__init__.py diff --git a/ucloud/imagescanner/main.py b/uncloud/imagescanner/main.py similarity index 100% rename from ucloud/imagescanner/main.py rename to uncloud/imagescanner/main.py diff --git a/ucloud/metadata/__init__.py b/uncloud/metadata/__init__.py similarity index 100% rename from ucloud/metadata/__init__.py rename to uncloud/metadata/__init__.py diff --git a/ucloud/metadata/main.py b/uncloud/metadata/main.py similarity index 100% rename from ucloud/metadata/main.py rename to uncloud/metadata/main.py diff --git a/ucloud/network/README b/uncloud/network/README similarity index 100% rename from ucloud/network/README rename to uncloud/network/README diff --git a/ucloud/network/__init__.py b/uncloud/network/__init__.py similarity index 100% rename from ucloud/network/__init__.py rename to uncloud/network/__init__.py diff --git a/ucloud/network/create-bridge.sh b/uncloud/network/create-bridge.sh similarity index 100% rename from ucloud/network/create-bridge.sh rename to uncloud/network/create-bridge.sh diff --git a/ucloud/network/create-tap.sh b/uncloud/network/create-tap.sh similarity index 100% rename from ucloud/network/create-tap.sh rename to uncloud/network/create-tap.sh diff --git a/ucloud/network/create-vxlan.sh b/uncloud/network/create-vxlan.sh similarity index 100% rename from ucloud/network/create-vxlan.sh rename to uncloud/network/create-vxlan.sh diff --git a/ucloud/network/radvd-template.conf b/uncloud/network/radvd-template.conf similarity index 100% rename from ucloud/network/radvd-template.conf rename to uncloud/network/radvd-template.conf diff --git a/ucloud/scheduler/__init__.py b/uncloud/scheduler/__init__.py similarity index 100% rename from ucloud/scheduler/__init__.py rename to uncloud/scheduler/__init__.py diff --git a/ucloud/scheduler/helper.py b/uncloud/scheduler/helper.py similarity index 100% rename from ucloud/scheduler/helper.py rename to uncloud/scheduler/helper.py diff --git a/ucloud/scheduler/main.py b/uncloud/scheduler/main.py similarity index 100% rename from ucloud/scheduler/main.py rename to uncloud/scheduler/main.py diff --git a/ucloud/scheduler/tests/__init__.py b/uncloud/scheduler/tests/__init__.py similarity index 100% rename from ucloud/scheduler/tests/__init__.py rename to uncloud/scheduler/tests/__init__.py diff --git a/ucloud/scheduler/tests/test_basics.py b/uncloud/scheduler/tests/test_basics.py similarity index 100% rename from ucloud/scheduler/tests/test_basics.py rename to uncloud/scheduler/tests/test_basics.py diff --git a/ucloud/scheduler/tests/test_dead_host_mechanism.py b/uncloud/scheduler/tests/test_dead_host_mechanism.py similarity index 100% rename from ucloud/scheduler/tests/test_dead_host_mechanism.py rename to uncloud/scheduler/tests/test_dead_host_mechanism.py diff --git a/ucloud/settings/__init__.py b/uncloud/settings/__init__.py similarity index 100% rename from ucloud/settings/__init__.py rename to uncloud/settings/__init__.py diff --git a/ucloud/shared/__init__.py b/uncloud/shared/__init__.py similarity index 100% rename from ucloud/shared/__init__.py rename to uncloud/shared/__init__.py diff --git a/ucloud/vmm/__init__.py b/uncloud/vmm/__init__.py similarity index 100% rename from ucloud/vmm/__init__.py rename to uncloud/vmm/__init__.py From 433a3b9817842113e5d0d5e027ab7517283b12e8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 11:30:02 +0100 Subject: [PATCH 060/163] refactor #2 Signed-off-by: Nico Schottelius --- setup.py | 6 +++--- uncloud/api/common_fields.py | 4 ++-- uncloud/api/create_image_store.py | 4 ++-- uncloud/api/helper.py | 4 ++-- uncloud/api/main.py | 10 +++++----- uncloud/api/schemas.py | 10 +++++----- uncloud/common/host.py | 2 +- uncloud/common/storage_handlers.py | 2 +- uncloud/configure/main.py | 4 ++-- uncloud/docs/source/conf.py | 2 +- uncloud/filescanner/main.py | 4 ++-- uncloud/host/main.py | 12 ++++++------ uncloud/host/virtualmachine.py | 16 ++++++++-------- uncloud/imagescanner/main.py | 6 +++--- uncloud/metadata/main.py | 4 ++-- uncloud/scheduler/helper.py | 10 +++++----- uncloud/scheduler/main.py | 6 +++--- uncloud/scheduler/tests/test_basics.py | 2 +- uncloud/settings/__init__.py | 10 +++++----- uncloud/shared/__init__.py | 10 +++++----- uncloud/vmm/__init__.py | 2 +- 21 files changed, 65 insertions(+), 65 deletions(-) diff --git a/setup.py b/setup.py index 3cf10a0..b204f93 100644 --- a/setup.py +++ b/setup.py @@ -17,10 +17,10 @@ except: setup( - name="ucloud", + name="uncloud", version=version, - description="All ucloud server components.", - url="https://code.ungleich.ch/ucloud/ucloud", + description="uncloud cloud management", + url="https://code.ungleich.ch/uncloud/uncloud", long_description=long_description, long_description_content_type="text/markdown", classifiers=[ diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py index 93f9e06..8bcf777 100755 --- a/uncloud/api/common_fields.py +++ b/uncloud/api/common_fields.py @@ -1,7 +1,7 @@ import os -from ucloud.shared import shared -from ucloud.settings import settings +from uncloud.shared import shared +from uncloud.settings import settings class Optional: diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index a433ce3..73b92f1 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -3,8 +3,8 @@ import os from uuid import uuid4 -from ucloud.shared import shared -from ucloud.settings import settings +from uncloud.shared import shared +from uncloud.settings import settings data = { "is_public": True, diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index 6fdeb30..c806814 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -7,8 +7,8 @@ import requests from pyotp import TOTP -from ucloud.shared import shared -from ucloud.settings import settings +from uncloud.shared import shared +from uncloud.settings import settings logger = logging.getLogger(__name__) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 85133df..6ac5d44 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -9,11 +9,11 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from ucloud.common import counters -from ucloud.common.vm import VMStatus -from ucloud.common.request import RequestEntry, RequestType -from ucloud.settings import settings -from ucloud.shared import shared +from uncloud.common import counters +from uncloud.common.vm import VMStatus +from uncloud.common.request import RequestEntry, RequestType +from uncloud.settings import settings +from uncloud.shared import shared from . import schemas from .helper import generate_mac, mac2ipv6 diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index 91289b0..65055c4 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -1,6 +1,6 @@ """ This module contain classes thats validates and intercept/modify -data coming from ucloud-cli (user) +data coming from uncloud-cli (user) It was primarily developed as an alternative to argument parser of Flask_Restful which is going to be deprecated. I also tried @@ -19,10 +19,10 @@ import os import bitmath -from ucloud.common.host import HostStatus -from ucloud.common.vm import VMStatus -from ucloud.shared import shared -from ucloud.settings import settings +from uncloud.common.host import HostStatus +from uncloud.common.vm import VMStatus +from uncloud.shared import shared +from uncloud.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name diff --git a/uncloud/common/host.py b/uncloud/common/host.py index 01e2091..f7bb7d5 100644 --- a/uncloud/common/host.py +++ b/uncloud/common/host.py @@ -7,7 +7,7 @@ from .classes import SpecificEtcdEntryBase class HostStatus: - """Possible Statuses of ucloud host.""" + """Possible Statuses of uncloud host.""" alive = "ALIVE" dead = "DEAD" diff --git a/uncloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py index b337f23..06751c4 100644 --- a/uncloud/common/storage_handlers.py +++ b/uncloud/common/storage_handlers.py @@ -7,7 +7,7 @@ from abc import ABC from . import logger from os.path import join as join_path -from ucloud.settings import settings as config +from uncloud.settings import settings as config class ImageStorageHandler(ABC): diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index 31201f6..a9b4901 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,7 +1,7 @@ import os -from ucloud.settings import settings -from ucloud.shared import shared +from uncloud.settings import settings +from uncloud.shared import shared def update_config(section, kwargs): diff --git a/uncloud/docs/source/conf.py b/uncloud/docs/source/conf.py index 70307f8..c8138a7 100644 --- a/uncloud/docs/source/conf.py +++ b/uncloud/docs/source/conf.py @@ -17,7 +17,7 @@ # -- Project information ----------------------------------------------------- -project = "ucloud" +project = "uncloud" copyright = "2019, ungleich" author = "ungleich" diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index b12797b..7ce8654 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -7,8 +7,8 @@ import time from uuid import uuid4 from . import logger -from ucloud.settings import settings -from ucloud.shared import shared +from uncloud.settings import settings +from uncloud.shared import shared def sha512sum(file: str): diff --git a/uncloud/host/main.py b/uncloud/host/main.py index ed734b5..80527c9 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -2,11 +2,11 @@ import argparse import multiprocessing as mp import time -from ucloud.common.request import RequestEntry, RequestType -from ucloud.shared import shared -from ucloud.settings import settings -from ucloud.common.vm import VMStatus -from ucloud.vmm import VMM +from uncloud.common.request import RequestEntry, RequestType +from uncloud.shared import shared +from uncloud.settings import settings +from uncloud.common.vm import VMStatus +from uncloud.vmm import VMM from os.path import join as join_path from . import virtualmachine, logger @@ -48,7 +48,7 @@ def main(hostname): heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() except Exception as e: - raise Exception("ucloud-host heartbeat updating mechanism is not working") from e + raise Exception("uncloud-host heartbeat updating mechanism is not working") from e for events_iterator in [ shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True), diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index 8f6c79e..0bd20bf 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -11,14 +11,14 @@ import ipaddress from string import Template from os.path import join as join_path -from ucloud.common.request import RequestEntry, RequestType -from ucloud.common.vm import VMStatus, declare_stopped -from ucloud.common.network import create_dev, delete_network_interface -from ucloud.common.schemas import VMSchema, NetworkSchema -from ucloud.host import logger -from ucloud.shared import shared -from ucloud.settings import settings -from ucloud.vmm import VMM +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.vm import VMStatus, declare_stopped +from uncloud.common.network import create_dev, delete_network_interface +from uncloud.common.schemas import VMSchema, NetworkSchema +from uncloud.host import logger +from uncloud.shared import shared +from uncloud.settings import settings +from uncloud.vmm import VMM from marshmallow import ValidationError diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index e1960bc..93e4dd5 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -3,9 +3,9 @@ import os import subprocess as sp from os.path import join as join_path -from ucloud.settings import settings -from ucloud.shared import shared -from ucloud.imagescanner import logger +from uncloud.settings import settings +from uncloud.shared import shared +from uncloud.imagescanner import logger def qemu_img_type(path): diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index 2974e33..da993ae 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -4,8 +4,8 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from ucloud.settings import settings -from ucloud.shared import shared +from uncloud.settings import settings +from uncloud.shared import shared app = Flask(__name__) api = Api(app) diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index 2fb7a22..7edf623 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -3,11 +3,11 @@ from functools import reduce import bitmath -from ucloud.common.host import HostStatus -from ucloud.common.request import RequestEntry, RequestType -from ucloud.common.vm import VMStatus -from ucloud.shared import shared -from ucloud.settings import settings +from uncloud.common.host import HostStatus +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.vm import VMStatus +from uncloud.shared import shared +from uncloud.settings import settings def accumulated_specs(vms_specs): diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index d64017a..5a4014f 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -4,9 +4,9 @@ # 2. Introduce a status endpoint of the scheduler - # maybe expose a prometheus compatible output -from ucloud.common.request import RequestEntry, RequestType -from ucloud.shared import shared -from ucloud.settings import settings +from uncloud.common.request import RequestEntry, RequestType +from uncloud.shared import shared +from uncloud.settings import settings from .helper import ( dead_host_mitigation, dead_host_detection, diff --git a/uncloud/scheduler/tests/test_basics.py b/uncloud/scheduler/tests/test_basics.py index 68bd8ec..defeb23 100755 --- a/uncloud/scheduler/tests/test_basics.py +++ b/uncloud/scheduler/tests/test_basics.py @@ -15,7 +15,7 @@ from main import ( main, ) -from ucloud.config import etcd_client +from uncloud.config import etcd_client class TestFunctions(unittest.TestCase): diff --git a/uncloud/settings/__init__.py b/uncloud/settings/__init__.py index 906e857..90b938c 100644 --- a/uncloud/settings/__init__.py +++ b/uncloud/settings/__init__.py @@ -3,7 +3,7 @@ import logging import sys import os -from ucloud.common.etcd_wrapper import Etcd3Wrapper +from uncloud.common.etcd_wrapper import Etcd3Wrapper logger = logging.getLogger(__name__) @@ -14,7 +14,7 @@ class CustomConfigParser(configparser.RawConfigParser): result = super().__getitem__(key) except KeyError as err: raise KeyError( - "Key '{}' not found in configuration. Make sure you configure ucloud.".format( + "Key '{}' not found in configuration. Make sure you configure uncloud.".format( key ) ) from err @@ -24,9 +24,9 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): def __init__(self, config_key="/uncloud/config/"): - conf_name = "ucloud.conf" + conf_name = "uncloud.conf" conf_dir = os.environ.get( - "UCLOUD_CONF_DIR", os.path.expanduser("~/ucloud/") + "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/") ) self.config_file = os.path.join(conf_dir, conf_name) @@ -109,7 +109,7 @@ class Settings(object): self.config_parser.read_dict(config_from_etcd.value) else: raise KeyError( - "Key '{}' not found in etcd. Please configure ucloud.".format( + "Key '{}' not found in etcd. Please configure uncloud.".format( self.config_key ) ) diff --git a/uncloud/shared/__init__.py b/uncloud/shared/__init__.py index 294e34a..db2093f 100644 --- a/uncloud/shared/__init__.py +++ b/uncloud/shared/__init__.py @@ -1,8 +1,8 @@ -from ucloud.settings import settings -from ucloud.common.vm import VmPool -from ucloud.common.host import HostPool -from ucloud.common.request import RequestPool -from ucloud.common.storage_handlers import get_storage_handler +from uncloud.settings import settings +from uncloud.common.vm import VmPool +from uncloud.common.host import HostPool +from uncloud.common.request import RequestPool +from uncloud.common.storage_handlers import get_storage_handler class Shared: diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 4a7fc2f..6cdd938 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -102,7 +102,7 @@ class VMM: def __init__( self, qemu_path="/usr/bin/qemu-system-x86_64", - vmm_backend=os.path.expanduser("~/ucloud/vmm/"), + vmm_backend=os.path.expanduser("~/uncloud/vmm/"), ): self.qemu_path = qemu_path self.vmm_backend = vmm_backend From 6682f127f10b9c3ae1976af1e7aaf79dc6a3ad5f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 11:35:51 +0100 Subject: [PATCH 061/163] Remove colors, remove sophisticated logging Go back to simple --- scripts/uncloud | 16 ++++------------ uncloud/common/logging.py | 27 --------------------------- 2 files changed, 4 insertions(+), 39 deletions(-) delete mode 100644 uncloud/common/logging.py diff --git a/scripts/uncloud b/scripts/uncloud index 05e47a5..e5c4081 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -6,9 +6,7 @@ import multiprocessing as mp import sys from logging.handlers import SysLogHandler - -from ucloud.common.logging import NoTracebackStreamHandler -from ucloud.configure.main import configure_parser +from uncloud.configure.main import configure_parser def exception_hook(exc_type, exc_value, exc_traceback): @@ -31,19 +29,13 @@ if __name__ == '__main__': syslog_formatter = logging.Formatter('%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s') syslog_handler.setFormatter(syslog_formatter) - stream_handler = NoTracebackStreamHandler() - stream_handler.setLevel(logging.INFO) - stream_formatter = logging.Formatter('%(message)s') - stream_handler.setFormatter(stream_formatter) - logger.addHandler(syslog_handler) - logger.addHandler(stream_handler) arg_parser = argparse.ArgumentParser() subparsers = arg_parser.add_subparsers(dest="command") - + api_parser = subparsers.add_parser("api") - + host_parser = subparsers.add_parser("host") host_parser.add_argument("--hostname", required=True) @@ -72,7 +64,7 @@ if __name__ == '__main__': arguments = vars(args) try: name = arguments.pop('command') - mod = importlib.import_module("ucloud.{}.main".format(name)) + mod = importlib.import_module("uncloud.{}.main".format(name)) main = getattr(mod, "main") main(**arguments) except Exception as err: diff --git a/uncloud/common/logging.py b/uncloud/common/logging.py deleted file mode 100644 index 9e0d2be..0000000 --- a/uncloud/common/logging.py +++ /dev/null @@ -1,27 +0,0 @@ -import logging -import colorama - - -class NoTracebackStreamHandler(logging.StreamHandler): - def handle(self, record): - info, cache = record.exc_info, record.exc_text - record.exc_info, record.exc_text = None, None - - if record.levelname in ["WARNING", "WARN"]: - color = colorama.Fore.LIGHTYELLOW_EX - elif record.levelname == "ERROR": - color = colorama.Fore.LIGHTRED_EX - elif record.levelname == "INFO": - color = colorama.Fore.LIGHTGREEN_EX - elif record.levelname == "CRITICAL": - color = colorama.Fore.LIGHTCYAN_EX - else: - color = colorama.Fore.WHITE - - try: - print(color, end="", flush=True) - super().handle(record) - finally: - record.exc_info = info - record.exc_text = cache - print(colorama.Style.RESET_ALL, end="", flush=True) From 4c7678618dbf7480bad92eed86a49cd0cf554f7d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 11:37:52 +0100 Subject: [PATCH 062/163] Also fix setup.py and the configuration file --- conf/{ucloud.conf => uncloud.conf} | 0 setup.py | 8 ++++---- 2 files changed, 4 insertions(+), 4 deletions(-) rename conf/{ucloud.conf => uncloud.conf} (100%) diff --git a/conf/ucloud.conf b/conf/uncloud.conf similarity index 100% rename from conf/ucloud.conf rename to conf/uncloud.conf diff --git a/setup.py b/setup.py index b204f93..0764d74 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,9 @@ with open("README.md", "r") as fh: long_description = fh.read() try: - import ucloud.version + import uncloud.version - version = ucloud.version.VERSION + version = uncloud.version.VERSION except: import subprocess @@ -42,9 +42,9 @@ setup( "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3", "marshmallow", ], - scripts=["scripts/ucloud"], + scripts=["scripts/uncloud"], data_files=[ - (os.path.expanduser("~/ucloud/"), ["conf/ucloud.conf"]) + (os.path.expanduser("~/uncloud/"), ["conf/uncloud.conf"]) ], zip_safe=False, ) From 1fba79ca319b4ee51650fc6176b8b3264cb3f43d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 11:56:28 +0100 Subject: [PATCH 063/163] remove syslog handler (cruft), add debug flag --- scripts/uncloud | 18 ++++++------------ uncloud/api/main.py | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index e5c4081..d22c6d0 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -22,32 +22,26 @@ sys.excepthook = exception_hook if __name__ == '__main__': # Setting up root logger logger = logging.getLogger() + logger.setLevel(logging.DEBUG) - syslog_handler = SysLogHandler(address='/dev/log') - syslog_handler.setLevel(logging.DEBUG) - syslog_formatter = logging.Formatter('%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s') - syslog_handler.setFormatter(syslog_formatter) - - logger.addHandler(syslog_handler) + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument("--debug", "-d", action='store_true') arg_parser = argparse.ArgumentParser() + subparsers = arg_parser.add_subparsers(dest="command") - api_parser = subparsers.add_parser("api") - + api_parser = subparsers.add_parser("api", parents=[parent_parser]) host_parser = subparsers.add_parser("host") host_parser.add_argument("--hostname", required=True) scheduler_parser = subparsers.add_parser("scheduler") - filescanner_parser = subparsers.add_parser("filescanner") - imagescanner_parser = subparsers.add_parser("imagescanner") - metadata_parser = subparsers.add_parser("metadata") - config_parser = subparsers.add_parser("configure") + configure_parser(config_parser) args = arg_parser.parse_args() diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 6ac5d44..37c6c5b 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -561,7 +561,7 @@ api.add_resource(ListHost, "/host/list") api.add_resource(CreateNetwork, "/network/create") -def main(): +def main(debug=False): image_stores = list( shared.etcd_client.get_prefix( settings["etcd"]["image_store_prefix"], value_in_json=True From bff12ed930a821d5ef988d73173e665f42798755 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 12:15:05 +0100 Subject: [PATCH 064/163] ++ exception handling --- scripts/uncloud | 3 +++ uncloud/__init__.py | 2 ++ uncloud/common/etcd_wrapper.py | 7 ++++--- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index d22c6d0..4625164 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -8,6 +8,7 @@ import sys from logging.handlers import SysLogHandler from uncloud.configure.main import configure_parser +from uncloud import UncloudException def exception_hook(exc_type, exc_value, exc_traceback): logging.getLogger(__name__).error( @@ -61,5 +62,7 @@ if __name__ == '__main__': mod = importlib.import_module("uncloud.{}.main".format(name)) main = getattr(mod, "main") main(**arguments) + except UncloudException as err: + logger.error(err) except Exception as err: logger.exception(err) diff --git a/uncloud/__init__.py b/uncloud/__init__.py index e69de29..2920f47 100644 --- a/uncloud/__init__.py +++ b/uncloud/__init__.py @@ -0,0 +1,2 @@ +class UncloudException(Exception): + pass diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index 7367a6c..6a979ba 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -2,6 +2,7 @@ import etcd3 import json import queue import copy +from uncloud import UncloudException from collections import namedtuple from functools import wraps @@ -29,9 +30,9 @@ def readable_errors(func): try: return func(*args, **kwargs) except etcd3.exceptions.ConnectionFailedError as err: - raise etcd3.exceptions.ConnectionFailedError( - "etcd connection failed." - ) from err + raise UncloudException( + "Cannot connect to etcd: is etcd running as configured in uncloud.conf?" + ) except etcd3.exceptions.ConnectionTimeoutError as err: raise etcd3.exceptions.ConnectionTimeoutError( "etcd connection timeout." From 29dfacfadb821c76c938d4079c21a5039c3f6f80 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 12:15:50 +0100 Subject: [PATCH 065/163] Update .gitignore for uncloud --- .gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 82146fa..5c55899 100644 --- a/.gitignore +++ b/.gitignore @@ -2,17 +2,17 @@ .vscode -ucloud/docs/build +uncloud/docs/build logs.txt -ucloud.egg-info +uncloud.egg-info # run artefacts default.etcd __pycache__ # build artefacts -ucloud/version.py +uncloud/version.py build/ venv/ dist/ From 71c3f9d97870324e952fdac606e3a53ca72d047a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 13:13:19 +0100 Subject: [PATCH 066/163] begin adding port support, catch OSError from Flask --- scripts/uncloud | 2 ++ uncloud/api/main.py | 55 ++++++++++++++++++++++++++++----------------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 4625164..28d8344 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -34,6 +34,8 @@ if __name__ == '__main__': subparsers = arg_parser.add_subparsers(dest="command") api_parser = subparsers.add_parser("api", parents=[parent_parser]) + api_parser.add_argument("--port", "-p") + host_parser = subparsers.add_parser("host") host_parser.add_argument("--hostname", required=True) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 37c6c5b..861c1bc 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -17,7 +17,7 @@ from uncloud.shared import shared from . import schemas from .helper import generate_mac, mac2ipv6 - +from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -561,29 +561,42 @@ api.add_resource(ListHost, "/host/list") api.add_resource(CreateNetwork, "/network/create") -def main(debug=False): - image_stores = list( - shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], value_in_json=True +def main(debug=False, port=None): + try: + image_stores = list( + shared.etcd_client.get_prefix( + settings["etcd"]["image_store_prefix"], value_in_json=True + ) ) - ) - if not image_stores: - data = { - "is_public": True, - "type": "ceph", - "name": "images", - "description": "first ever public image-store", - "attributes": {"list": [], "key": [], "pool": "images"}, - } + except KeyError: + image_stores = False - shared.etcd_client.put( - join_path( - settings["etcd"]["image_store_prefix"], uuid4().hex - ), - json.dumps(data), - ) + # Do not inject default values that might be very wrong + # fail when required, not before + # + # if not image_stores: + # data = { + # "is_public": True, + # "type": "ceph", + # "name": "images", + # "description": "first ever public image-store", + # "attributes": {"list": [], "key": [], "pool": "images"}, + # } - app.run(host="::", debug=False) + # shared.etcd_client.put( + # join_path( + # settings["etcd"]["image_store_prefix"], uuid4().hex + # ), + # json.dumps(data), + # ) + + if port: + app_port = port + + try: + app.run(host="::", debug=False) + except OSError as e: + raise UncloudException("Failed to start Flask: {}".format(e)) if __name__ == "__main__": From 9662e02eb79f598ad5eb50358bb91b9df9c53c64 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 13:50:56 +0100 Subject: [PATCH 067/163] Allow to not have keys in etcd --- uncloud/settings/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/uncloud/settings/__init__.py b/uncloud/settings/__init__.py index 90b938c..629660e 100644 --- a/uncloud/settings/__init__.py +++ b/uncloud/settings/__init__.py @@ -115,7 +115,13 @@ class Settings(object): ) def __getitem__(self, key): - self.read_values_from_etcd() + # Allow failing to read from etcd if we have + # it locally + try: + self.read_values_from_etcd() + except KeyError as e: + pass + return self.config_parser[key] From e7755708846f62bb132a21b4380f24476804996a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 14:06:51 +0100 Subject: [PATCH 068/163] Make uncloud host running --- uncloud/api/main.py | 7 +++---- uncloud/host/main.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 861c1bc..93bada7 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -590,11 +590,10 @@ def main(debug=False, port=None): # json.dumps(data), # ) - if port: - app_port = port - try: - app.run(host="::", debug=False) + app.run(host="::", + port=port, + debug=debug) except OSError as e: raise UncloudException("Failed to start Flask: {}".format(e)) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index 80527c9..d1e7c9a 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -1,6 +1,7 @@ import argparse import multiprocessing as mp import time +from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType from uncloud.shared import shared @@ -42,7 +43,21 @@ def maintenance(host): def main(hostname): host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) - assert host is not None, "No such host with name = {}".format(hostname) + + # Does not yet exist, create it + if not host: + host_key = join_path( + settings["etcd"]["host_prefix"], uuid4().hex + ) + host_entry = { + "specs": "", + "hostname": hostname, + "status": "DEAD", + "last_heartbeat": "", + } + shared.etcd_client.put( + host_key, host_entry, value_in_json=True + ) try: heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) From 2566e86f1e92647a0cd1e0552c2196a841b39abe Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 14:13:08 +0100 Subject: [PATCH 069/163] [host] get ourselves from etcd --- uncloud/host/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index d1e7c9a..5ce9e6e 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -59,6 +59,9 @@ def main(hostname): host_key, host_entry, value_in_json=True ) + # update, get ourselves now for sure + host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + try: heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() From eb19b1033363322e10823d8461c7f6645c13eccd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 14:22:44 +0100 Subject: [PATCH 070/163] [scheduler] partial debug support --- scripts/uncloud | 3 ++- uncloud/scheduler/main.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 28d8344..5efc2a5 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -39,7 +39,8 @@ if __name__ == '__main__': host_parser = subparsers.add_parser("host") host_parser.add_argument("--hostname", required=True) - scheduler_parser = subparsers.add_parser("scheduler") + scheduler_parser = subparsers.add_parser("scheduler", parents=[parent_parser]) + filescanner_parser = subparsers.add_parser("filescanner") imagescanner_parser = subparsers.add_parser("imagescanner") metadata_parser = subparsers.add_parser("metadata") diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 5a4014f..20fa0d6 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -16,7 +16,7 @@ from .helper import ( from . import logger -def main(): +def main(debug=False): for request_iterator in [ shared.etcd_client.get_prefix( settings["etcd"]["request_prefix"], value_in_json=True From b95037f624e60be4df96f57c49d61d83028296a0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 31 Dec 2019 15:35:49 +0100 Subject: [PATCH 071/163] [metadata] allow passing in the port --- scripts/uncloud | 4 ++++ uncloud/metadata/main.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 5efc2a5..8add1d6 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -41,9 +41,13 @@ if __name__ == '__main__': scheduler_parser = subparsers.add_parser("scheduler", parents=[parent_parser]) + filescanner_parser = subparsers.add_parser("filescanner") imagescanner_parser = subparsers.add_parser("imagescanner") + metadata_parser = subparsers.add_parser("metadata") + metadata_parser.add_argument("--port", "-p") + config_parser = subparsers.add_parser("configure") configure_parser(config_parser) diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index da993ae..389b9a0 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -111,8 +111,8 @@ class Root(Resource): api.add_resource(Root, "/") -def main(): - app.run(debug=True, host="::", port="80") +def main(port=None): + app.run(debug=True, host="::", port=port) if __name__ == "__main__": From 2afb37daca75faede6fd14f4bd2b6a615967e2d7 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 31 Dec 2019 20:33:55 +0500 Subject: [PATCH 072/163] get() methods converted to post() --- uncloud/api/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 93bada7..1cb736f 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -90,7 +90,7 @@ class CreateVM(Resource): class VmStatus(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.VMStatusSchema(data) if validator.is_valid(): @@ -244,7 +244,7 @@ class VMMigration(Resource): class ListUserVM(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) @@ -277,7 +277,7 @@ class ListUserVM(Resource): class ListUserFiles(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) @@ -344,7 +344,7 @@ class ListHost(Resource): class GetSSHKeys(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.GetSSHSchema(data) if validator.is_valid(): @@ -430,7 +430,7 @@ class AddSSHKey(Resource): class RemoveSSHKey(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.RemoveSSHSchema(data) if validator.is_valid(): @@ -518,7 +518,7 @@ class CreateNetwork(Resource): class ListUserNetwork(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) From cd2f0aaa0d84f0844a2af2d019a572625d3dcf01 Mon Sep 17 00:00:00 2001 From: meow Date: Wed, 1 Jan 2020 14:59:47 +0500 Subject: [PATCH 073/163] Using click instead of argparse in uncloud script --- uncloud/cli/__init__.py | 0 uncloud/cli/commands/__init__.py | 0 uncloud/cli/commands/helper.py | 62 +++++++++++++++++++++++++++++++ uncloud/cli/commands/host.py | 31 ++++++++++++++++ uncloud/cli/commands/image.py | 24 ++++++++++++ uncloud/cli/commands/network.py | 17 +++++++++ uncloud/cli/commands/user.py | 48 ++++++++++++++++++++++++ uncloud/cli/commands/vm.py | 63 ++++++++++++++++++++++++++++++++ uncloud/cli/main.py | 24 ++++++++++++ uncloud/filescanner/main.py | 2 +- uncloud/host/main.py | 2 +- uncloud/imagescanner/main.py | 2 +- uncloud/metadata/main.py | 4 +- uncloud/scheduler/main.py | 20 +++------- 14 files changed, 279 insertions(+), 20 deletions(-) create mode 100644 uncloud/cli/__init__.py create mode 100755 uncloud/cli/commands/__init__.py create mode 100755 uncloud/cli/commands/helper.py create mode 100755 uncloud/cli/commands/host.py create mode 100755 uncloud/cli/commands/image.py create mode 100644 uncloud/cli/commands/network.py create mode 100755 uncloud/cli/commands/user.py create mode 100755 uncloud/cli/commands/vm.py create mode 100644 uncloud/cli/main.py diff --git a/uncloud/cli/__init__.py b/uncloud/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/cli/commands/__init__.py b/uncloud/cli/commands/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/uncloud/cli/commands/helper.py b/uncloud/cli/commands/helper.py new file mode 100755 index 0000000..6bef690 --- /dev/null +++ b/uncloud/cli/commands/helper.py @@ -0,0 +1,62 @@ +import json +import binascii +import click +import requests + +from os.path import join as join_path + +from pyotp import TOTP +from uncloud.settings import settings + + +def load_dump_pretty(content): + if isinstance(content, bytes): + content = content.decode('utf-8') + parsed = json.loads(content) + return json.dumps(parsed, indent=4, sort_keys=True) + + +def make_request(*args, data=None, request_method=requests.post): + r = request_method( + join_path(settings['client']['api_server'], *args), json=data + ) + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') + + +def get_token(_, param, value): + if value is not None: + try: + token = TOTP(value).now() + except binascii.Error: + raise click.BadParameter('') + else: + param.name = 'token' + return token + + +def add_otp_options(f): + options = [ + click.option('--name', show_default='name mentioned in config file.', prompt=True), + click.option('--realm', show_default='realm mentioned in config file.', prompt=True), + click.option('--seed', callback=get_token, show_default='seed mentioned in config file', + prompt=True) + ] + + for opt in reversed(options): + f = opt(f) + + return f + + +def add_vm_options(f): + options = [ + click.option('--vm-name', required=True), + click.option('--action', required=True, default=f.__name__) + ] + for opt in reversed(options): + f = opt(f) + + return f diff --git a/uncloud/cli/commands/host.py b/uncloud/cli/commands/host.py new file mode 100755 index 0000000..29ee417 --- /dev/null +++ b/uncloud/cli/commands/host.py @@ -0,0 +1,31 @@ +import click +import requests + +from .helper import add_otp_options, make_request + + +@click.group() +def host(): + pass + + +@host.command('create') +@add_otp_options +@click.option('--hostname', required=True) +@click.option('--cpu', required=True, type=int) +@click.option('--ram', required=True) +@click.option('--os-ssd', required=True) +@click.option('--hdd', default=list(), multiple=True) +def create(**kwargs): + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + make_request('host', 'create', data=kwargs) + + +@host.command('list') +def list_host(): + make_request('host', 'list', request_method=requests.get) diff --git a/uncloud/cli/commands/image.py b/uncloud/cli/commands/image.py new file mode 100755 index 0000000..6f9fe86 --- /dev/null +++ b/uncloud/cli/commands/image.py @@ -0,0 +1,24 @@ +import click +import requests + +from .helper import make_request + + +@click.group() +def image(): + pass + + +@image.command('list') +@click.option('--public', is_flag=True) +def _list(public): + if public: + make_request('image', 'list-public', request_method=requests.get) + + +@image.command('create-from-file') +@click.option('--name', required=True) +@click.option('--uuid', required=True) +@click.option('--image-store-name', 'image_store', required=True) +def create_from_file(**kwargs): + make_request('image', 'create', data=kwargs) diff --git a/uncloud/cli/commands/network.py b/uncloud/cli/commands/network.py new file mode 100644 index 0000000..81e22d5 --- /dev/null +++ b/uncloud/cli/commands/network.py @@ -0,0 +1,17 @@ +import click + +from .helper import add_otp_options, make_request + + +@click.group() +def network(): + pass + + +@network.command('create') +@add_otp_options +@click.option('--network-name', required=True) +@click.option('--network-type', 'type', required=True) +@click.option('--user', required=True, type=bool, default=False) +def create(**kwargs): + make_request('network', 'create', data=kwargs) diff --git a/uncloud/cli/commands/user.py b/uncloud/cli/commands/user.py new file mode 100755 index 0000000..2116520 --- /dev/null +++ b/uncloud/cli/commands/user.py @@ -0,0 +1,48 @@ +import click + +from .helper import add_otp_options, make_request + + +@click.group() +def user(): + pass + + +@user.command('files') +@add_otp_options +def list_files(**kwargs): + make_request('user', 'files', data=kwargs) + + +@user.command('vms') +@add_otp_options +def list_vms(**kwargs): + make_request('user', 'vms', data=kwargs) + + +@user.command('networks') +@add_otp_options +def list_networks(**kwargs): + make_request('user', 'network', data=kwargs) + + +@user.command('add-ssh') +@add_otp_options +@click.option('--key-name', required=True) +@click.option('--key', required=True) +def add_ssh(**kwargs): + make_request('user', 'add-ssh', data=kwargs) + + +@user.command('remove-ssh') +@add_otp_options +@click.option('--key-name', required=True) +def remove_ssh(**kwargs): + make_request('user', 'remove-ssh', data=kwargs) + + +@user.command('get-ssh') +@add_otp_options +@click.option('--key-name', default='') +def get_ssh(**kwargs): + make_request('user', 'get-ssh', data=kwargs) diff --git a/uncloud/cli/commands/vm.py b/uncloud/cli/commands/vm.py new file mode 100755 index 0000000..8527ac2 --- /dev/null +++ b/uncloud/cli/commands/vm.py @@ -0,0 +1,63 @@ +import click + +from .helper import add_otp_options, make_request, add_vm_options + + +@click.group() +def vm(): + pass + + +@vm.command('create') +@add_otp_options +@add_vm_options +@click.option('--cpu', required=True, type=int) +@click.option('--ram', required=True) +@click.option('--os-ssd', required=True) +@click.option('--hdd', default=list(), multiple=True) +@click.option('--image', required=True) +@click.option('--network', default=list(), multiple=True) +def create(**kwargs): + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + make_request('vm', kwargs.pop('action'), data=kwargs) + + +@vm.command('start') +@add_otp_options +@add_vm_options +def start(**kwargs): + make_request('vm', 'action', data=kwargs) + + +@vm.command('stop') +@add_otp_options +@add_vm_options +def stop(**kwargs): + make_request('vm', 'action', data=kwargs) + + +@vm.command('delete') +@add_otp_options +@add_vm_options +def delete(**kwargs): + make_request('vm', 'action', data=kwargs) + + +@vm.command('status') +@add_otp_options +@click.option('--vm-name', required=True) +def status(**kwargs): + make_request('vm', 'status', data=kwargs) + + +@vm.command('migrate') +@add_otp_options +@click.option('--vm-name', required=True) +@click.option('--destination', required=True) +def vm_migration(**kwargs): + make_request('vm', 'migrate', data=kwargs) diff --git a/uncloud/cli/main.py b/uncloud/cli/main.py new file mode 100644 index 0000000..7d625c4 --- /dev/null +++ b/uncloud/cli/main.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import click + +from uncloud.cli.commands.vm import vm +from uncloud.cli.commands.user import user +from uncloud.cli.commands.host import host +from uncloud.cli.commands.image import image +from uncloud.cli.commands.network import network + + +@click.group() +def cli(): + pass + + +cli.add_command(vm) +cli.add_command(user) +cli.add_command(image) +cli.add_command(host) +cli.add_command(network) + +if __name__ == '__main__': + cli() diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index 7ce8654..bb318c3 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -67,7 +67,7 @@ def track_file(file, base_dir): os.setxattr(file, "user.utracked", b"True") -def main(): +def main(debug=False): base_dir = settings["storage"]["file_dir"] # Recursively Get All Files and Folder below BASE_DIR diff --git a/uncloud/host/main.py b/uncloud/host/main.py index 5ce9e6e..e469725 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -40,7 +40,7 @@ def maintenance(host): shared.vm_pool.put(vm) -def main(hostname): +def main(hostname, debug=False): host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index 93e4dd5..fc77809 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -26,7 +26,7 @@ def qemu_img_type(path): return qemu_img_info["format"] -def main(): +def main(debug=False): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( settings["etcd"]["image_prefix"], value_in_json=True diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index 389b9a0..a59a998 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -111,8 +111,8 @@ class Root(Resource): api.add_resource(Root, "/") -def main(port=None): - app.run(debug=True, host="::", port=port) +def main(port=None, debug=False): + app.run(debug=debug, host="::", port=port) if __name__ == "__main__": diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 20fa0d6..79b1edc 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -43,29 +43,19 @@ def main(debug=False): dead_host_mitigation(dead_hosts) elif request_entry.type == RequestType.ScheduleVM: - print(request_event.value) - logger.debug( - "%s, %s", request_entry.key, request_entry.value - ) + logger.debug("%s, %s", request_entry.key, request_entry.value) vm_entry = shared.vm_pool.get(request_entry.uuid) if vm_entry is None: - logger.info( - "Trying to act on {} but it is deleted".format( - request_entry.uuid - ) - ) + logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) continue - shared.etcd_client.client.delete( - request_entry.key - ) # consume Request + + shared.etcd_client.client.delete(request_entry.key) # consume Request try: assign_host(vm_entry) except NoSuitableHostFound: - vm_entry.add_log( - "Can't schedule VM. No Resource Left." - ) + vm_entry.add_log("Can't schedule VM. No Resource Left.") shared.vm_pool.put(vm_entry) logger.info("No Resource Left. Emailing admin....") From 50fb135726a17b4bf63e7e691ec34bcebca05fe9 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 3 Jan 2020 15:02:39 +0500 Subject: [PATCH 074/163] uncloud cli converted to argparse, code isn't beautiful yet. Would make it soom --- uncloud/cli/commands/helper.py | 62 ------------------ uncloud/cli/commands/host.py | 31 --------- uncloud/cli/commands/image.py | 24 ------- uncloud/cli/commands/network.py | 17 ----- uncloud/cli/commands/user.py | 48 -------------- uncloud/cli/commands/vm.py | 63 ------------------- uncloud/cli/helper.py | 42 +++++++++++++ uncloud/cli/host.py | 45 +++++++++++++ uncloud/cli/image.py | 38 +++++++++++ uncloud/cli/main.py | 37 ++++++----- uncloud/cli/network.py | 32 ++++++++++ uncloud/cli/user.py | 41 ++++++++++++ uncloud/cli/vm.py | 62 ++++++++++++++++++ .../commands/__init__.py => common/parser.py} | 0 14 files changed, 278 insertions(+), 264 deletions(-) delete mode 100755 uncloud/cli/commands/helper.py delete mode 100755 uncloud/cli/commands/host.py delete mode 100755 uncloud/cli/commands/image.py delete mode 100644 uncloud/cli/commands/network.py delete mode 100755 uncloud/cli/commands/user.py delete mode 100755 uncloud/cli/commands/vm.py create mode 100644 uncloud/cli/helper.py create mode 100644 uncloud/cli/host.py create mode 100644 uncloud/cli/image.py create mode 100644 uncloud/cli/network.py create mode 100755 uncloud/cli/user.py create mode 100644 uncloud/cli/vm.py rename uncloud/{cli/commands/__init__.py => common/parser.py} (100%) mode change 100755 => 100644 diff --git a/uncloud/cli/commands/helper.py b/uncloud/cli/commands/helper.py deleted file mode 100755 index 6bef690..0000000 --- a/uncloud/cli/commands/helper.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -import binascii -import click -import requests - -from os.path import join as join_path - -from pyotp import TOTP -from uncloud.settings import settings - - -def load_dump_pretty(content): - if isinstance(content, bytes): - content = content.decode('utf-8') - parsed = json.loads(content) - return json.dumps(parsed, indent=4, sort_keys=True) - - -def make_request(*args, data=None, request_method=requests.post): - r = request_method( - join_path(settings['client']['api_server'], *args), json=data - ) - try: - print(load_dump_pretty(r.content)) - except Exception: - print('Error occurred while getting output from api server.') - - -def get_token(_, param, value): - if value is not None: - try: - token = TOTP(value).now() - except binascii.Error: - raise click.BadParameter('') - else: - param.name = 'token' - return token - - -def add_otp_options(f): - options = [ - click.option('--name', show_default='name mentioned in config file.', prompt=True), - click.option('--realm', show_default='realm mentioned in config file.', prompt=True), - click.option('--seed', callback=get_token, show_default='seed mentioned in config file', - prompt=True) - ] - - for opt in reversed(options): - f = opt(f) - - return f - - -def add_vm_options(f): - options = [ - click.option('--vm-name', required=True), - click.option('--action', required=True, default=f.__name__) - ] - for opt in reversed(options): - f = opt(f) - - return f diff --git a/uncloud/cli/commands/host.py b/uncloud/cli/commands/host.py deleted file mode 100755 index 29ee417..0000000 --- a/uncloud/cli/commands/host.py +++ /dev/null @@ -1,31 +0,0 @@ -import click -import requests - -from .helper import add_otp_options, make_request - - -@click.group() -def host(): - pass - - -@host.command('create') -@add_otp_options -@click.option('--hostname', required=True) -@click.option('--cpu', required=True, type=int) -@click.option('--ram', required=True) -@click.option('--os-ssd', required=True) -@click.option('--hdd', default=list(), multiple=True) -def create(**kwargs): - kwargs['specs'] = { - 'cpu': kwargs.pop('cpu'), - 'ram': kwargs.pop('ram'), - 'os-ssd': kwargs.pop('os_ssd'), - 'hdd': kwargs.pop('hdd') - } - make_request('host', 'create', data=kwargs) - - -@host.command('list') -def list_host(): - make_request('host', 'list', request_method=requests.get) diff --git a/uncloud/cli/commands/image.py b/uncloud/cli/commands/image.py deleted file mode 100755 index 6f9fe86..0000000 --- a/uncloud/cli/commands/image.py +++ /dev/null @@ -1,24 +0,0 @@ -import click -import requests - -from .helper import make_request - - -@click.group() -def image(): - pass - - -@image.command('list') -@click.option('--public', is_flag=True) -def _list(public): - if public: - make_request('image', 'list-public', request_method=requests.get) - - -@image.command('create-from-file') -@click.option('--name', required=True) -@click.option('--uuid', required=True) -@click.option('--image-store-name', 'image_store', required=True) -def create_from_file(**kwargs): - make_request('image', 'create', data=kwargs) diff --git a/uncloud/cli/commands/network.py b/uncloud/cli/commands/network.py deleted file mode 100644 index 81e22d5..0000000 --- a/uncloud/cli/commands/network.py +++ /dev/null @@ -1,17 +0,0 @@ -import click - -from .helper import add_otp_options, make_request - - -@click.group() -def network(): - pass - - -@network.command('create') -@add_otp_options -@click.option('--network-name', required=True) -@click.option('--network-type', 'type', required=True) -@click.option('--user', required=True, type=bool, default=False) -def create(**kwargs): - make_request('network', 'create', data=kwargs) diff --git a/uncloud/cli/commands/user.py b/uncloud/cli/commands/user.py deleted file mode 100755 index 2116520..0000000 --- a/uncloud/cli/commands/user.py +++ /dev/null @@ -1,48 +0,0 @@ -import click - -from .helper import add_otp_options, make_request - - -@click.group() -def user(): - pass - - -@user.command('files') -@add_otp_options -def list_files(**kwargs): - make_request('user', 'files', data=kwargs) - - -@user.command('vms') -@add_otp_options -def list_vms(**kwargs): - make_request('user', 'vms', data=kwargs) - - -@user.command('networks') -@add_otp_options -def list_networks(**kwargs): - make_request('user', 'network', data=kwargs) - - -@user.command('add-ssh') -@add_otp_options -@click.option('--key-name', required=True) -@click.option('--key', required=True) -def add_ssh(**kwargs): - make_request('user', 'add-ssh', data=kwargs) - - -@user.command('remove-ssh') -@add_otp_options -@click.option('--key-name', required=True) -def remove_ssh(**kwargs): - make_request('user', 'remove-ssh', data=kwargs) - - -@user.command('get-ssh') -@add_otp_options -@click.option('--key-name', default='') -def get_ssh(**kwargs): - make_request('user', 'get-ssh', data=kwargs) diff --git a/uncloud/cli/commands/vm.py b/uncloud/cli/commands/vm.py deleted file mode 100755 index 8527ac2..0000000 --- a/uncloud/cli/commands/vm.py +++ /dev/null @@ -1,63 +0,0 @@ -import click - -from .helper import add_otp_options, make_request, add_vm_options - - -@click.group() -def vm(): - pass - - -@vm.command('create') -@add_otp_options -@add_vm_options -@click.option('--cpu', required=True, type=int) -@click.option('--ram', required=True) -@click.option('--os-ssd', required=True) -@click.option('--hdd', default=list(), multiple=True) -@click.option('--image', required=True) -@click.option('--network', default=list(), multiple=True) -def create(**kwargs): - kwargs['specs'] = { - 'cpu': kwargs.pop('cpu'), - 'ram': kwargs.pop('ram'), - 'os-ssd': kwargs.pop('os_ssd'), - 'hdd': kwargs.pop('hdd') - } - make_request('vm', kwargs.pop('action'), data=kwargs) - - -@vm.command('start') -@add_otp_options -@add_vm_options -def start(**kwargs): - make_request('vm', 'action', data=kwargs) - - -@vm.command('stop') -@add_otp_options -@add_vm_options -def stop(**kwargs): - make_request('vm', 'action', data=kwargs) - - -@vm.command('delete') -@add_otp_options -@add_vm_options -def delete(**kwargs): - make_request('vm', 'action', data=kwargs) - - -@vm.command('status') -@add_otp_options -@click.option('--vm-name', required=True) -def status(**kwargs): - make_request('vm', 'status', data=kwargs) - - -@vm.command('migrate') -@add_otp_options -@click.option('--vm-name', required=True) -@click.option('--destination', required=True) -def vm_migration(**kwargs): - make_request('vm', 'migrate', data=kwargs) diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py new file mode 100644 index 0000000..bdcce78 --- /dev/null +++ b/uncloud/cli/helper.py @@ -0,0 +1,42 @@ +import requests +import json +import argparse +import binascii + +from pyotp import TOTP +from os.path import join as join_path +from uncloud.settings import settings + + +def get_otp_parser(): + otp_parser = argparse.ArgumentParser('otp') + otp_parser.add_argument('--name', default=settings['client']['name']) + otp_parser.add_argument('--realm', default=settings['client']['realm']) + otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'], + dest='token', metavar='SEED') + return otp_parser + + +def load_dump_pretty(content): + if isinstance(content, bytes): + content = content.decode('utf-8') + parsed = json.loads(content) + return json.dumps(parsed, indent=4, sort_keys=True) + + +def make_request(*args, data=None, request_method=requests.post): + r = request_method(join_path(settings['client']['api_server'], *args), json=data) + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except binascii.Error: + raise argparse.ArgumentTypeError('Invalid seed') + else: + return token diff --git a/uncloud/cli/host.py b/uncloud/cli/host.py new file mode 100644 index 0000000..e912567 --- /dev/null +++ b/uncloud/cli/host.py @@ -0,0 +1,45 @@ +import requests + +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class HostParser(BaseParser): + def __init__(self): + super().__init__('host') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs) + p.add_argument('--hostname', required=True) + p.add_argument('--cpu', required=True, type=int) + p.add_argument('--ram', required=True) + p.add_argument('--os-ssd', required=True) + p.add_argument('--hdd', default=list()) + + def list(self, **kwargs): + self.subparser.add_parser('list', **kwargs) + + +parser = HostParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('host_subcommand') + if not subcommand: + arg_parser.print_help() + else: + request_method = requests.post + data = None + if subcommand == 'create': + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + data = kwargs + elif subcommand == 'list': + request_method = requests.get + + make_request('host', subcommand, data=data, request_method=request_method) diff --git a/uncloud/cli/image.py b/uncloud/cli/image.py new file mode 100644 index 0000000..3db9577 --- /dev/null +++ b/uncloud/cli/image.py @@ -0,0 +1,38 @@ +import requests + +from uncloud.cli.helper import make_request +from uncloud.common.parser import BaseParser + + +class ImageParser(BaseParser): + def __init__(self): + super().__init__('image') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', **kwargs) + p.add_argument('--name', required=True) + p.add_argument('--uuid', required=True) + p.add_argument('--image-store-name', default='image_store') + + def list(self, **kwargs): + self.subparser.add_parser('list', add_help=False, **kwargs) + + +parser = ImageParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('image_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = None + request_method = requests.post + if subcommand == 'list': + subcommand = 'list-public' + request_method = requests.get + elif subcommand == 'create': + data = kwargs + + make_request('image', subcommand, data=data, request_method=request_method) diff --git a/uncloud/cli/main.py b/uncloud/cli/main.py index 7d625c4..7f5e367 100644 --- a/uncloud/cli/main.py +++ b/uncloud/cli/main.py @@ -1,24 +1,23 @@ #!/usr/bin/env python3 -import click +import argparse +import importlib -from uncloud.cli.commands.vm import vm -from uncloud.cli.commands.user import user -from uncloud.cli.commands.host import host -from uncloud.cli.commands.image import image -from uncloud.cli.commands.network import network +arg_parser = argparse.ArgumentParser('cli', add_help=False) +subparser = arg_parser.add_subparsers(dest='subcommand') + +for component in ['user', 'host', 'image', 'network', 'vm']: + module = importlib.import_module('uncloud.cli.{}'.format(component)) + parser = getattr(module, 'arg_parser') + subparser.add_parser(name=parser.prog, parents=[parser]) -@click.group() -def cli(): - pass - - -cli.add_command(vm) -cli.add_command(user) -cli.add_command(image) -cli.add_command(host) -cli.add_command(network) - -if __name__ == '__main__': - cli() +def main(**kwargs): + if not kwargs['subcommand']: + arg_parser.print_help() + else: + name = kwargs.pop('subcommand') + kwargs.pop('debug') + mod = importlib.import_module('uncloud.cli.{}'.format(name)) + _main = getattr(mod, 'main') + _main(**kwargs) diff --git a/uncloud/cli/network.py b/uncloud/cli/network.py new file mode 100644 index 0000000..33e41a9 --- /dev/null +++ b/uncloud/cli/network.py @@ -0,0 +1,32 @@ +import requests + +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class NetworkParser(BaseParser): + def __init__(self): + super().__init__('network') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', add_help=False, parents=[get_otp_parser()], **kwargs) + p.add_argument('--network-name', required=True) + p.add_argument('--network-type', required=True, dest='type') + p.add_argument('--user', action='store_true') + + +parser = NetworkParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('network_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = None + request_method = requests.post + if subcommand == 'create': + data = kwargs + + make_request('network', subcommand, data=data, request_method=request_method) diff --git a/uncloud/cli/user.py b/uncloud/cli/user.py new file mode 100755 index 0000000..3a4cc4e --- /dev/null +++ b/uncloud/cli/user.py @@ -0,0 +1,41 @@ +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class UserParser(BaseParser): + def __init__(self): + super().__init__('user') + + def files(self, **kwargs): + self.subparser.add_parser('files', parents=[get_otp_parser()], **kwargs) + + def vms(self, **kwargs): + self.subparser.add_parser('vms', parents=[get_otp_parser()], **kwargs) + + def networks(self, **kwargs): + self.subparser.add_parser('networks', parents=[get_otp_parser()], **kwargs) + + def add_ssh(self, **kwargs): + p = self.subparser.add_parser('add-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', required=True) + p.add_argument('--key', required=True) + + def get_ssh(self, **kwargs): + p = self.subparser.add_parser('get-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', default='') + + def remove_ssh(self, **kwargs): + p = self.subparser.add_parser('remove-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', required=True) + + +parser = UserParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('user_subcommand') + if not subcommand: + arg_parser.print_help() + else: + make_request('user', subcommand, data=kwargs) diff --git a/uncloud/cli/vm.py b/uncloud/cli/vm.py new file mode 100644 index 0000000..396530e --- /dev/null +++ b/uncloud/cli/vm.py @@ -0,0 +1,62 @@ +from uncloud.common.parser import BaseParser +from uncloud.cli.helper import make_request, get_otp_parser + + +class VMParser(BaseParser): + def __init__(self): + super().__init__('vm') + + def start(self, **args): + p = self.subparser.add_parser('start', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def stop(self, **args): + p = self.subparser.add_parser('stop', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def status(self, **args): + p = self.subparser.add_parser('status', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def delete(self, **args): + p = self.subparser.add_parser('delete', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def migrate(self, **args): + p = self.subparser.add_parser('migrate', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + p.add_argument('--destination', required=True) + + def create(self, **args): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **args) + p.add_argument('--cpu', required=True) + p.add_argument('--ram', required=True) + p.add_argument('--os-ssd', required=True) + p.add_argument('--hdd', action='append', default=list()) + p.add_argument('--image', required=True) + p.add_argument('--network', action='append', default=[]) + p.add_argument('--vm-name', required=True) + + +parser = VMParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('vm_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = kwargs + endpoint = subcommand + if subcommand in ['start', 'stop', 'delete']: + endpoint = 'action' + data['action'] = subcommand + elif subcommand == 'create': + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + make_request('vm', endpoint, data=data) diff --git a/uncloud/cli/commands/__init__.py b/uncloud/common/parser.py old mode 100755 new mode 100644 similarity index 100% rename from uncloud/cli/commands/__init__.py rename to uncloud/common/parser.py From 3296e524cc3e2d2d36be68072425e54a2ece74e1 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 3 Jan 2020 18:38:59 +0500 Subject: [PATCH 075/163] uncloud cli converted to argparse --- conf/uncloud.conf | 6 +- scripts/uncloud | 49 ++---- setup.py | 2 +- uncloud/api/main.py | 331 ++++++++++++++++++----------------- uncloud/api/schemas.py | 18 +- uncloud/common/parser.py | 13 ++ uncloud/configure/main.py | 91 ++++------ uncloud/filescanner/main.py | 4 + uncloud/host/main.py | 4 + uncloud/imagescanner/main.py | 4 + uncloud/metadata/main.py | 4 + uncloud/scheduler/main.py | 11 +- uncloud/settings/__init__.py | 34 ++-- 13 files changed, 284 insertions(+), 287 deletions(-) diff --git a/conf/uncloud.conf b/conf/uncloud.conf index 334bbeb..9d4358d 100644 --- a/conf/uncloud.conf +++ b/conf/uncloud.conf @@ -1,7 +1,11 @@ [etcd] url = localhost port = 2379 - ca_cert cert_cert cert_key + +[client] +name = replace_me +realm = replace_me +seed = replace_me diff --git a/scripts/uncloud b/scripts/uncloud index 8add1d6..968ace6 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -1,14 +1,13 @@ #!/usr/bin/env python3 -import argparse import logging -import importlib -import multiprocessing as mp import sys - -from logging.handlers import SysLogHandler -from uncloud.configure.main import configure_parser +import importlib +import argparse +import multiprocessing as mp from uncloud import UncloudException +from contextlib import suppress + def exception_hook(exc_type, exc_value, exc_traceback): logging.getLogger(__name__).error( @@ -19,40 +18,25 @@ def exception_hook(exc_type, exc_value, exc_traceback): sys.excepthook = exception_hook - if __name__ == '__main__': # Setting up root logger logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument("--debug", "-d", action='store_true') - arg_parser = argparse.ArgumentParser() + subparsers = arg_parser.add_subparsers(dest='command') - subparsers = arg_parser.add_subparsers(dest="command") + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument('--debug', '-d', action='store_true', default=False, + help='More verbose logging') - api_parser = subparsers.add_parser("api", parents=[parent_parser]) - api_parser.add_argument("--port", "-p") + for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', + 'metadata', 'configure', 'cli']: + mod = importlib.import_module('uncloud.{}.main'.format(component)) + parser = getattr(mod, 'arg_parser') + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - host_parser = subparsers.add_parser("host") - host_parser.add_argument("--hostname", required=True) - - scheduler_parser = subparsers.add_parser("scheduler", parents=[parent_parser]) - - - filescanner_parser = subparsers.add_parser("filescanner") - imagescanner_parser = subparsers.add_parser("imagescanner") - - metadata_parser = subparsers.add_parser("metadata") - metadata_parser.add_argument("--port", "-p") - - config_parser = subparsers.add_parser("configure") - - configure_parser(config_parser) args = arg_parser.parse_args() - if not args.command: arg_parser.print_help() else: @@ -62,12 +46,11 @@ if __name__ == '__main__': # errors out, so the following command configure multiprocessing # module to not inherit anything from parent. mp.set_start_method('spawn') - arguments = vars(args) try: name = arguments.pop('command') - mod = importlib.import_module("uncloud.{}.main".format(name)) - main = getattr(mod, "main") + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') main(**arguments) except UncloudException as err: logger.error(err) diff --git a/setup.py b/setup.py index 0764d74..12da6b8 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ setup( "pynetbox", "colorama", "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3", - "marshmallow", + "marshmallow" ], scripts=["scripts/uncloud"], data_files=[ diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 1cb736f..47e7003 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -1,6 +1,7 @@ import json import pynetbox import logging +import argparse from uuid import uuid4 from os.path import join as join_path @@ -14,7 +15,6 @@ from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.settings import settings from uncloud.shared import shared - from . import schemas from .helper import generate_mac, mac2ipv6 from uncloud import UncloudException @@ -25,6 +25,9 @@ app = Flask(__name__) api = Api(app) app.logger.handlers.clear() +arg_parser = argparse.ArgumentParser('api', add_help=False) +arg_parser.add_argument('--port', '-p') + @app.errorhandler(Exception) def handle_exception(e): @@ -34,11 +37,11 @@ def handle_exception(e): return e # now you're handling non-HTTP exceptions only - return {"message": "Server Error"}, 500 + return {'message': 'Server Error'}, 500 class CreateVM(Resource): - """API Request to Handle Creation of VM""" + '''API Request to Handle Creation of VM''' @staticmethod def post(): @@ -46,33 +49,33 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(settings["etcd"]["vm_prefix"], vm_uuid) + vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) specs = { - "cpu": validator.specs["cpu"], - "ram": validator.specs["ram"], - "os-ssd": validator.specs["os-ssd"], - "hdd": validator.specs["hdd"], + 'cpu': validator.specs['cpu'], + 'ram': validator.specs['ram'], + 'os-ssd': validator.specs['os-ssd'], + 'hdd': validator.specs['hdd'], } - macs = [generate_mac() for _ in range(len(data["network"]))] + macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, "/v1/counter/tap" + shared.etcd_client, '/v1/counter/tap' ) - for _ in range(len(data["network"])) + for _ in range(len(data['network'])) ] vm_entry = { - "name": data["vm_name"], - "owner": data["name"], - "owner_realm": data["realm"], - "specs": specs, - "hostname": "", - "status": VMStatus.stopped, - "image_uuid": validator.image_uuid, - "log": [], - "vnc_socket": "", - "network": list(zip(data["network"], macs, tap_ids)), - "metadata": {"ssh-keys": []}, - "in_migration": False, + 'name': data['vm_name'], + 'owner': data['name'], + 'owner_realm': data['realm'], + 'specs': specs, + 'hostname': '', + 'status': VMStatus.stopped, + 'image_uuid': validator.image_uuid, + 'log': [], + 'vnc_socket': '', + 'network': list(zip(data['network'], macs, tap_ids)), + 'metadata': {'ssh-keys': []}, + 'in_migration': False, } shared.etcd_client.put(vm_key, vm_entry, value_in_json=True) @@ -80,11 +83,11 @@ class CreateVM(Resource): r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=settings['etcd']['request_prefix'], ) shared.request_pool.put(r) - return {"message": "VM Creation Queued"}, 200 + return {'message': 'VM Creation Queued'}, 200 return validator.get_errors(), 400 @@ -95,24 +98,24 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], data["uuid"]) + join_path(settings['etcd']['vm_prefix'], data['uuid']) ) vm_value = vm.value.copy() - vm_value["ip"] = [] + vm_value['ip'] = [] for network_mac_and_tap in vm.network: network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], - data["name"], + settings['etcd']['network_prefix'], + data['name'], network_name, ), value_in_json=True, ) ipv6_addr = ( - network.value.get("ipv6").split("::")[0] + "::" + network.value.get('ipv6').split('::')[0] + '::' ) - vm_value["ip"].append(mac2ipv6(mac, ipv6_addr)) + vm_value['ip'].append(mac2ipv6(mac, ipv6_addr)) vm.value = vm_value return vm.value else: @@ -126,26 +129,26 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = shared.etcd_client.get( - join_path(settings["etcd"]["file_prefix"], data["uuid"]) + join_path(settings['etcd']['file_prefix'], data['uuid']) ) file_entry_value = json.loads(file_entry.value) image_entry_json = { - "status": "TO_BE_CREATED", - "owner": file_entry_value["owner"], - "filename": file_entry_value["filename"], - "name": data["name"], - "store_name": data["image_store"], - "visibility": "public", + 'status': 'TO_BE_CREATED', + 'owner': file_entry_value['owner'], + 'filename': file_entry_value['filename'], + 'name': data['name'], + 'store_name': data['image_store'], + 'visibility': 'public', } shared.etcd_client.put( join_path( - settings["etcd"]["image_prefix"], data["uuid"] + settings['etcd']['image_prefix'], data['uuid'] ), json.dumps(image_entry_json), ) - return {"message": "Image queued for creation."} + return {'message': 'Image queued for creation.'} return validator.get_errors(), 400 @@ -153,15 +156,15 @@ class ListPublicImages(Resource): @staticmethod def get(): images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + settings['etcd']['image_prefix'], value_in_json=True ) - r = {"images": []} + r = {'images': []} for image in images: - image_key = "{}:{}".format( - image.value["store_name"], image.value["name"] + image_key = '{}:{}'.format( + image.value['store_name'], image.value['name'] ) - r["images"].append( - {"name": image_key, "status": image.value["status"]} + r['images'].append( + {'name': image_key, 'status': image.value['status']} ) return r, 200 @@ -174,14 +177,14 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], data["uuid"]) + join_path(settings['etcd']['vm_prefix'], data['uuid']) ) - action = data["action"] + action = data['action'] - if action == "start": - action = "schedule" + if action == 'start': + action = 'schedule' - if action == "delete" and vm_entry.hostname == "": + if action == 'delete' and vm_entry.hostname == '': if shared.storage_handler.is_vm_image_exists( vm_entry.uuid ): @@ -190,25 +193,25 @@ class VMAction(Resource): ) if r_status: shared.etcd_client.client.delete(vm_entry.key) - return {"message": "VM successfully deleted"} + return {'message': 'VM successfully deleted'} else: logger.error( - "Some Error Occurred while deleting VM" + 'Some Error Occurred while deleting VM' ) - return {"message": "VM deletion unsuccessfull"} + return {'message': 'VM deletion unsuccessfull'} else: shared.etcd_client.client.delete(vm_entry.key) - return {"message": "VM successfully deleted"} + return {'message': 'VM successfully deleted'} r = RequestEntry.from_scratch( - type="{}VM".format(action.title()), - uuid=data["uuid"], + type='{}VM'.format(action.title()), + uuid=data['uuid'], hostname=vm_entry.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( - {"message": "VM {} Queued".format(action.title())}, + {'message': 'VM {} Queued'.format(action.title())}, 200, ) else: @@ -222,20 +225,20 @@ class VMMigration(Resource): validator = schemas.VmMigrationSchema(data) if validator.is_valid(): - vm = shared.vm_pool.get(data["uuid"]) + vm = shared.vm_pool.get(data['uuid']) r = RequestEntry.from_scratch( type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( - settings["etcd"]["host_prefix"], + settings['etcd']['host_prefix'], validator.destination.value, ), - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( - {"message": "VM Migration Initialization Queued"}, + {'message': 'VM Migration Initialization Queued'}, 200, ) else: @@ -250,26 +253,26 @@ class ListUserVM(Resource): if validator.is_valid(): vms = shared.etcd_client.get_prefix( - settings["etcd"]["vm_prefix"], value_in_json=True + settings['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter( - lambda v: v.value["owner"] == data["name"], vms + lambda v: v.value['owner'] == data['name'], vms ) for vm in user_vms: return_vms.append( { - "name": vm.value["name"], - "vm_uuid": vm.key.split("/")[-1], - "specs": vm.value["specs"], - "status": vm.value["status"], - "hostname": vm.value["hostname"], - "vnc_socket": vm.value.get("vnc_socket", None), + 'name': vm.value['name'], + 'vm_uuid': vm.key.split('/')[-1], + 'specs': vm.value['specs'], + 'status': vm.value['status'], + 'hostname': vm.value['hostname'], + 'vnc_socket': vm.value.get('vnc_socket', None), } ) if return_vms: - return {"message": return_vms}, 200 - return {"message": "No VM found"}, 404 + return {'message': return_vms}, 200 + return {'message': 'No VM found'}, 404 else: return validator.get_errors(), 400 @@ -283,22 +286,22 @@ class ListUserFiles(Resource): if validator.is_valid(): files = shared.etcd_client.get_prefix( - settings["etcd"]["file_prefix"], value_in_json=True + settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] user_files = list( filter( - lambda f: f.value["owner"] == data["name"], files + lambda f: f.value['owner'] == data['name'], files ) ) for file in user_files: return_files.append( { - "filename": file.value["filename"], - "uuid": file.key.split("/")[-1], + 'filename': file.value['filename'], + 'uuid': file.key.split('/')[-1], } ) - return {"message": return_files}, 200 + return {'message': return_files}, 200 else: return validator.get_errors(), 400 @@ -310,19 +313,19 @@ class CreateHost(Resource): validator = schemas.CreateHostSchema(data) if validator.is_valid(): host_key = join_path( - settings["etcd"]["host_prefix"], uuid4().hex + settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { - "specs": data["specs"], - "hostname": data["hostname"], - "status": "DEAD", - "last_heartbeat": "", + 'specs': data['specs'], + 'hostname': data['hostname'], + 'status': 'DEAD', + 'last_heartbeat': '', } shared.etcd_client.put( host_key, host_entry, value_in_json=True ) - return {"message": "Host Created"}, 200 + return {'message': 'Host Created'}, 200 return validator.get_errors(), 400 @@ -333,9 +336,9 @@ class ListHost(Resource): hosts = shared.host_pool.hosts r = { host.key: { - "status": host.status, - "specs": host.specs, - "hostname": host.hostname, + 'status': host.status, + 'specs': host.specs, + 'hostname': host.hostname, } for host in hosts } @@ -352,29 +355,29 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", + settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', ) etcd_entry = shared.etcd_client.get_prefix( etcd_key, value_in_json=True ) keys = { - key.key.split("/")[-1]: key.value + key.key.split('/')[-1]: key.value for key in etcd_entry } - return {"keys": keys} + return {'keys': keys} else: # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True @@ -382,14 +385,14 @@ class GetSSHKeys(Resource): if etcd_entry: return { - "keys": { - etcd_entry.key.split("/")[ + 'keys': { + etcd_entry.key.split('/')[ -1 ]: etcd_entry.value } } else: - return {"keys": {}} + return {'keys': {}} else: return validator.get_errors(), 400 @@ -403,27 +406,27 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True ) if etcd_entry: return { - "message": "Key with name '{}' already exists".format( - data["key_name"] + 'message': 'Key with name "{}" already exists'.format( + data['key_name'] ) } else: # Key Not Found. It implies user' haven't added any key yet. shared.etcd_client.put( - etcd_key, data["key"], value_in_json=True + etcd_key, data['key'], value_in_json=True ) - return {"message": "Key added successfully"} + return {'message': 'Key added successfully'} else: return validator.get_errors(), 400 @@ -437,22 +440,22 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True ) if etcd_entry: shared.etcd_client.client.delete(etcd_key) - return {"message": "Key successfully removed."} + return {'message': 'Key successfully removed.'} else: return { - "message": "No Key with name '{}' Exists at all.".format( - data["key_name"] + 'message': 'No Key with name "{}" Exists at all.'.format( + data['key_name'] ) } else: @@ -468,50 +471,50 @@ class CreateNetwork(Resource): if validator.is_valid(): network_entry = { - "id": counters.increment_etcd_counter( - shared.etcd_client, "/v1/counter/vxlan" + 'id': counters.increment_etcd_counter( + shared.etcd_client, '/v1/counter/vxlan' ), - "type": data["type"], + 'type': data['type'], } if validator.user.value: try: nb = pynetbox.api( - url=settings["netbox"]["url"], - token=settings["netbox"]["token"], + url=settings['netbox']['url'], + token=settings['netbox']['token'], ) nb_prefix = nb.ipam.prefixes.get( - prefix=settings["network"]["prefix"] + prefix=settings['network']['prefix'] ) prefix = nb_prefix.available_prefixes.create( data={ - "prefix_length": int( - settings["network"]["prefix_length"] + 'prefix_length': int( + settings['network']['prefix_length'] ), - "description": '{}\'s network "{}"'.format( - data["name"], data["network_name"] + 'description': '{}\'s network "{}"'.format( + data['name'], data['network_name'] ), - "is_pool": True, + 'is_pool': True, } ) except Exception as err: app.logger.error(err) return { - "message": "Error occured while creating network." + 'message': 'Error occured while creating network.' } else: - network_entry["ipv6"] = prefix["prefix"] + network_entry['ipv6'] = prefix['prefix'] else: - network_entry["ipv6"] = "fd00::/64" + network_entry['ipv6'] = 'fd00::/64' network_key = join_path( - settings["etcd"]["network_prefix"], - data["name"], - data["network_name"], + settings['etcd']['network_prefix'], + data['name'], + data['network_name'], ) shared.etcd_client.put( network_key, network_entry, value_in_json=True ) - return {"message": "Network successfully added."} + return {'message': 'Network successfully added.'} else: return validator.get_errors(), 400 @@ -524,48 +527,48 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - settings["etcd"]["network_prefix"], data["name"] + settings['etcd']['network_prefix'], data['name'] ) networks = shared.etcd_client.get_prefix( prefix, value_in_json=True ) user_networks = [] for net in networks: - net.value["name"] = net.key.split("/")[-1] + net.value['name'] = net.key.split('/')[-1] user_networks.append(net.value) - return {"networks": user_networks}, 200 + return {'networks': user_networks}, 200 else: return validator.get_errors(), 400 -api.add_resource(CreateVM, "/vm/create") -api.add_resource(VmStatus, "/vm/status") +api.add_resource(CreateVM, '/vm/create') +api.add_resource(VmStatus, '/vm/status') -api.add_resource(VMAction, "/vm/action") -api.add_resource(VMMigration, "/vm/migrate") +api.add_resource(VMAction, '/vm/action') +api.add_resource(VMMigration, '/vm/migrate') -api.add_resource(CreateImage, "/image/create") -api.add_resource(ListPublicImages, "/image/list-public") +api.add_resource(CreateImage, '/image/create') +api.add_resource(ListPublicImages, '/image/list-public') -api.add_resource(ListUserVM, "/user/vms") -api.add_resource(ListUserFiles, "/user/files") -api.add_resource(ListUserNetwork, "/user/networks") +api.add_resource(ListUserVM, '/user/vms') +api.add_resource(ListUserFiles, '/user/files') +api.add_resource(ListUserNetwork, '/user/networks') -api.add_resource(AddSSHKey, "/user/add-ssh") -api.add_resource(RemoveSSHKey, "/user/remove-ssh") -api.add_resource(GetSSHKeys, "/user/get-ssh") +api.add_resource(AddSSHKey, '/user/add-ssh') +api.add_resource(RemoveSSHKey, '/user/remove-ssh') +api.add_resource(GetSSHKeys, '/user/get-ssh') -api.add_resource(CreateHost, "/host/create") -api.add_resource(ListHost, "/host/list") +api.add_resource(CreateHost, '/host/create') +api.add_resource(ListHost, '/host/list') -api.add_resource(CreateNetwork, "/network/create") +api.add_resource(CreateNetwork, '/network/create') def main(debug=False, port=None): try: image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], value_in_json=True + settings['etcd']['image_store_prefix'], value_in_json=True ) ) except KeyError: @@ -576,27 +579,27 @@ def main(debug=False, port=None): # # if not image_stores: # data = { - # "is_public": True, - # "type": "ceph", - # "name": "images", - # "description": "first ever public image-store", - # "attributes": {"list": [], "key": [], "pool": "images"}, + # 'is_public': True, + # 'type': 'ceph', + # 'name': 'images', + # 'description': 'first ever public image-store', + # 'attributes': {'list': [], 'key': [], 'pool': 'images'}, # } # shared.etcd_client.put( # join_path( - # settings["etcd"]["image_store_prefix"], uuid4().hex + # settings['etcd']['image_store_prefix'], uuid4().hex # ), # json.dumps(data), # ) try: - app.run(host="::", + app.run(host='::', port=port, debug=debug) except OSError as e: - raise UncloudException("Failed to start Flask: {}".format(e)) + raise UncloudException('Failed to start Flask: {}'.format(e)) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index 65055c4..8e06e8d 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -322,7 +322,7 @@ class CreateVMSchema(OTPSchema): "Your specified OS-SSD is not in correct units" ) - if _cpu < 1: + if int(_cpu) < 1: self.add_error("CPU must be atleast 1") if parsed_ram < bitmath.GB(1): @@ -528,9 +528,7 @@ class GetSSHSchema(OTPSchema): class CreateNetwork(OTPSchema): def __init__(self, data): - self.network_name = Field( - "network_name", str, data.get("network_name", KeyError) - ) + self.network_name = Field("network_name", str, data.get("network_name", KeyError)) self.type = Field("type", str, data.get("type", KeyError)) self.user = Field("user", bool, bool(data.get("user", False))) @@ -541,14 +539,10 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = shared.etcd_client.get( - os.path.join( - settings["etcd"]["network_prefix"], - self.name.value, - self.network_name.value, - ), - value_in_json=True, - ) + print(self.name.value, self.network_name.value) + key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) + print(key) + network = shared.etcd_client.get(key, value_in_json=True) if network: self.add_error( "Network with name {} already exists".format( diff --git a/uncloud/common/parser.py b/uncloud/common/parser.py index e69de29..576f0e7 100644 --- a/uncloud/common/parser.py +++ b/uncloud/common/parser.py @@ -0,0 +1,13 @@ +import argparse + + +class BaseParser: + def __init__(self, command): + self.arg_parser = argparse.ArgumentParser(command, add_help=False) + self.subparser = self.arg_parser.add_subparsers(dest='{}_subcommand'.format(command)) + self.common_args = {'add_help': False} + + methods = [attr for attr in dir(self) if not attr.startswith('__') + and type(getattr(self, attr)).__name__ == 'method'] + for method in methods: + getattr(self, method)(**self.common_args) diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index a9b4901..f89a30c 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,8 +1,43 @@ import os +import argparse from uncloud.settings import settings from uncloud.shared import shared +arg_parser = argparse.ArgumentParser('configure', add_help=False) +configure_subparsers = arg_parser.add_subparsers(dest='subcommand') + +otp_parser = configure_subparsers.add_parser('otp') +otp_parser.add_argument('--verification-controller-url', required=True, metavar='URL') +otp_parser.add_argument('--auth-name', required=True, metavar='OTP-NAME') +otp_parser.add_argument('--auth-realm', required=True, metavar='OTP-REALM') +otp_parser.add_argument('--auth-seed', required=True, metavar='OTP-SEED') + +network_parser = configure_subparsers.add_parser('network') +network_parser.add_argument('--prefix-length', required=True, type=int) +network_parser.add_argument('--prefix', required=True) +network_parser.add_argument('--vxlan-phy-dev', required=True) + +netbox_parser = configure_subparsers.add_parser('netbox') +netbox_parser.add_argument('--url', required=True) +netbox_parser.add_argument('--token', required=True) + +ssh_parser = configure_subparsers.add_parser('ssh') +ssh_parser.add_argument('--username', default='root') +ssh_parser.add_argument('--private-key-path', default=os.path.expanduser('~/.ssh/id_rsa'),) + +storage_parser = configure_subparsers.add_parser('storage') +storage_parser.add_argument('--file-dir', required=True) +storage_parser_subparsers = storage_parser.add_subparsers(dest='storage_backend') + +filesystem_storage_parser = storage_parser_subparsers.add_parser('filesystem') +filesystem_storage_parser.add_argument('--vm-dir', required=True) +filesystem_storage_parser.add_argument('--image-dir', required=True) + +ceph_storage_parser = storage_parser_subparsers.add_parser('ceph') +ceph_storage_parser.add_argument('--ceph-vm-pool', required=True) +ceph_storage_parser.add_argument('--ceph-image-pool', required=True) + def update_config(section, kwargs): uncloud_config = shared.etcd_client.get( @@ -19,61 +54,9 @@ def update_config(section, kwargs): ) -def configure_parser(parser): - configure_subparsers = parser.add_subparsers(dest="subcommand") - - otp_parser = configure_subparsers.add_parser("otp") - otp_parser.add_argument( - "--verification-controller-url", required=True, metavar="URL" - ) - otp_parser.add_argument( - "--auth-name", required=True, metavar="OTP-NAME" - ) - otp_parser.add_argument( - "--auth-realm", required=True, metavar="OTP-REALM" - ) - otp_parser.add_argument( - "--auth-seed", required=True, metavar="OTP-SEED" - ) - - network_parser = configure_subparsers.add_parser("network") - network_parser.add_argument( - "--prefix-length", required=True, type=int - ) - network_parser.add_argument("--prefix", required=True) - network_parser.add_argument("--vxlan-phy-dev", required=True) - - netbox_parser = configure_subparsers.add_parser("netbox") - netbox_parser.add_argument("--url", required=True) - netbox_parser.add_argument("--token", required=True) - - ssh_parser = configure_subparsers.add_parser("ssh") - ssh_parser.add_argument("--username", default="root") - ssh_parser.add_argument( - "--private-key-path", - default=os.path.expanduser("~/.ssh/id_rsa"), - ) - - storage_parser = configure_subparsers.add_parser("storage") - storage_parser.add_argument("--file-dir", required=True) - storage_parser_subparsers = storage_parser.add_subparsers( - dest="storage_backend" - ) - - filesystem_storage_parser = storage_parser_subparsers.add_parser( - "filesystem" - ) - filesystem_storage_parser.add_argument("--vm-dir", required=True) - filesystem_storage_parser.add_argument("--image-dir", required=True) - - ceph_storage_parser = storage_parser_subparsers.add_parser("ceph") - ceph_storage_parser.add_argument("--ceph-vm-pool", required=True) - ceph_storage_parser.add_argument("--ceph-image-pool", required=True) - - def main(**kwargs): - subcommand = kwargs.pop("subcommand") + subcommand = kwargs.pop('subcommand') if not subcommand: - pass + arg_parser.print_help() else: update_config(subcommand, kwargs) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index bb318c3..c81fbbe 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -3,6 +3,7 @@ import os import pathlib import subprocess as sp import time +import argparse from uuid import uuid4 @@ -11,6 +12,9 @@ from uncloud.settings import settings from uncloud.shared import shared +arg_parser = argparse.ArgumentParser('filescanner', add_help=False) + + def sha512sum(file: str): """Use sha512sum utility to compute sha512 sum of arg:file diff --git a/uncloud/host/main.py b/uncloud/host/main.py index e469725..ec2ef4d 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -1,6 +1,7 @@ import argparse import multiprocessing as mp import time + from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType @@ -12,6 +13,9 @@ from os.path import join as join_path from . import virtualmachine, logger +arg_parser = argparse.ArgumentParser('host', add_help=False) +arg_parser.add_argument('--hostname', required=True) + def update_heartbeat(hostname): """Update Last HeartBeat Time for :param hostname: in etcd""" diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index fc77809..a43a36c 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -1,5 +1,6 @@ import json import os +import argparse import subprocess as sp from os.path import join as join_path @@ -8,6 +9,9 @@ from uncloud.shared import shared from uncloud.imagescanner import logger +arg_parser = argparse.ArgumentParser('imagescanner', add_help=False) + + def qemu_img_type(path): qemu_img_info_command = [ "qemu-img", diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index a59a998..e2199b8 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -1,4 +1,5 @@ import os +import argparse from flask import Flask, request from flask_restful import Resource, Api @@ -12,6 +13,9 @@ api = Api(app) app.logger.handlers.clear() +arg_parser = argparse.ArgumentParser('metadata', add_help=False) +arg_parser.add_argument('--port', '-p', default=80, help='By default bind to port 80') + @app.errorhandler(Exception) def handle_exception(e): diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 79b1edc..1ef6226 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -4,17 +4,16 @@ # 2. Introduce a status endpoint of the scheduler - # maybe expose a prometheus compatible output +import argparse + from uncloud.common.request import RequestEntry, RequestType from uncloud.shared import shared from uncloud.settings import settings -from .helper import ( - dead_host_mitigation, - dead_host_detection, - assign_host, - NoSuitableHostFound, -) +from .helper import (dead_host_mitigation, dead_host_detection, assign_host, NoSuitableHostFound) from . import logger +arg_parser = argparse.ArgumentParser('scheduler', add_help=False) + def main(debug=False): for request_iterator in [ diff --git a/uncloud/settings/__init__.py b/uncloud/settings/__init__.py index 629660e..f6da61c 100644 --- a/uncloud/settings/__init__.py +++ b/uncloud/settings/__init__.py @@ -3,6 +3,8 @@ import logging import sys import os +from datetime import datetime + from uncloud.common.etcd_wrapper import Etcd3Wrapper logger = logging.getLogger(__name__) @@ -29,10 +31,13 @@ class Settings(object): "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/") ) self.config_file = os.path.join(conf_dir, conf_name) - self.config_parser = CustomConfigParser(allow_no_value=True) self.config_key = config_key + # this is used to cache config from etcd for 1 minutes. Without this we + # would make a lot of requests to etcd which slows down everything. + self.last_config_update = datetime.fromtimestamp(0) + self.read_internal_values() try: self.config_parser.read(self.config_file) @@ -102,25 +107,22 @@ class Settings(object): def read_values_from_etcd(self): etcd_client = self.get_etcd_client() - config_from_etcd = etcd_client.get( - self.config_key, value_in_json=True - ) - if config_from_etcd: - self.config_parser.read_dict(config_from_etcd.value) - else: - raise KeyError( - "Key '{}' not found in etcd. Please configure uncloud.".format( - self.config_key - ) - ) + if (datetime.utcnow() - self.last_config_update).total_seconds() > 60: + config_from_etcd = etcd_client.get(self.config_key, value_in_json=True) + if config_from_etcd: + self.config_parser.read_dict(config_from_etcd.value) + self.last_config_update = datetime.utcnow() + else: + raise KeyError("Key '{}' not found in etcd. Please configure uncloud.".format(self.config_key)) def __getitem__(self, key): # Allow failing to read from etcd if we have # it locally - try: - self.read_values_from_etcd() - except KeyError as e: - pass + if key not in self.config_parser.sections(): + try: + self.read_values_from_etcd() + except KeyError as e: + pass return self.config_parser[key] From 344a957a3fe4bcc1f93b048672ce90ff204daf31 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 3 Jan 2020 18:42:20 +0500 Subject: [PATCH 076/163] Removed duplicate add_help from argument parsers in cli/image and cli/network --- uncloud/cli/image.py | 2 +- uncloud/cli/network.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/uncloud/cli/image.py b/uncloud/cli/image.py index 3db9577..641a00f 100644 --- a/uncloud/cli/image.py +++ b/uncloud/cli/image.py @@ -15,7 +15,7 @@ class ImageParser(BaseParser): p.add_argument('--image-store-name', default='image_store') def list(self, **kwargs): - self.subparser.add_parser('list', add_help=False, **kwargs) + self.subparser.add_parser('list', **kwargs) parser = ImageParser() diff --git a/uncloud/cli/network.py b/uncloud/cli/network.py index 33e41a9..55798bf 100644 --- a/uncloud/cli/network.py +++ b/uncloud/cli/network.py @@ -9,7 +9,7 @@ class NetworkParser(BaseParser): super().__init__('network') def create(self, **kwargs): - p = self.subparser.add_parser('create', add_help=False, parents=[get_otp_parser()], **kwargs) + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs) p.add_argument('--network-name', required=True) p.add_argument('--network-type', required=True, dest='type') p.add_argument('--user', action='store_true') From 180f6f4989133b4a27833d6f9eebc7cc26cf02ce Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 5 Jan 2020 17:21:26 +0500 Subject: [PATCH 077/163] No longer using xattrs as they don't work on tmpfs/rootfs --- uncloud/filescanner/main.py | 78 +++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index c81fbbe..e4d807c 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -4,6 +4,7 @@ import pathlib import subprocess as sp import time import argparse +import bitmath from uuid import uuid4 @@ -28,66 +29,59 @@ def sha512sum(file: str): if not isinstance(file, str): raise TypeError try: - output = sp.check_output(["sha512sum", file], stderr=sp.PIPE) + output = sp.check_output(['sha512sum', file], stderr=sp.PIPE) except sp.CalledProcessError as e: - error = e.stderr.decode("utf-8") - if "No such file or directory" in error: + error = e.stderr.decode('utf-8') + if 'No such file or directory' in error: raise FileNotFoundError from None else: - output = output.decode("utf-8").strip() - output = output.split(" ") + output = output.decode('utf-8').strip() + output = output.split(' ') return output[0] return None def track_file(file, base_dir): - file_id = uuid4() + file_path = file.relative_to(base_dir) # Get Username - owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] + try: + owner = file_path.parts[0] + except IndexError: + pass + else: + file_path = file_path.relative_to(owner) + creation_date = time.ctime(os.stat(file).st_ctime) - # Get Creation Date of File - # Here, we are assuming that ctime is creation time - # which is mostly not true. - creation_date = time.ctime(os.stat(file).st_ctime) + entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) + entry_value = { + 'filename': str(file_path), + 'owner': owner, + 'sha512sum': sha512sum(str(file)), + 'creation_date': creation_date, + 'size': str(bitmath.Byte(os.path.getsize(str(file))).to_MB()), + } - file_path = pathlib.Path(file).parts[-1] + logger.info('Tracking %s', file) - # Create Entry - entry_key = os.path.join( - settings["etcd"]["file_prefix"], str(file_id) - ) - entry_value = { - "filename": file_path, - "owner": owner, - "sha512sum": sha512sum(file), - "creation_date": creation_date, - "size": os.path.getsize(file), - } - - logger.info("Tracking %s", file) - - shared.etcd_client.put(entry_key, entry_value, value_in_json=True) - os.setxattr(file, "user.utracked", b"True") + shared.etcd_client.put(entry_key, entry_value, value_in_json=True) def main(debug=False): - base_dir = settings["storage"]["file_dir"] - + base_dir = pathlib.Path(settings['storage']['file_dir']) # Recursively Get All Files and Folder below BASE_DIR - files = glob.glob("{}/**".format(base_dir), recursive=True) + files = glob.glob('{}/**'.format(base_dir), recursive=True) + files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] - # Retain only Files - files = [file for file in files if os.path.isfile(file)] - - untracked_files = [] - for file in files: - try: - os.getxattr(file, "user.utracked") - except OSError: - track_file(file, base_dir) - untracked_files.append(file) + # Files that are already tracked + tracked_files = [ + pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) + for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) + ] + untracked_files = set(files) - set(tracked_files) + for file in untracked_files: + track_file(file, base_dir) -if __name__ == "__main__": +if __name__ == '__main__': main() From 6847a0d3232c2725852397f0e22dd8edcf7bfddd Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 5 Jan 2020 17:56:42 +0500 Subject: [PATCH 078/163] base dir reverted back to str path --- uncloud/filescanner/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index e4d807c..89d32d4 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -68,7 +68,7 @@ def track_file(file, base_dir): def main(debug=False): - base_dir = pathlib.Path(settings['storage']['file_dir']) + base_dir = settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] From 7fff280c7945e979f860e9383b481d037e9fd754 Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 5 Jan 2020 18:00:05 +0500 Subject: [PATCH 079/163] uncloud filescanner os.path.getsize expects str given Path instead --- uncloud/filescanner/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index 89d32d4..19c43ef 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -43,7 +43,7 @@ def sha512sum(file: str): def track_file(file, base_dir): file_path = file.relative_to(base_dir) - + file_str = str(file) # Get Username try: owner = file_path.parts[0] @@ -51,18 +51,18 @@ def track_file(file, base_dir): pass else: file_path = file_path.relative_to(owner) - creation_date = time.ctime(os.stat(file).st_ctime) + creation_date = time.ctime(os.stat(file_str).st_ctime) entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) entry_value = { 'filename': str(file_path), 'owner': owner, - 'sha512sum': sha512sum(str(file)), + 'sha512sum': sha512sum(file_str), 'creation_date': creation_date, - 'size': str(bitmath.Byte(os.path.getsize(str(file))).to_MB()), + 'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()), } - logger.info('Tracking %s', file) + logger.info('Tracking %s', file_str) shared.etcd_client.put(entry_key, entry_value, value_in_json=True) From 6f51ddbb3623e36dde7dc410cfe35fa7ea9adfbd Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 5 Jan 2020 18:31:48 +0500 Subject: [PATCH 080/163] renamed argument, and changed destination and make it required (uncloud.cli.image.create_image_from_file) --- uncloud/api/create_image_store.py | 12 ++++++------ uncloud/cli/image.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 73b92f1..9259be6 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -7,14 +7,14 @@ from uncloud.shared import shared from uncloud.settings import settings data = { - "is_public": True, - "type": "ceph", - "name": "images", - "description": "first ever public image-store", - "attributes": {"list": [], "key": [], "pool": "images"}, + 'is_public': True, + 'type': 'ceph', + 'name': 'images', + 'description': 'first ever public image-store', + 'attributes': {'list': [], 'key': [], 'pool': 'images'}, } shared.etcd_client.put( - os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex), + os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data), ) diff --git a/uncloud/cli/image.py b/uncloud/cli/image.py index 641a00f..2f59c32 100644 --- a/uncloud/cli/image.py +++ b/uncloud/cli/image.py @@ -12,7 +12,7 @@ class ImageParser(BaseParser): p = self.subparser.add_parser('create', **kwargs) p.add_argument('--name', required=True) p.add_argument('--uuid', required=True) - p.add_argument('--image-store-name', default='image_store') + p.add_argument('--image-store', required=True, dest='image_store') def list(self, **kwargs): self.subparser.add_parser('list', **kwargs) From b7f3ba1a3488d9212e500fade9b6186082626cf7 Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 5 Jan 2020 19:46:38 +0500 Subject: [PATCH 081/163] remove cache=none from QEMU args as it is not supported on tmpfs/rootfs --- uncloud/api/main.py | 2 +- uncloud/host/virtualmachine.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 47e7003..84eb28a 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -41,7 +41,7 @@ def handle_exception(e): class CreateVM(Resource): - '''API Request to Handle Creation of VM''' + """API Request to Handle Creation of VM""" @staticmethod def post(): diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index 0bd20bf..cbb3bbe 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -42,7 +42,7 @@ class VM: def get_qemu_args(self): command = ( - "-drive file={file},format=raw,if=virtio,cache=none" + "-drive file={file},format=raw,if=virtio" " -device virtio-rng-pci" " -m {memory} -smp cores={cores},threads={threads}" " -name {owner}_{name}" From ec40d6b1e0b2c00ad36c9ba4666f7951e55bbbff Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 5 Jan 2020 20:20:00 +0500 Subject: [PATCH 082/163] don't suppress error when changing permissions in uncloud vmm --- uncloud/host/main.py | 36 +++++++++++------------ uncloud/settings/__init__.py | 56 +++++++++++++++++------------------- uncloud/vmm/__init__.py | 16 +++-------- 3 files changed, 49 insertions(+), 59 deletions(-) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index ec2ef4d..b7c8b1c 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -33,10 +33,10 @@ def maintenance(host): vmm = VMM() running_vms = vmm.discover() for vm_uuid in running_vms: - if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running": + if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], vm_uuid) + join_path(settings['etcd']['vm_prefix'], vm_uuid) ) vm.status = VMStatus.running vm.vnc_socket = vmm.get_vnc(vm_uuid) @@ -51,13 +51,13 @@ def main(hostname, debug=False): # Does not yet exist, create it if not host: host_key = join_path( - settings["etcd"]["host_prefix"], uuid4().hex + settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { - "specs": "", - "hostname": hostname, - "status": "DEAD", - "last_heartbeat": "", + 'specs': '', + 'hostname': hostname, + 'status': 'DEAD', + 'last_heartbeat': '', } shared.etcd_client.put( host_key, host_entry, value_in_json=True @@ -70,25 +70,25 @@ def main(hostname, debug=False): heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() except Exception as e: - raise Exception("uncloud-host heartbeat updating mechanism is not working") from e + raise Exception('uncloud-host heartbeat updating mechanism is not working') from e for events_iterator in [ - shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True), - shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True) + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True), + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], timeout=10, value_in_json=True) ]: for request_event in events_iterator: request_event = RequestEntry(request_event) - if request_event.type == "TIMEOUT": + if request_event.type == 'TIMEOUT': maintenance(host.key) elif request_event.hostname == host.key: - logger.debug("VM Request: %s on Host %s", request_event, host.hostname) + logger.debug('VM Request: %s on Host %s', request_event, host.hostname) shared.request_pool.client.client.delete(request_event.key) vm_entry = shared.etcd_client.get( - join_path(settings["etcd"]["vm_prefix"], request_event.uuid) + join_path(settings['etcd']['vm_prefix'], request_event.uuid) ) - logger.debug("VM hostname: {}".format(vm_entry.value)) + logger.debug('VM hostname: {}'.format(vm_entry.value)) vm = virtualmachine.VM(vm_entry) if request_event.type == RequestType.StartVM: vm.start() @@ -110,14 +110,14 @@ def main(hostname, debug=False): destination_sock_path=request_event.destination_sock_path, ) else: - logger.error("Host %s not found!", request_event.destination_host_key) + logger.error('Host %s not found!', request_event.destination_host_key) -if __name__ == "__main__": +if __name__ == '__main__': argparser = argparse.ArgumentParser() argparser.add_argument( - "hostname", help="Name of this host. e.g uncloud1.ungleich.ch" + 'hostname', help='Name of this host. e.g uncloud1.ungleich.ch' ) args = argparser.parse_args() - mp.set_start_method("spawn") + mp.set_start_method('spawn') main(args.hostname) diff --git a/uncloud/settings/__init__.py b/uncloud/settings/__init__.py index f6da61c..0654b9b 100644 --- a/uncloud/settings/__init__.py +++ b/uncloud/settings/__init__.py @@ -16,7 +16,7 @@ class CustomConfigParser(configparser.RawConfigParser): result = super().__getitem__(key) except KeyError as err: raise KeyError( - "Key '{}' not found in configuration. Make sure you configure uncloud.".format( + 'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format( key ) ) from err @@ -25,10 +25,10 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): - def __init__(self, config_key="/uncloud/config/"): - conf_name = "uncloud.conf" + def __init__(self, config_key='/uncloud/config/'): + conf_name = 'uncloud.conf' conf_dir = os.environ.get( - "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/") + 'UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/') ) self.config_file = os.path.join(conf_dir, conf_name) self.config_parser = CustomConfigParser(allow_no_value=True) @@ -42,23 +42,21 @@ class Settings(object): try: self.config_parser.read(self.config_file) except Exception as err: - logger.error("%s", err) + logger.error('%s', err) def get_etcd_client(self): args = tuple() try: kwargs = { - "host": self.config_parser.get("etcd", "url"), - "port": self.config_parser.get("etcd", "port"), - "ca_cert": self.config_parser.get("etcd", "ca_cert"), - "cert_cert": self.config_parser.get( - "etcd", "cert_cert" - ), - "cert_key": self.config_parser.get("etcd", "cert_key"), + 'host': self.config_parser.get('etcd', 'url'), + 'port': self.config_parser.get('etcd', 'port'), + 'ca_cert': self.config_parser.get('etcd', 'ca_cert'), + 'cert_cert': self.config_parser.get('etcd', 'cert_cert'), + 'cert_key': self.config_parser.get('etcd', 'cert_key'), } except configparser.Error as err: raise configparser.Error( - "{} in config file {}".format( + '{} in config file {}'.format( err.message, self.config_file ) ) from err @@ -67,8 +65,8 @@ class Settings(object): wrapper = Etcd3Wrapper(*args, **kwargs) except Exception as err: logger.error( - "etcd connection not successfull. Please check your config file." - "\nDetails: %s\netcd connection parameters: %s", + 'etcd connection not successfull. Please check your config file.' + '\nDetails: %s\netcd connection parameters: %s', err, kwargs, ) @@ -79,15 +77,15 @@ class Settings(object): def read_internal_values(self): self.config_parser.read_dict( { - "etcd": { - "file_prefix": "/files/", - "host_prefix": "/hosts/", - "image_prefix": "/images/", - "image_store_prefix": "/imagestore/", - "network_prefix": "/networks/", - "request_prefix": "/requests/", - "user_prefix": "/users/", - "vm_prefix": "/vms/", + 'etcd': { + 'file_prefix': '/files/', + 'host_prefix': '/hosts/', + 'image_prefix': '/images/', + 'image_store_prefix': '/imagestore/', + 'network_prefix': '/networks/', + 'request_prefix': '/requests/', + 'user_prefix': '/users/', + 'vm_prefix': '/vms/', } } ) @@ -95,15 +93,15 @@ class Settings(object): def read_config_file_values(self, config_file): try: # Trying to read configuration file - with open(config_file, "r") as config_file_handle: + with open(config_file, 'r') as config_file_handle: self.config_parser.read_file(config_file_handle) except FileNotFoundError: sys.exit( - "Configuration file {} not found!".format(config_file) + 'Configuration file {} not found!'.format(config_file) ) except Exception as err: logger.exception(err) - sys.exit("Error occurred while reading configuration file") + sys.exit('Error occurred while reading configuration file') def read_values_from_etcd(self): etcd_client = self.get_etcd_client() @@ -113,7 +111,7 @@ class Settings(object): self.config_parser.read_dict(config_from_etcd.value) self.last_config_update = datetime.utcnow() else: - raise KeyError("Key '{}' not found in etcd. Please configure uncloud.".format(self.config_key)) + raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key)) def __getitem__(self, key): # Allow failing to read from etcd if we have @@ -121,7 +119,7 @@ class Settings(object): if key not in self.config_parser.sections(): try: self.read_values_from_etcd() - except KeyError as e: + except KeyError: pass return self.config_parser[key] diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 6cdd938..4c893f6 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -190,18 +190,10 @@ class VMM: err.stderr.decode("utf-8"), ) else: - with suppress(sp.CalledProcessError): - sp.check_output( - [ - "sudo", - "-p", - "Enter password to correct permission for uncloud-vmm's directory", - "chmod", - "-R", - "o=rwx,g=rwx", - self.vmm_backend, - ] - ) + sp.check_output( + ["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory", + "chmod", "-R", "o=rwx,g=rwx", self.vmm_backend] + ) # TODO: Find some good way to check whether the virtual machine is up and # running without relying on non-guarenteed ways. From ef0f13534a12ad492782c1baa583b6bab934f223 Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 5 Jan 2020 21:59:24 +0500 Subject: [PATCH 083/163] bug fixed that add extra space in QEMU command when there is no network to be attached --- uncloud/host/virtualmachine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index cbb3bbe..b9a9e36 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -153,7 +153,10 @@ class VM: ) ) - return command.split(" ") + if command: + command = command.split(' ') + + return command def delete_network_dev(self): try: From 388127bd11e45eb9eda1be1279030ca4d2901ae6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 5 Jan 2020 18:32:14 +0100 Subject: [PATCH 084/163] [hack] add scripts to start VM --- uncloud/hack/uncloud-hack-init-host | 26 ++++++++++++++++++++++++++ uncloud/hack/uncloud-run-vm | 21 +++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 uncloud/hack/uncloud-hack-init-host create mode 100644 uncloud/hack/uncloud-run-vm diff --git a/uncloud/hack/uncloud-hack-init-host b/uncloud/hack/uncloud-hack-init-host new file mode 100644 index 0000000..787ff80 --- /dev/null +++ b/uncloud/hack/uncloud-hack-init-host @@ -0,0 +1,26 @@ +id=100 +rawdev=eth0 + +# create vxlan +ip -6 link add vxlan${id} type vxlan \ + id ${id} \ + dstport 4789 \ + group ff05::${id} \ + dev ${rawdev} \ + ttl 5 + +ip link set vxlan${id} up + +# create bridge +ip link set vxlan${id} up +ip link set br${id} up + +# Add vxlan into bridge +ip link set vxlan${id} master br${id} + + +# useradd -m uncloud +# [18:05] tablett.place10:~# id uncloud +# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu) +# apk add qemu-system-x86_64 +# also needs group netdev diff --git a/uncloud/hack/uncloud-run-vm b/uncloud/hack/uncloud-run-vm new file mode 100644 index 0000000..1af2037 --- /dev/null +++ b/uncloud/hack/uncloud-run-vm @@ -0,0 +1,21 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo $0 vmid + exit 1 +fi + +id=$1; shift + +memory=512 +macaddress=02:00:b9:cb:70:${id} +netname=net${id}-1 + +qemu-system-x86_64 \ + -name uncloud-${id} \ + -accel kvm \ + -m ${memory} \ + -smp 2,sockets=2,cores=1,threads=1 \ + -device virtio-net-pci,netdev=net0,mac=$macaddress \ + -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \ + -vnc [::]:5900 From 6086fec633d6f368f574215631e8449c374b78dd Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 6 Jan 2020 12:25:59 +0500 Subject: [PATCH 085/163] move settings under uncloud.common --- uncloud/api/common_fields.py | 2 +- uncloud/api/create_image_store.py | 2 +- uncloud/api/helper.py | 2 +- uncloud/api/main.py | 2 +- uncloud/api/schemas.py | 2 +- uncloud/cli/helper.py | 2 +- uncloud/common/network.py | 2 -- uncloud/{settings/__init__.py => common/settings.py} | 0 uncloud/common/storage_handlers.py | 2 +- uncloud/configure/main.py | 2 +- uncloud/docs/README.md | 12 ++++++++++++ uncloud/filescanner/main.py | 2 +- uncloud/host/main.py | 2 +- uncloud/host/virtualmachine.py | 2 +- uncloud/imagescanner/main.py | 2 +- uncloud/metadata/main.py | 2 +- uncloud/scheduler/helper.py | 2 +- uncloud/scheduler/main.py | 2 +- uncloud/shared/__init__.py | 2 +- 19 files changed, 28 insertions(+), 18 deletions(-) rename uncloud/{settings/__init__.py => common/settings.py} (100%) create mode 100644 uncloud/docs/README.md diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py index 8bcf777..adf7cdc 100755 --- a/uncloud/api/common_fields.py +++ b/uncloud/api/common_fields.py @@ -1,7 +1,7 @@ import os from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.settings import settings class Optional: diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 9259be6..075f26f 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -4,7 +4,7 @@ import os from uuid import uuid4 from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.settings import settings data = { 'is_public': True, diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index c806814..0e5fa19 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -8,7 +8,7 @@ import requests from pyotp import TOTP from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.settings import settings logger = logging.getLogger(__name__) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 84eb28a..401c11f 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -13,7 +13,7 @@ from werkzeug.exceptions import HTTPException from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.shared import shared from . import schemas from .helper import generate_mac, mac2ipv6 diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index 8e06e8d..f606803 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -22,7 +22,7 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py index bdcce78..0495fac 100644 --- a/uncloud/cli/helper.py +++ b/uncloud/cli/helper.py @@ -5,7 +5,7 @@ import binascii from pyotp import TOTP from os.path import join as join_path -from uncloud.settings import settings +from uncloud.common.settings import settings def get_otp_parser(): diff --git a/uncloud/common/network.py b/uncloud/common/network.py index adba108..32f6951 100644 --- a/uncloud/common/network.py +++ b/uncloud/common/network.py @@ -1,8 +1,6 @@ import subprocess as sp import random import logging -import socket -from contextlib import closing logger = logging.getLogger(__name__) diff --git a/uncloud/settings/__init__.py b/uncloud/common/settings.py similarity index 100% rename from uncloud/settings/__init__.py rename to uncloud/common/settings.py diff --git a/uncloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py index 06751c4..6f9b29e 100644 --- a/uncloud/common/storage_handlers.py +++ b/uncloud/common/storage_handlers.py @@ -7,7 +7,7 @@ from abc import ABC from . import logger from os.path import join as join_path -from uncloud.settings import settings as config +from uncloud.common.settings import settings as config class ImageStorageHandler(ABC): diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index f89a30c..64b40c0 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,7 +1,7 @@ import os import argparse -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.shared import shared arg_parser = argparse.ArgumentParser('configure', add_help=False) diff --git a/uncloud/docs/README.md b/uncloud/docs/README.md new file mode 100644 index 0000000..a5afbaa --- /dev/null +++ b/uncloud/docs/README.md @@ -0,0 +1,12 @@ +# uncloud docs + +## Requirements +1. Python3 +2. Sphinx + +## Usage +Run `make build` to build docs. + +Run `make clean` to remove build directory. + +Run `make publish` to push build dir to https://ungleich.ch/ucloud/ \ No newline at end of file diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index 19c43ef..9d2b2f6 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -9,7 +9,7 @@ import bitmath from uuid import uuid4 from . import logger -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.shared import shared diff --git a/uncloud/host/main.py b/uncloud/host/main.py index b7c8b1c..bed068b 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -6,7 +6,7 @@ from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.common.vm import VMStatus from uncloud.vmm import VMM from os.path import join as join_path diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index b9a9e36..a37dee4 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -17,7 +17,7 @@ from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.vmm import VMM from marshmallow import ValidationError diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index a43a36c..cb13ac7 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -4,7 +4,7 @@ import argparse import subprocess as sp from os.path import join as join_path -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.shared import shared from uncloud.imagescanner import logger diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index e2199b8..d20122e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -5,7 +5,7 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.shared import shared app = Flask(__name__) diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index 7edf623..a7fec15 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -7,7 +7,7 @@ from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.settings import settings def accumulated_specs(vms_specs): diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 1ef6226..5143537 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -8,7 +8,7 @@ import argparse from uncloud.common.request import RequestEntry, RequestType from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.settings import settings from .helper import (dead_host_mitigation, dead_host_detection, assign_host, NoSuitableHostFound) from . import logger diff --git a/uncloud/shared/__init__.py b/uncloud/shared/__init__.py index db2093f..918dd0c 100644 --- a/uncloud/shared/__init__.py +++ b/uncloud/shared/__init__.py @@ -1,4 +1,4 @@ -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.common.vm import VmPool from uncloud.common.host import HostPool from uncloud.common.request import RequestPool From 48cc37c438c015ad7b09e58349e8fad75dd0f49f Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 7 Jan 2020 17:57:44 +0500 Subject: [PATCH 086/163] add hostname to file entry (uncloud filescanner) --- uncloud/filescanner/main.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index 9d2b2f6..cb5f2b7 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -14,6 +14,7 @@ from uncloud.shared import shared arg_parser = argparse.ArgumentParser('filescanner', add_help=False) +arg_parser.add_argument('--hostname', required=True) def sha512sum(file: str): @@ -41,7 +42,7 @@ def sha512sum(file: str): return None -def track_file(file, base_dir): +def track_file(file, base_dir, host): file_path = file.relative_to(base_dir) file_str = str(file) # Get Username @@ -60,6 +61,7 @@ def track_file(file, base_dir): 'sha512sum': sha512sum(file_str), 'creation_date': creation_date, 'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()), + 'host': host } logger.info('Tracking %s', file_str) @@ -67,7 +69,7 @@ def track_file(file, base_dir): shared.etcd_client.put(entry_key, entry_value, value_in_json=True) -def main(debug=False): +def main(hostname, debug=False): base_dir = settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) @@ -77,11 +79,8 @@ def main(debug=False): tracked_files = [ pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) + if f.value['host'] == hostname ] untracked_files = set(files) - set(tracked_files) for file in untracked_files: - track_file(file, base_dir) - - -if __name__ == '__main__': - main() + track_file(file, base_dir, hostname) From b4292615de88018abfb19383eec3b37b680d600d Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 7 Jan 2020 18:27:22 +0500 Subject: [PATCH 087/163] Display more info about tracked files to user e.g creation_date, host on which it is stored, size etc --- uncloud/api/main.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 401c11f..50bc201 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -289,18 +289,16 @@ class ListUserFiles(Resource): settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] - user_files = list( - filter( - lambda f: f.value['owner'] == data['name'], files - ) - ) + user_files = [f for f in files if f.value['owner'] == data['name']] for file in user_files: - return_files.append( - { - 'filename': file.value['filename'], - 'uuid': file.key.split('/')[-1], - } - ) + file_uuid = file.key.split('/')[-1] + file = file.value + file['uuid'] = file_uuid + + file.pop('sha512sum', None) + file.pop('owner', None) + + return_files.append(file) return {'message': return_files}, 200 else: return validator.get_errors(), 400 From 6046015c3d56067d0eafa3848c59ea333e2a50e7 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 7 Jan 2020 20:26:10 +0500 Subject: [PATCH 088/163] Add base prefix option for uncloud so that we can run independent instance on uncloud --- conf/uncloud.conf | 2 ++ uncloud/common/settings.py | 38 +++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/conf/uncloud.conf b/conf/uncloud.conf index 9d4358d..9995696 100644 --- a/conf/uncloud.conf +++ b/conf/uncloud.conf @@ -1,6 +1,7 @@ [etcd] url = localhost port = 2379 +prefix = / ca_cert cert_cert cert_key @@ -9,3 +10,4 @@ cert_key name = replace_me realm = replace_me seed = replace_me +api_server = http://localhost:5000 \ No newline at end of file diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 0654b9b..7004055 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -4,8 +4,8 @@ import sys import os from datetime import datetime - from uncloud.common.etcd_wrapper import Etcd3Wrapper +from os.path import join as join_path logger = logging.getLogger(__name__) @@ -25,25 +25,28 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): - def __init__(self, config_key='/uncloud/config/'): + def __init__(self): conf_name = 'uncloud.conf' - conf_dir = os.environ.get( - 'UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/') - ) - self.config_file = os.path.join(conf_dir, conf_name) - self.config_parser = CustomConfigParser(allow_no_value=True) - self.config_key = config_key + conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/')) + self.config_file = join_path(conf_dir, conf_name) # this is used to cache config from etcd for 1 minutes. Without this we # would make a lot of requests to etcd which slows down everything. self.last_config_update = datetime.fromtimestamp(0) - self.read_internal_values() + self.config_parser = CustomConfigParser(allow_no_value=True) + self.config_parser.add_section('etcd') + self.config_parser.set('etcd', 'prefix', '/') + + self.config_key = join_path(self['etcd']['prefix'], '/uncloud/config/') + try: self.config_parser.read(self.config_file) except Exception as err: logger.error('%s', err) + self.read_internal_values() + def get_etcd_client(self): args = tuple() try: @@ -75,17 +78,18 @@ class Settings(object): return wrapper def read_internal_values(self): + prefix = self['etcd']['prefix'] self.config_parser.read_dict( { 'etcd': { - 'file_prefix': '/files/', - 'host_prefix': '/hosts/', - 'image_prefix': '/images/', - 'image_store_prefix': '/imagestore/', - 'network_prefix': '/networks/', - 'request_prefix': '/requests/', - 'user_prefix': '/users/', - 'vm_prefix': '/vms/', + 'file_prefix': join_path(prefix, '/files/'), + 'host_prefix': join_path(prefix, '/hosts/'), + 'image_prefix': join_path(prefix, '/images/'), + 'image_store_prefix': join_path(prefix, '/imagestore/'), + 'network_prefix': join_path(prefix, '/networks/'), + 'request_prefix': join_path(prefix, '/requests/'), + 'user_prefix': join_path(prefix, '/users/'), + 'vm_prefix': join_path(prefix, '/vms/'), } } ) From 5a646aeac951ad28b146769275a55a14576f161c Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 7 Jan 2020 21:45:11 +0500 Subject: [PATCH 089/163] prefix is renamed to base_prefix, uncloud now respects base_prefix and put things under it --- conf/uncloud.conf | 2 +- uncloud/api/main.py | 4 ++-- uncloud/common/settings.py | 28 ++++++++++++++++------------ uncloud/configure/main.py | 8 ++------ uncloud/metadata/main.py | 27 --------------------------- 5 files changed, 21 insertions(+), 48 deletions(-) diff --git a/conf/uncloud.conf b/conf/uncloud.conf index 9995696..6a1b500 100644 --- a/conf/uncloud.conf +++ b/conf/uncloud.conf @@ -1,7 +1,7 @@ [etcd] url = localhost port = 2379 -prefix = / +base_prefix = / ca_cert cert_cert cert_key diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 50bc201..d8beb49 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -59,7 +59,7 @@ class CreateVM(Resource): macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, '/v1/counter/tap' + shared.etcd_client, settings['etcd']['counter']['tap'] ) for _ in range(len(data['network'])) ] @@ -470,7 +470,7 @@ class CreateNetwork(Resource): network_entry = { 'id': counters.increment_etcd_counter( - shared.etcd_client, '/v1/counter/vxlan' + shared.etcd_client, settings['etcd']['counter']['vxlan'] ), 'type': data['type'], } diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 7004055..9db4afe 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -36,15 +36,15 @@ class Settings(object): self.config_parser = CustomConfigParser(allow_no_value=True) self.config_parser.add_section('etcd') - self.config_parser.set('etcd', 'prefix', '/') - - self.config_key = join_path(self['etcd']['prefix'], '/uncloud/config/') + self.config_parser.set('etcd', 'base_prefix', '/') try: self.config_parser.read(self.config_file) except Exception as err: logger.error('%s', err) + self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') + self.read_internal_values() def get_etcd_client(self): @@ -78,18 +78,22 @@ class Settings(object): return wrapper def read_internal_values(self): - prefix = self['etcd']['prefix'] + base_prefix = self['etcd']['base_prefix'] self.config_parser.read_dict( { 'etcd': { - 'file_prefix': join_path(prefix, '/files/'), - 'host_prefix': join_path(prefix, '/hosts/'), - 'image_prefix': join_path(prefix, '/images/'), - 'image_store_prefix': join_path(prefix, '/imagestore/'), - 'network_prefix': join_path(prefix, '/networks/'), - 'request_prefix': join_path(prefix, '/requests/'), - 'user_prefix': join_path(prefix, '/users/'), - 'vm_prefix': join_path(prefix, '/vms/'), + 'file_prefix': join_path(base_prefix, 'files/'), + 'host_prefix': join_path(base_prefix, 'hosts/'), + 'image_prefix': join_path(base_prefix, 'images/'), + 'image_store_prefix': join_path(base_prefix, 'imagestore/'), + 'network_prefix': join_path(base_prefix, 'networks/'), + 'request_prefix': join_path(base_prefix, 'requests/'), + 'user_prefix': join_path(base_prefix, 'users/'), + 'vm_prefix': join_path(base_prefix, 'vms/'), + 'counter': { + 'vxlan': join_path(base_prefix, 'counters/vxlan'), + 'tap': join_path(base_prefix, 'counters/tap') + } } } ) diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index 64b40c0..f3e9717 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -40,18 +40,14 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True) def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get( - settings.config_key, value_in_json=True - ) + uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - shared.etcd_client.put( - settings.config_key, uncloud_config, value_in_json=True - ) + shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) def main(**kwargs): diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index d20122e..03469a5 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -84,33 +84,6 @@ class Root(Resource): data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys return data.value["metadata"], 200 - @staticmethod - def post(): - return {"message": "Previous Implementation is deprecated."} - # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True) - # print(data) - # if data: - # for k in request.json: - # if k not in data.value: - # data.value[k] = request.json[k] - # if k.endswith("-list"): - # data.value[k] = [request.json[k]] - # else: - # if k.endswith("-list"): - # data.value[k].append(request.json[k]) - # else: - # data.value[k] = request.json[k] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data.value, value_in_json=True) - # else: - # data = {} - # for k in request.json: - # data[k] = request.json[k] - # if k.endswith("-list"): - # data[k] = [request.json[k]] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data, value_in_json=True) - api.add_resource(Root, "/") From f8f790e7fcdc8ee2787810d3b65d23f9a18a11a1 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 7 Jan 2020 22:18:13 +0500 Subject: [PATCH 090/163] nested dict doesn't play well with configparser --- uncloud/api/main.py | 4 ++-- uncloud/common/settings.py | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index d8beb49..e8e85fb 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -59,7 +59,7 @@ class CreateVM(Resource): macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['counter']['tap'] + shared.etcd_client, settings['etcd']['tap_counter'] ) for _ in range(len(data['network'])) ] @@ -470,7 +470,7 @@ class CreateNetwork(Resource): network_entry = { 'id': counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['counter']['vxlan'] + shared.etcd_client, settings['etcd']['vxlan_counter'] ), 'type': data['type'], } diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 9db4afe..47ad5a7 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -90,10 +90,8 @@ class Settings(object): 'request_prefix': join_path(base_prefix, 'requests/'), 'user_prefix': join_path(base_prefix, 'users/'), 'vm_prefix': join_path(base_prefix, 'vms/'), - 'counter': { - 'vxlan': join_path(base_prefix, 'counters/vxlan'), - 'tap': join_path(base_prefix, 'counters/tap') - } + 'vxlan_counter': join_path(base_prefix, 'counters/vxlan'), + 'tap_counter': join_path(base_prefix, 'counters/tap') } } ) @@ -129,7 +127,6 @@ class Settings(object): self.read_values_from_etcd() except KeyError: pass - return self.config_parser[key] From 48efcdf08cf620e31489df840d815e7369a41dfa Mon Sep 17 00:00:00 2001 From: meow Date: Thu, 9 Jan 2020 00:40:05 +0500 Subject: [PATCH 091/163] 1. mp.set_start_method('spawn') commented out from scripts/uncloud 2. uncloud.shared moved under uncloud.common 3. Refactoring in etcd_wrapper e.g timeout mechanism removed and few other things 4. uncloud-{scheduler,host} now better handle etcd events in their block state (waiting for requests to come) --- scripts/uncloud | 2 +- uncloud/api/common_fields.py | 2 +- uncloud/api/create_image_store.py | 2 +- uncloud/api/helper.py | 3 +- uncloud/api/main.py | 3 +- uncloud/api/schemas.py | 2 +- uncloud/common/etcd_wrapper.py | 113 +++++++----------- uncloud/common/request.py | 11 +- .../{shared/__init__.py => common/shared.py} | 0 uncloud/configure/main.py | 2 +- uncloud/filescanner/main.py | 3 +- uncloud/host/main.py | 82 ++++++------- uncloud/host/virtualmachine.py | 2 +- uncloud/imagescanner/main.py | 2 +- uncloud/metadata/main.py | 2 +- uncloud/scheduler/helper.py | 2 +- uncloud/scheduler/main.py | 76 ++++++------ 17 files changed, 136 insertions(+), 173 deletions(-) rename uncloud/{shared/__init__.py => common/shared.py} (100%) diff --git a/scripts/uncloud b/scripts/uncloud index 968ace6..1ca9c68 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -45,7 +45,7 @@ if __name__ == '__main__': # i.e inheriting few things from parent process etcd3 module # errors out, so the following command configure multiprocessing # module to not inherit anything from parent. - mp.set_start_method('spawn') + # mp.set_start_method('spawn') arguments = vars(args) try: name = arguments.pop('command') diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py index adf7cdc..d1fcb64 100755 --- a/uncloud/api/common_fields.py +++ b/uncloud/api/common_fields.py @@ -1,6 +1,6 @@ import os -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.common.settings import settings diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 075f26f..1040e97 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -3,7 +3,7 @@ import os from uuid import uuid4 -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.common.settings import settings data = { diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index 0e5fa19..0805280 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -1,13 +1,12 @@ import binascii import ipaddress import random -import subprocess as sp import logging import requests from pyotp import TOTP -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.common.settings import settings logger = logging.getLogger(__name__) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index e8e85fb..2d8d035 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -10,11 +10,12 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException +from uncloud.common.shared import shared + from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.settings import settings -from uncloud.shared import shared from . import schemas from .helper import generate_mac, mac2ipv6 from uncloud import UncloudException diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index f606803..e4de9a8 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -21,7 +21,7 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.common.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index 6a979ba..fe768ac 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -1,24 +1,21 @@ import etcd3 import json -import queue -import copy -from uncloud import UncloudException -from collections import namedtuple from functools import wraps -from . import logger - -PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) +from uncloud import UncloudException +from uncloud.common import logger class EtcdEntry: - # key: str - # value: str - - def __init__(self, meta, value, value_in_json=False): - self.key = meta.key.decode("utf-8") - self.value = value.decode("utf-8") + def __init__(self, meta_or_key, value, value_in_json=False): + if hasattr(meta_or_key, 'key'): + # if meta has attr 'key' then get it + self.key = meta_or_key.key.decode('utf-8') + else: + # otherwise meta is the 'key' + self.key = meta_or_key + self.value = value.decode('utf-8') if value_in_json: self.value = json.loads(self.value) @@ -29,18 +26,12 @@ def readable_errors(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except etcd3.exceptions.ConnectionFailedError as err: - raise UncloudException( - "Cannot connect to etcd: is etcd running as configured in uncloud.conf?" - ) + except etcd3.exceptions.ConnectionFailedError: + raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') except etcd3.exceptions.ConnectionTimeoutError as err: - raise etcd3.exceptions.ConnectionTimeoutError( - "etcd connection timeout." - ) from err + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err except Exception: - logger.exception( - "Some etcd error occured. See syslog for details." - ) + logger.exception('Some etcd error occured. See syslog for details.') return wrapper @@ -64,55 +55,39 @@ class Etcd3Wrapper: _value = json.dumps(_value) if not isinstance(_key, str): - _key = _key.decode("utf-8") + _key = _key.decode('utf-8') return self.client.put(_key, _value, **kwargs) @readable_errors - def get_prefix(self, *args, value_in_json=False, **kwargs): - r = self.client.get_prefix(*args, **kwargs) - for entry in r: - e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) - if e.value: - yield e + def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): + try: + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) + except Exception as err: + if raise_exception: + raise Exception('Exception in etcd_wrapper.get_prefix') from err + else: + logger.exception('Error in etcd_wrapper') + return iter([]) @readable_errors - def watch_prefix(self, key, timeout=0, value_in_json=False): - timeout_event = EtcdEntry( - PseudoEtcdMeta(key=b"TIMEOUT"), - value=str.encode( - json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"}) - ), - value_in_json=value_in_json, - ) - - event_queue = queue.Queue() - - def add_event_to_queue(event): - if hasattr(event, "events"): - for e in event.events: - if e.value: - event_queue.put( - EtcdEntry( - e, e.value, value_in_json=value_in_json - ) - ) - - self.client.add_watch_prefix_callback(key, add_event_to_queue) - - while True: - try: - while True: - v = event_queue.get(timeout=timeout) - yield v - except queue.Empty: - event_queue.put(copy.deepcopy(timeout_event)) - - -class PsuedoEtcdEntry(EtcdEntry): - def __init__(self, key, value, value_in_json=False): - super().__init__( - PseudoEtcdMeta(key=key.encode("utf-8")), - value, - value_in_json=value_in_json, - ) + def watch_prefix(self, key, raise_exception=True, value_in_json=False): + try: + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = e._event + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) + except Exception as err: + if raise_exception: + raise Exception('Exception in etcd_wrapper.get_prefix') from err + else: + logger.exception('Error in etcd_wrapper.watch_prefix') + try: + cancel() + except Exception: + pass + return iter([]) diff --git a/uncloud/common/request.py b/uncloud/common/request.py index a8c2d0a..cb0add5 100644 --- a/uncloud/common/request.py +++ b/uncloud/common/request.py @@ -2,8 +2,8 @@ import json from os.path import join from uuid import uuid4 -from .etcd_wrapper import PsuedoEtcdEntry -from .classes import SpecificEtcdEntryBase +from uncloud.common.etcd_wrapper import EtcdEntry +from uncloud.common.classes import SpecificEtcdEntryBase class RequestType: @@ -29,11 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase): @classmethod def from_scratch(cls, request_prefix, **kwargs): - e = PsuedoEtcdEntry( - join(request_prefix, uuid4().hex), - value=json.dumps(kwargs).encode("utf-8"), - value_in_json=True, - ) + e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex), + value=json.dumps(kwargs).encode('utf-8'), value_in_json=True) return cls(e) diff --git a/uncloud/shared/__init__.py b/uncloud/common/shared.py similarity index 100% rename from uncloud/shared/__init__.py rename to uncloud/common/shared.py diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index f3e9717..e190460 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -2,7 +2,7 @@ import os import argparse from uncloud.common.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('configure', add_help=False) configure_subparsers = arg_parser.add_subparsers(dest='subcommand') diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index cb5f2b7..314481f 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -10,8 +10,7 @@ from uuid import uuid4 from . import logger from uncloud.common.settings import settings -from uncloud.shared import shared - +from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('filescanner', add_help=False) arg_parser.add_argument('--hostname', required=True) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index bed068b..695e3d1 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -5,7 +5,7 @@ import time from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.common.settings import settings from uncloud.common.vm import VMStatus from uncloud.vmm import VMM @@ -72,52 +72,52 @@ def main(hostname, debug=False): except Exception as e: raise Exception('uncloud-host heartbeat updating mechanism is not working') from e - for events_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], timeout=10, value_in_json=True) - ]: - for request_event in events_iterator: - request_event = RequestEntry(request_event) + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for events_iterator in [ + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False) + ]: + for request_event in events_iterator: + request_event = RequestEntry(request_event) - if request_event.type == 'TIMEOUT': maintenance(host.key) - elif request_event.hostname == host.key: - logger.debug('VM Request: %s on Host %s', request_event, host.hostname) - shared.request_pool.client.client.delete(request_event.key) - vm_entry = shared.etcd_client.get( - join_path(settings['etcd']['vm_prefix'], request_event.uuid) - ) - logger.debug('VM hostname: {}'.format(vm_entry.value)) - vm = virtualmachine.VM(vm_entry) - if request_event.type == RequestType.StartVM: - vm.start() + if request_event.hostname == host.key: + logger.debug('VM Request: %s on Host %s', request_event, host.hostname) - elif request_event.type == RequestType.StopVM: - vm.stop() + shared.request_pool.client.client.delete(request_event.key) + vm_entry = shared.etcd_client.get( + join_path(settings['etcd']['vm_prefix'], request_event.uuid) + ) - elif request_event.type == RequestType.DeleteVM: - vm.delete() + logger.debug('VM hostname: {}'.format(vm_entry.value)) - elif request_event.type == RequestType.InitVMMigration: - vm.start(destination_host_key=host.key) + vm = virtualmachine.VM(vm_entry) + if request_event.type == RequestType.StartVM: + vm.start() - elif request_event.type == RequestType.TransferVM: - destination_host = host_pool.get(request_event.destination_host_key) - if destination_host: - vm.migrate( - destination_host=destination_host.hostname, - destination_sock_path=request_event.destination_sock_path, - ) - else: - logger.error('Host %s not found!', request_event.destination_host_key) + elif request_event.type == RequestType.StopVM: + vm.stop() + elif request_event.type == RequestType.DeleteVM: + vm.delete() -if __name__ == '__main__': - argparser = argparse.ArgumentParser() - argparser.add_argument( - 'hostname', help='Name of this host. e.g uncloud1.ungleich.ch' - ) - args = argparser.parse_args() - mp.set_start_method('spawn') - main(args.hostname) + elif request_event.type == RequestType.InitVMMigration: + vm.start(destination_host_key=host.key) + + elif request_event.type == RequestType.TransferVM: + destination_host = host_pool.get(request_event.destination_host_key) + if destination_host: + vm.migrate( + destination_host=destination_host.hostname, + destination_sock_path=request_event.destination_sock_path, + ) + else: + logger.error('Host %s not found!', request_event.destination_host_key) diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index a37dee4..2f6a5e3 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -16,7 +16,7 @@ from uncloud.common.vm import VMStatus, declare_stopped from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.common.settings import settings from uncloud.vmm import VMM diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index cb13ac7..91f100e 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -5,7 +5,7 @@ import subprocess as sp from os.path import join as join_path from uncloud.common.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.imagescanner import logger diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index 03469a5..73d59cd 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -6,7 +6,7 @@ from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException from uncloud.common.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared app = Flask(__name__) api = Api(app) diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index a7fec15..108d126 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -6,7 +6,7 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.common.settings import settings diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 5143537..20a52cb 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -6,59 +6,51 @@ import argparse -from uncloud.common.request import RequestEntry, RequestType -from uncloud.shared import shared from uncloud.common.settings import settings -from .helper import (dead_host_mitigation, dead_host_detection, assign_host, NoSuitableHostFound) -from . import logger +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.shared import shared +from uncloud.scheduler import logger +from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection, + assign_host, NoSuitableHostFound) arg_parser = argparse.ArgumentParser('scheduler', add_help=False) def main(debug=False): - for request_iterator in [ - shared.etcd_client.get_prefix( - settings["etcd"]["request_prefix"], value_in_json=True - ), - shared.etcd_client.watch_prefix( - settings["etcd"]["request_prefix"], - timeout=5, - value_in_json=True, - ), - ]: - for request_event in request_iterator: - request_entry = RequestEntry(request_event) - # Never Run time critical mechanism inside timeout - # mechanism because timeout mechanism only comes - # when no other event is happening. It means under - # heavy load there would not be a timeout event. - if request_entry.type == "TIMEOUT": + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for request_iterator in [ + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + ]: + for request_event in request_iterator: + dead_host_mitigation(dead_host_detection()) + request_entry = RequestEntry(request_event) - # Detect hosts that are dead and set their status - # to "DEAD", and their VMs' status to "KILLED" - dead_hosts = dead_host_detection() - if dead_hosts: - logger.debug("Dead hosts: %s", dead_hosts) - dead_host_mitigation(dead_hosts) + if request_entry.type == RequestType.ScheduleVM: + logger.debug('%s, %s', request_entry.key, request_entry.value) - elif request_entry.type == RequestType.ScheduleVM: - logger.debug("%s, %s", request_entry.key, request_entry.value) + vm_entry = shared.vm_pool.get(request_entry.uuid) + if vm_entry is None: + logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid)) + continue - vm_entry = shared.vm_pool.get(request_entry.uuid) - if vm_entry is None: - logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) - continue + shared.etcd_client.client.delete(request_entry.key) # consume Request - shared.etcd_client.client.delete(request_entry.key) # consume Request + try: + assign_host(vm_entry) + except NoSuitableHostFound: + vm_entry.add_log('Can\'t schedule VM. No Resource Left.') + shared.vm_pool.put(vm_entry) - try: - assign_host(vm_entry) - except NoSuitableHostFound: - vm_entry.add_log("Can't schedule VM. No Resource Left.") - shared.vm_pool.put(vm_entry) - - logger.info("No Resource Left. Emailing admin....") + logger.info('No Resource Left. Emailing admin....') -if __name__ == "__main__": +if __name__ == '__main__': main() From feb334cf0451655d8dfc210beef4be11f817781d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 10:07:01 +0100 Subject: [PATCH 092/163] Exit code == 1 in case we died with an exception --- scripts/uncloud | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/uncloud b/scripts/uncloud index 1ca9c68..533fc4b 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -40,7 +40,6 @@ if __name__ == '__main__': if not args.command: arg_parser.print_help() else: - # if we start etcd in seperate process with default settings # i.e inheriting few things from parent process etcd3 module # errors out, so the following command configure multiprocessing @@ -54,5 +53,7 @@ if __name__ == '__main__': main(**arguments) except UncloudException as err: logger.error(err) + sys.exit(1) except Exception as err: logger.exception(err) + sys.exit(1) From 92f985c857ac872a2a027cec032a65f536d231e1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 10:10:37 +0100 Subject: [PATCH 093/163] Handle etcd connection error --- uncloud/common/etcd_wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index fe768ac..211bd3c 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -65,6 +65,8 @@ class Etcd3Wrapper: event_iterator = self.client.get_prefix(*args, **kwargs) for e in event_iterator: yield EtcdEntry(*e[::-1], value_in_json=value_in_json) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException("Cannot connect to etcd: {}".format(e)) except Exception as err: if raise_exception: raise Exception('Exception in etcd_wrapper.get_prefix') from err From 71fd0ca7d9c141045ea42c34cda4a38730fda789 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:00:00 +0100 Subject: [PATCH 094/163] Remove double try/except blocks (with wraps) --- uncloud/common/etcd_wrapper.py | 38 ++++++++-------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index 211bd3c..38471ab 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -61,35 +61,15 @@ class Etcd3Wrapper: @readable_errors def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): - try: - event_iterator = self.client.get_prefix(*args, **kwargs) - for e in event_iterator: - yield EtcdEntry(*e[::-1], value_in_json=value_in_json) - except etcd3.exceptions.ConnectionFailedError as e: - raise UncloudException("Cannot connect to etcd: {}".format(e)) - except Exception as err: - if raise_exception: - raise Exception('Exception in etcd_wrapper.get_prefix') from err - else: - logger.exception('Error in etcd_wrapper') - return iter([]) + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) @readable_errors def watch_prefix(self, key, raise_exception=True, value_in_json=False): - try: - event_iterator, cancel = self.client.watch_prefix(key) - for e in event_iterator: - if hasattr(e, '_event'): - e = e._event - if e.type == e.PUT: - yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) - except Exception as err: - if raise_exception: - raise Exception('Exception in etcd_wrapper.get_prefix') from err - else: - logger.exception('Error in etcd_wrapper.watch_prefix') - try: - cancel() - except Exception: - pass - return iter([]) + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = e._event + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) From b7596e071a127c5a3bd1af17bff5efbd400d3dd7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:30:23 +0100 Subject: [PATCH 095/163] begin phasing in arguments instead of **arguments --- scripts/uncloud | 35 +++++++++++++++++++++++++++++++---- uncloud/api/main.py | 5 ++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 533fc4b..4f83e9e 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -8,6 +8,17 @@ import multiprocessing as mp from uncloud import UncloudException from contextlib import suppress +# the components that use etcd +ETCD_COMPONENTS= ['api', + 'scheduler', + 'host', + 'filescanner', + 'imagescanner', + 'metadata', + 'configure' ] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('cli') def exception_hook(exc_type, exc_value, exc_traceback): logging.getLogger(__name__).error( @@ -30,11 +41,25 @@ if __name__ == '__main__': parent_parser.add_argument('--debug', '-d', action='store_true', default=False, help='More verbose logging') - for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', - 'metadata', 'configure', 'cli']: + etcd_parser = argparse.ArgumentParser(add_help=False) + etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-port') + etcd_parser.add_argument('--etcd-ca-cert', + help="CA that signed the etcd certificate") + etcd_parser.add_argument('--etcd-cert-cert', + help="Path to client certificate") + etcd_parser.add_argument('--etcd-cert-key', + help="Path to client certificate key") + + for component in ALL_COMPONENTS: mod = importlib.import_module('uncloud.{}.main'.format(component)) parser = getattr(mod, 'arg_parser') - subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) + + if component in ETCD_COMPONENTS: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) + else: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) + args = arg_parser.parse_args() if not args.command: @@ -46,11 +71,13 @@ if __name__ == '__main__': # module to not inherit anything from parent. # mp.set_start_method('spawn') arguments = vars(args) + print(arguments) + print(etcd_parser) try: name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') - main(**arguments) + main(arguments) except UncloudException as err: logger.error(err) sys.exit(1) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 2d8d035..de75f07 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -563,7 +563,10 @@ api.add_resource(ListHost, '/host/list') api.add_resource(CreateNetwork, '/network/create') -def main(debug=False, port=None): +def main(arguments): + debug = arguments['debug'] + port = arguments['port'] + try: image_stores = list( shared.etcd_client.get_prefix( From d9dd6b48dcce5e70f94c0a3c29548258ed6c05ca Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:35:04 +0100 Subject: [PATCH 096/163] No try: needed for pop/importlib/getattr --- scripts/uncloud | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 4f83e9e..6da98e0 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -73,10 +73,14 @@ if __name__ == '__main__': arguments = vars(args) print(arguments) print(etcd_parser) + + name = arguments.pop('command') + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') + + if component in ETCD_COMPONENTS: + import etcd3 try: - name = arguments.pop('command') - mod = importlib.import_module('uncloud.{}.main'.format(name)) - main = getattr(mod, 'main') main(arguments) except UncloudException as err: logger.error(err) From 82a69701ceb554d3838089ef4f92326ebf603d16 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:43:53 +0100 Subject: [PATCH 097/163] catch etcd in scripts/ --- scripts/uncloud | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 6da98e0..4f9b38b 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -71,20 +71,22 @@ if __name__ == '__main__': # module to not inherit anything from parent. # mp.set_start_method('spawn') arguments = vars(args) - print(arguments) - print(etcd_parser) - name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') - if component in ETCD_COMPONENTS: + # If the component requires etcd3, we import it and catch the + # etcd3.exceptions.ConnectionFailedError + if name in ETCD_COMPONENTS: import etcd3 + try: main(arguments) except UncloudException as err: logger.error(err) sys.exit(1) + except etcd3.exceptions.ConnectionFailedError as err: + logger.error("Cannot connect to etcd") except Exception as err: logger.exception(err) sys.exit(1) From 31ec024be621be09b564979cab2f2702e6141329 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 10 Jan 2020 15:45:48 +0500 Subject: [PATCH 098/163] passing arguments dict to componenets instead of **kwargs --- scripts/uncloud | 30 +++++++----------------------- uncloud/api/main.py | 10 ++-------- uncloud/filescanner/main.py | 3 ++- uncloud/host/main.py | 3 ++- uncloud/imagescanner/main.py | 2 +- uncloud/metadata/main.py | 8 +++----- uncloud/scheduler/main.py | 6 +----- 7 files changed, 18 insertions(+), 44 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 6da98e0..1975f84 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -3,23 +3,16 @@ import logging import sys import importlib import argparse -import multiprocessing as mp from uncloud import UncloudException -from contextlib import suppress # the components that use etcd -ETCD_COMPONENTS= ['api', - 'scheduler', - 'host', - 'filescanner', - 'imagescanner', - 'metadata', - 'configure' ] +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() ALL_COMPONENTS.append('cli') + def exception_hook(exc_type, exc_value, exc_traceback): logging.getLogger(__name__).error( 'Uncaught exception', @@ -44,12 +37,9 @@ if __name__ == '__main__': etcd_parser = argparse.ArgumentParser(add_help=False) etcd_parser.add_argument('--etcd-host') etcd_parser.add_argument('--etcd-port') - etcd_parser.add_argument('--etcd-ca-cert', - help="CA that signed the etcd certificate") - etcd_parser.add_argument('--etcd-cert-cert', - help="Path to client certificate") - etcd_parser.add_argument('--etcd-cert-key', - help="Path to client certificate key") + etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') + etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') + etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') for component in ALL_COMPONENTS: mod = importlib.import_module('uncloud.{}.main'.format(component)) @@ -60,19 +50,13 @@ if __name__ == '__main__': else: subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - args = arg_parser.parse_args() if not args.command: arg_parser.print_help() else: - # if we start etcd in seperate process with default settings - # i.e inheriting few things from parent process etcd3 module - # errors out, so the following command configure multiprocessing - # module to not inherit anything from parent. - # mp.set_start_method('spawn') arguments = vars(args) - print(arguments) - print(etcd_parser) + # print(arguments) + # print(etcd_parser) name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index de75f07..34e1dd1 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -565,7 +565,7 @@ api.add_resource(CreateNetwork, '/network/create') def main(arguments): debug = arguments['debug'] - port = arguments['port'] + port = arguments['port'] try: image_stores = list( @@ -596,12 +596,6 @@ def main(arguments): # ) try: - app.run(host='::', - port=port, - debug=debug) + app.run(host='::', port=port, debug=debug) except OSError as e: raise UncloudException('Failed to start Flask: {}'.format(e)) - - -if __name__ == '__main__': - main() diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index 314481f..c5660dd 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -68,7 +68,8 @@ def track_file(file, base_dir, host): shared.etcd_client.put(entry_key, entry_value, value_in_json=True) -def main(hostname, debug=False): +def main(arguments): + hostname = arguments['hostname'] base_dir = settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index 695e3d1..ccffd77 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -44,7 +44,8 @@ def maintenance(host): shared.vm_pool.put(vm) -def main(hostname, debug=False): +def main(arguments): + hostname = arguments['hostname'] host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index 91f100e..1803213 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -30,7 +30,7 @@ def qemu_img_type(path): return qemu_img_info["format"] -def main(debug=False): +def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( settings["etcd"]["image_prefix"], value_in_json=True diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index 73d59cd..ccda60e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -88,9 +88,7 @@ class Root(Resource): api.add_resource(Root, "/") -def main(port=None, debug=False): +def main(arguments): + port = arguments['port'] + debug = arguments['debug'] app.run(debug=debug, host="::", port=port) - - -if __name__ == "__main__": - main() diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 20a52cb..c25700b 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -16,7 +16,7 @@ from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection, arg_parser = argparse.ArgumentParser('scheduler', add_help=False) -def main(debug=False): +def main(arguments): # The below while True is neccessary for gracefully handling leadership transfer and temporary # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) @@ -50,7 +50,3 @@ def main(debug=False): shared.vm_pool.put(vm_entry) logger.info('No Resource Left. Emailing admin....') - - -if __name__ == '__main__': - main() From ec66a756a011325a96719a2aada9c7b1b1d6c2fc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:56:47 +0100 Subject: [PATCH 099/163] ++confdir --- scripts/uncloud | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/uncloud b/scripts/uncloud index 4f9b38b..a6747be 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -38,8 +38,12 @@ if __name__ == '__main__': subparsers = arg_parser.add_subparsers(dest='command') parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument('--debug', '-d', action='store_true', default=False, + parent_parser.add_argument('--debug', '-d', + action='store_true', + default=False, help='More verbose logging') + parent_parser.add_argument('--conf-dir', '-c', + help='Configuration directory') etcd_parser = argparse.ArgumentParser(add_help=False) etcd_parser.add_argument('--etcd-host') From e91fd9e24af37bcf7703dfd39a8496bfb28ac1f7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 12:00:02 +0100 Subject: [PATCH 100/163] disable cli until bug #25 is fixed --- scripts/uncloud | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uncloud b/scripts/uncloud index bfb3174..3690ad9 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -10,7 +10,7 @@ from uncloud import UncloudException ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('cli') +#ALL_COMPONENTS.append('cli') def exception_hook(exc_type, exc_value, exc_traceback): From 00d876aea1a92c8aaf2be2a2e95bd42d3985335a Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 10 Jan 2020 16:39:40 +0500 Subject: [PATCH 101/163] Do not break if client section/or OTP creds missing from conf file --- scripts/uncloud | 2 -- uncloud/cli/helper.py | 17 +++++++++++++---- uncloud/cli/main.py | 10 +++++----- uncloud/common/settings.py | 6 ++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 3690ad9..8aac240 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -73,8 +73,6 @@ if __name__ == '__main__': except UncloudException as err: logger.error(err) sys.exit(1) - except etcd3.exceptions.ConnectionFailedError as err: - logger.error("Cannot connect to etcd") except Exception as err: logger.exception(err) sys.exit(1) diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py index 0495fac..3c63073 100644 --- a/uncloud/cli/helper.py +++ b/uncloud/cli/helper.py @@ -10,10 +10,19 @@ from uncloud.common.settings import settings def get_otp_parser(): otp_parser = argparse.ArgumentParser('otp') - otp_parser.add_argument('--name', default=settings['client']['name']) - otp_parser.add_argument('--realm', default=settings['client']['realm']) - otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'], - dest='token', metavar='SEED') + try: + name = settings['client']['name'] + realm = settings['client']['realm'] + seed = settings['client']['seed'] + except Exception: + otp_parser.add_argument('--name', required=True) + otp_parser.add_argument('--realm', required=True) + otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED') + else: + otp_parser.add_argument('--name', default=name) + otp_parser.add_argument('--realm', default=realm) + otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED') + return otp_parser diff --git a/uncloud/cli/main.py b/uncloud/cli/main.py index 7f5e367..9a42497 100644 --- a/uncloud/cli/main.py +++ b/uncloud/cli/main.py @@ -12,12 +12,12 @@ for component in ['user', 'host', 'image', 'network', 'vm']: subparser.add_parser(name=parser.prog, parents=[parser]) -def main(**kwargs): - if not kwargs['subcommand']: +def main(arguments): + if not arguments['subcommand']: arg_parser.print_help() else: - name = kwargs.pop('subcommand') - kwargs.pop('debug') + name = arguments.pop('subcommand') + arguments.pop('debug') mod = importlib.import_module('uncloud.cli.{}'.format(name)) _main = getattr(mod, 'main') - _main(**kwargs) + _main(**arguments) diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 47ad5a7..0d524a7 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -99,12 +99,10 @@ class Settings(object): def read_config_file_values(self, config_file): try: # Trying to read configuration file - with open(config_file, 'r') as config_file_handle: + with open(config_file) as config_file_handle: self.config_parser.read_file(config_file_handle) except FileNotFoundError: - sys.exit( - 'Configuration file {} not found!'.format(config_file) - ) + sys.exit('Configuration file {} not found!'.format(config_file)) except Exception as err: logger.exception(err) sys.exit('Error occurred while reading configuration file') From cf4930ee84c0b588fd322a34a40e232d9d4c583c Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 10 Jan 2020 16:42:07 +0500 Subject: [PATCH 102/163] cli enabled again --- scripts/uncloud | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uncloud b/scripts/uncloud index 8aac240..a6e61aa 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -10,7 +10,7 @@ from uncloud import UncloudException ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() -#ALL_COMPONENTS.append('cli') +ALL_COMPONENTS.append('cli') def exception_hook(exc_type, exc_value, exc_traceback): From ebcb1680d70114364dfdcab9cc6e8699ffc52f50 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 23:27:21 +0100 Subject: [PATCH 103/163] add hack scripts --- uncloud/hack/hackcloud/foo | 2 ++ uncloud/hack/hackcloud/ifup.sh | 8 ++++++ uncloud/hack/hackcloud/net.sh | 21 +++++++++++++++ uncloud/hack/hackcloud/vm.sh | 48 ++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 uncloud/hack/hackcloud/foo create mode 100755 uncloud/hack/hackcloud/ifup.sh create mode 100755 uncloud/hack/hackcloud/net.sh create mode 100755 uncloud/hack/hackcloud/vm.sh diff --git a/uncloud/hack/hackcloud/foo b/uncloud/hack/hackcloud/foo new file mode 100644 index 0000000..1033abf --- /dev/null +++ b/uncloud/hack/hackcloud/foo @@ -0,0 +1,2 @@ +tap0 +tap0 diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh new file mode 100755 index 0000000..95bfe5a --- /dev/null +++ b/uncloud/hack/hackcloud/ifup.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +echo $@ >> foo + +dev=$1; shift + +# bridge is setup from outside +ip link set dev "$dev" master ${bridge} diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh new file mode 100755 index 0000000..e56822f --- /dev/null +++ b/uncloud/hack/hackcloud/net.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +netid=100 +dev=wlp2s0 +dev=wlp0s20f3 +dev=wlan0 + +vxlandev=vxlan${netid} +bridgedev=br${netid} + +ip -6 link add ${vxlandev} type vxlan \ + id ${netid} \ + dstport 4789 \ + group ff05::${netid} \ + dev ${dev} \ + ttl 5 + +ip link set ${vxlandev} up + +ip link add ${bridgedev} type bridge +ip link set ${bridgedev} up diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh new file mode 100755 index 0000000..2a8b794 --- /dev/null +++ b/uncloud/hack/hackcloud/vm.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +vmid=$1; shift + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +accel=tcg + +memory=1024 +cores=2 +uuid=732e08c7-84f8-4d43-9571-263db4f80080 + +export bridge=br100 + +$qemu -name uc${vmid} \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e + + + +exit 0 + +-S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-17-one-24992/master-key.aes +-machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off + +-m 2048 +-realtime mlock=off +-smp 1,sockets=1,cores=1,threads=1 +-uuid 732e08c7-84f8-4d43-9571-263db4f80080 -no-user-config \ + -nodefaults +-chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-17-one-24992/monitor.sock,server,nowait +-mon chardev=charmonitor,id=monitor,mode=control +-rtc base=utc -no-shutdown +-boot strict=on +-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 +-drive file=rbd:ssd/one-292-24992-0:id=libvirt:auth_supported=cephx\;none:mon_host=ceph1\:6789\;ceph2\:6789\;ceph3\:6789,format=raw,if=none,id=drive-virtio-disk0,cache=none +-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 +-drive file=/var/lib/one//datastores/104/24992/disk.1,format=raw,if=none,id=drive-ide0-0-0,readonly=on +-device ide-cd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0 +-netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38 +-device virtio-net-pci,netdev=hostnet0,id=net0,mac=02:00:f0:a9:c4:4e,bus=pci.0,addr=0x3 +-vnc [::]:4414 -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on From b9c9a5e0eca41009f5461ff601b11a3bf2151ea7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 23:55:21 +0100 Subject: [PATCH 104/163] add working network --- uncloud/hack/hackcloud/.gitignore | 3 +++ uncloud/hack/hackcloud/foo | 2 -- uncloud/hack/hackcloud/ifup.sh | 1 + uncloud/hack/hackcloud/net.sh | 3 +++ uncloud/hack/hackcloud/radvd.conf | 13 +++++++++++++ uncloud/hack/hackcloud/radvd.sh | 3 +++ 6 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 uncloud/hack/hackcloud/.gitignore delete mode 100644 uncloud/hack/hackcloud/foo create mode 100644 uncloud/hack/hackcloud/radvd.conf create mode 100644 uncloud/hack/hackcloud/radvd.sh diff --git a/uncloud/hack/hackcloud/.gitignore b/uncloud/hack/hackcloud/.gitignore new file mode 100644 index 0000000..0ad647b --- /dev/null +++ b/uncloud/hack/hackcloud/.gitignore @@ -0,0 +1,3 @@ +*.iso +radvdpid +foo diff --git a/uncloud/hack/hackcloud/foo b/uncloud/hack/hackcloud/foo deleted file mode 100644 index 1033abf..0000000 --- a/uncloud/hack/hackcloud/foo +++ /dev/null @@ -1,2 +0,0 @@ -tap0 -tap0 diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh index 95bfe5a..99e8690 100755 --- a/uncloud/hack/hackcloud/ifup.sh +++ b/uncloud/hack/hackcloud/ifup.sh @@ -6,3 +6,4 @@ dev=$1; shift # bridge is setup from outside ip link set dev "$dev" master ${bridge} +ip link set dev "$dev" up diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh index e56822f..7d4b88f 100755 --- a/uncloud/hack/hackcloud/net.sh +++ b/uncloud/hack/hackcloud/net.sh @@ -5,6 +5,7 @@ dev=wlp2s0 dev=wlp0s20f3 dev=wlan0 +ip=2a0a:e5c1:111:888::42/64 vxlandev=vxlan${netid} bridgedev=br${netid} @@ -19,3 +20,5 @@ ip link set ${vxlandev} up ip link add ${bridgedev} type bridge ip link set ${bridgedev} up + +ip addr add ${ip} dev ${bridgedev} diff --git a/uncloud/hack/hackcloud/radvd.conf b/uncloud/hack/hackcloud/radvd.conf new file mode 100644 index 0000000..3d8ce4d --- /dev/null +++ b/uncloud/hack/hackcloud/radvd.conf @@ -0,0 +1,13 @@ +interface br100 +{ + AdvSendAdvert on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 5; + AdvDefaultLifetime 3600; + + prefix 2a0a:e5c1:111:888::/64 { + }; + + RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; }; + DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ; +}; diff --git a/uncloud/hack/hackcloud/radvd.sh b/uncloud/hack/hackcloud/radvd.sh new file mode 100644 index 0000000..9d0e7d1 --- /dev/null +++ b/uncloud/hack/hackcloud/radvd.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +radvd -C ./radvd.conf -n -p ./radvdpid From 3825c7c210fefa9112a0efdeea3f8c2e20ad7bec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 00:23:55 +0100 Subject: [PATCH 105/163] Add vxlan into the bridge --- uncloud/hack/hackcloud/net.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh index 7d4b88f..0a96fe1 100755 --- a/uncloud/hack/hackcloud/net.sh +++ b/uncloud/hack/hackcloud/net.sh @@ -18,7 +18,10 @@ ip -6 link add ${vxlandev} type vxlan \ ip link set ${vxlandev} up + ip link add ${bridgedev} type bridge ip link set ${bridgedev} up +ip link set ${vxlandev} master ${bridgedev} up + ip addr add ${ip} dev ${bridgedev} From 23d805f04fc8349d5e503414174819f7d26a1f78 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 00:24:17 +0100 Subject: [PATCH 106/163] ++stuff --- uncloud/hack/hackcloud/net.sh | 4 ++-- uncloud/hack/uncloud-run-vm | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh index 7d4b88f..4cb6498 100755 --- a/uncloud/hack/hackcloud/net.sh +++ b/uncloud/hack/hackcloud/net.sh @@ -3,9 +3,9 @@ netid=100 dev=wlp2s0 dev=wlp0s20f3 -dev=wlan0 +#dev=wlan0 -ip=2a0a:e5c1:111:888::42/64 +ip=2a0a:e5c1:111:888::48/64 vxlandev=vxlan${netid} bridgedev=br${netid} diff --git a/uncloud/hack/uncloud-run-vm b/uncloud/hack/uncloud-run-vm index 1af2037..33e5860 100644 --- a/uncloud/hack/uncloud-run-vm +++ b/uncloud/hack/uncloud-run-vm @@ -18,4 +18,8 @@ qemu-system-x86_64 \ -smp 2,sockets=2,cores=1,threads=1 \ -device virtio-net-pci,netdev=net0,mac=$macaddress \ -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \ - -vnc [::]:5900 + -vnc [::]:0 + +# To be changed: +# -vnc to unix path +# or -spice From c1cabb7220e208ed2ba976c695f786a23edac185 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 02:42:04 +0100 Subject: [PATCH 107/163] add working nft --- uncloud/hack/hackcloud/nftrules | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 uncloud/hack/hackcloud/nftrules diff --git a/uncloud/hack/hackcloud/nftrules b/uncloud/hack/hackcloud/nftrules new file mode 100644 index 0000000..661d91f --- /dev/null +++ b/uncloud/hack/hackcloud/nftrules @@ -0,0 +1,32 @@ +flush ruleset + +table bridge filter { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; + ibrname br100 jump netpublic + } + chain netpublic { + + iifname tap1 jump vm1 + + icmpv6 type {nd-router-solicit, nd-router-advert, + nd-neighbor-solicit, nd-neighbor-advert, nd-redirect } log + + } + chain vm1 { + ether saddr != 02:00:f0:a9:c4:4e drop + } +} + +table ip6 filter { + chain forward { + type filter hook forward priority 0; + + # policy drop; + + ct state established,related accept; + + } + +} From 029ef36d62d0fd4767cbfc9da7a721f3973f1bd2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 15:54:19 +0100 Subject: [PATCH 108/163] net +debug --- uncloud/hack/hackcloud/net.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh index 5c7a6f5..4e2bfa1 100755 --- a/uncloud/hack/hackcloud/net.sh +++ b/uncloud/hack/hackcloud/net.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -x + netid=100 dev=wlp2s0 dev=wlp0s20f3 From 3b68a589d47f40b8d6a0e4020e2a4e0254741424 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 16:17:35 +0100 Subject: [PATCH 109/163] cleanup vm.sh --- uncloud/hack/hackcloud/vm.sh | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index 2a8b794..2cc0480 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -21,28 +21,3 @@ $qemu -name uc${vmid} \ -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ -netdev tap,id=netmain,script=./ifup.sh \ -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e - - - -exit 0 - --S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-17-one-24992/master-key.aes --machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off - --m 2048 --realtime mlock=off --smp 1,sockets=1,cores=1,threads=1 --uuid 732e08c7-84f8-4d43-9571-263db4f80080 -no-user-config \ - -nodefaults --chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-17-one-24992/monitor.sock,server,nowait --mon chardev=charmonitor,id=monitor,mode=control --rtc base=utc -no-shutdown --boot strict=on --device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 --drive file=rbd:ssd/one-292-24992-0:id=libvirt:auth_supported=cephx\;none:mon_host=ceph1\:6789\;ceph2\:6789\;ceph3\:6789,format=raw,if=none,id=drive-virtio-disk0,cache=none --device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 --drive file=/var/lib/one//datastores/104/24992/disk.1,format=raw,if=none,id=drive-ide0-0-0,readonly=on --device ide-cd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0 --netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38 --device virtio-net-pci,netdev=hostnet0,id=net0,mac=02:00:f0:a9:c4:4e,bus=pci.0,addr=0x3 --vnc [::]:4414 -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on From 708e3ebb971745a3ea8e6ad81179365829bc7398 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 16:20:29 +0100 Subject: [PATCH 110/163] cleanup ifup.sh --- uncloud/hack/hackcloud/ifup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh index 99e8690..e0a3ca0 100755 --- a/uncloud/hack/hackcloud/ifup.sh +++ b/uncloud/hack/hackcloud/ifup.sh @@ -1,7 +1,5 @@ #!/bin/sh -echo $@ >> foo - dev=$1; shift # bridge is setup from outside From 8544df8bad808910f31eaa85bf0808f056f09512 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 16:36:41 +0100 Subject: [PATCH 111/163] don't use tcg --- uncloud/hack/hackcloud/vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index 2cc0480..dfef8cc 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -5,7 +5,7 @@ vmid=$1; shift qemu=/usr/bin/qemu-system-x86_64 accel=kvm -accel=tcg +#accel=tcg memory=1024 cores=2 From c6b7152464f8a68fc2e9e7383c5044fef3a84005 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 21:21:30 +0100 Subject: [PATCH 112/163] update nftrules example --- uncloud/hack/hackcloud/nftrules | 72 ++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/uncloud/hack/hackcloud/nftrules b/uncloud/hack/hackcloud/nftrules index 661d91f..035b3a8 100644 --- a/uncloud/hack/hackcloud/nftrules +++ b/uncloud/hack/hackcloud/nftrules @@ -1,32 +1,64 @@ flush ruleset table bridge filter { - chain prerouting { - type filter hook prerouting priority 0; - policy accept; - ibrname br100 jump netpublic - } - chain netpublic { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; - iifname tap1 jump vm1 + ibrname br100 jump netpublic + } - icmpv6 type {nd-router-solicit, nd-router-advert, - nd-neighbor-solicit, nd-neighbor-advert, nd-redirect } log + chain netpublic { + iifname vxlan100 jump from_uncloud - } - chain vm1 { - ether saddr != 02:00:f0:a9:c4:4e drop - } + # Default blocks: router advertisements, dhcpv6, dhcpv4 + icmpv6 type nd-router-advert drop + ip6 version 6 udp sport 547 drop + ip version 4 udp sport 67 drop + + # Individual blocks + iifname tap1 jump vm1 + } + + chain vm1 { + ether saddr != 02:00:f0:a9:c4:4e drop + ip6 saddr != 2a0a:e5c1:111:888:0:f0ff:fea9:c44e drop + } + + chain from_uncloud { + accept + } } -table ip6 filter { - chain forward { - type filter hook forward priority 0; +# table ip6 filter { +# chain forward { +# type filter hook forward priority 0; - # policy drop; +# # policy drop; - ct state established,related accept; +# ct state established,related accept; - } +# } -} +# } + +# table ip filter { +# chain input { +# type filter hook input priority filter; policy drop; +# iif "lo" accept +# icmp type { echo-reply, destination-unreachable, source-quench, redirect, echo-request, router-advertisement, router-solicitation, time-exceeded, parameter-problem, timestamp-request, timestamp-reply, info-request, info-reply, address-mask-request, address-mask-reply } accept +# ct state established,related accept +# tcp dport { 22 } accept +# log prefix "firewall-ipv4: " +# udp sport 67 drop +# } + +# chain forward { +# type filter hook forward priority filter; policy drop; +# log prefix "firewall-ipv4: " +# } + +# chain output { +# type filter hook output priority filter; policy accept; +# } +# } From 6d51e2a8c4cfe5bb6695e390fd55414657beb8bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 00:32:17 +0100 Subject: [PATCH 113/163] [metadata] change default port to 1234 --- uncloud/metadata/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index ccda60e..c47364e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -13,8 +13,10 @@ api = Api(app) app.logger.handlers.clear() +DEFAULT_PORT=1234 + arg_parser = argparse.ArgumentParser('metadata', add_help=False) -arg_parser.add_argument('--port', '-p', default=80, help='By default bind to port 80') +arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT)) @app.errorhandler(Exception) From aaf29adcbb6c1aea64952099d14c1aaceb644e43 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 00:41:31 +0100 Subject: [PATCH 114/163] + mac prefix --- uncloud/hack/hackcloud/mac-prefix | 1 + 1 file changed, 1 insertion(+) create mode 100644 uncloud/hack/hackcloud/mac-prefix diff --git a/uncloud/hack/hackcloud/mac-prefix b/uncloud/hack/hackcloud/mac-prefix new file mode 100644 index 0000000..5084a2f --- /dev/null +++ b/uncloud/hack/hackcloud/mac-prefix @@ -0,0 +1 @@ +02:00 From b017df4879a4daf8e1f4542fd9f7fcf0aab7fc40 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 13:20:38 +0100 Subject: [PATCH 115/163] ignore iso, update nft rules --- .gitignore | 2 ++ uncloud/hack/hackcloud/nftrules | 57 +++++++-------------------------- 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 5c55899..6f0d9df 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ uncloud/version.py build/ venv/ dist/ + +*.iso diff --git a/uncloud/hack/hackcloud/nftrules b/uncloud/hack/hackcloud/nftrules index 035b3a8..636c63d 100644 --- a/uncloud/hack/hackcloud/nftrules +++ b/uncloud/hack/hackcloud/nftrules @@ -5,60 +5,27 @@ table bridge filter { type filter hook prerouting priority 0; policy accept; - ibrname br100 jump netpublic + ibrname br100 jump br100 } - chain netpublic { - iifname vxlan100 jump from_uncloud + chain br100 { + # Allow all incoming traffic from outside + iifname vxlan100 accept # Default blocks: router advertisements, dhcpv6, dhcpv4 icmpv6 type nd-router-advert drop ip6 version 6 udp sport 547 drop ip version 4 udp sport 67 drop - # Individual blocks - iifname tap1 jump vm1 + jump br100_vmlist + drop } + chain br100_vmlist { + # VM1 + iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept - chain vm1 { - ether saddr != 02:00:f0:a9:c4:4e drop - ip6 saddr != 2a0a:e5c1:111:888:0:f0ff:fea9:c44e drop - } - - chain from_uncloud { - accept + # VM2 + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept } } - -# table ip6 filter { -# chain forward { -# type filter hook forward priority 0; - -# # policy drop; - -# ct state established,related accept; - -# } - -# } - -# table ip filter { -# chain input { -# type filter hook input priority filter; policy drop; -# iif "lo" accept -# icmp type { echo-reply, destination-unreachable, source-quench, redirect, echo-request, router-advertisement, router-solicitation, time-exceeded, parameter-problem, timestamp-request, timestamp-reply, info-request, info-reply, address-mask-request, address-mask-reply } accept -# ct state established,related accept -# tcp dport { 22 } accept -# log prefix "firewall-ipv4: " -# udp sport 67 drop -# } - -# chain forward { -# type filter hook forward priority filter; policy drop; -# log prefix "firewall-ipv4: " -# } - -# chain output { -# type filter hook output priority filter; policy accept; -# } -# } From 64ab011299fa230399cba5c401962974a4b6c069 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 13:41:54 +0100 Subject: [PATCH 116/163] import mac.py from cinv --- uncloud/hack/hackcloud/mac-gen.py | 171 ++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 uncloud/hack/hackcloud/mac-gen.py diff --git a/uncloud/hack/hackcloud/mac-gen.py b/uncloud/hack/hackcloud/mac-gen.py new file mode 100644 index 0000000..9f23854 --- /dev/null +++ b/uncloud/hack/hackcloud/mac-gen.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2012 Nico Schottelius (nico-cinv at schottelius.org) +# +# This file is part of cinv. +# +# cinv is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cinv is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cinv. If not, see . +# +# + +import argparse +import logging +import os.path +import os +import re + +import cinv +from cinv import fsproperty + +log = logging.getLogger(__name__) + +class Error(cinv.Error): + pass + + +class Mac(object): + + def __init__(self): + self.base_dir = self.get_base_dir() + + _prefix = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_dir, "prefix")) + free = fsproperty.FileListProperty(lambda obj: os.path.join(obj.base_dir, "free")) + last = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_dir, "last")) + + def _init_base_dir(self): + try: + os.makedirs(self.base_dir, exist_ok=True) + except OSError as e: + raise Error(e) + + @staticmethod + def validate_mac(mac): + if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): + raise Error("Not a valid mac address: %s" % mac) + + def free_append(self, mac): + if mac in self.free: + raise Error("Mac already in free database: %s" % mac) + + self._init_base_dir() + self.free.append(mac) + + @staticmethod + def get_base_dir(): + return cinv.get_base_dir("db/mac") + + @classmethod + def exists(cls): + return os.path.exists(cls.get_base_dir()) + + def get_next(self): + self._init_base_dir() + + if self.free: + return self.free.pop() + + if not self.prefix: + raise Error("Cannot generate address without prefix - use prefix-set") + + if self.last: + suffix = re.search(r'([0-9A-F]{2}[-:]){2}[0-9A-F]{2}$', self.last, re.I) + last_number_hex = "0x%s" % suffix.group().replace(":", "") + last_number = int(last_number_hex, 16) + + if last_number == int('0xffffff', 16): + raise Error("Exhausted all possible mac addresses - try to free some") + + next_number = last_number + 1 + else: + next_number = 0 + + next_number_hex = "%0.6x" % next_number + next_suffix = "%s:%s:%s" % (next_number_hex[0:2], next_number_hex[2:4], next_number_hex[4:6]) + + next_mac = "%s:%s" % (self.prefix, next_suffix) + + self.last = next_mac + + return next_mac + + + @property + def prefix(self): + return self._prefix + + @prefix.setter + def prefix(self, prefix): + if not re.match(r'([0-9A-F]{2}[-:]){2}[0-9A-F]{2}$', prefix, re.I): + raise Error("Wrong mac address format - use 00:11:22") + + self._init_base_dir() + self._prefix = prefix + + @classmethod + def commandline_generate(cls, args): + mac = Mac() + print(mac.get_next()) + + @classmethod + def commandline_free_add(cls, args): + mac = Mac() + mac.validate_mac(args.address) + mac.free_append(args.address) + + @classmethod + def commandline_free_list(cls, args): + mac = Mac() + for mac in mac.free: + print(mac) + + @classmethod + def commandline_prefix_set(cls, args): + mac = Mac() + mac.prefix = args.prefix + + @classmethod + def commandline_prefix_get(cls, args): + mac = cls() + print(mac.prefix) + + @classmethod + def commandline_add(cls, args): + host = cls(fqdn=args.fqdn) + host.host_type = args.type + + @classmethod + def commandline_args(cls, parent_parser, parents): + """Add us to the parent parser and add all parents to our parsers""" + + parser = {} + parser['sub'] = parent_parser.add_subparsers(title="Mac Commands") + + parser['free-add'] = parser['sub'].add_parser('free-add', parents=parents) + parser['free-add'].add_argument('address', help='Address to add to free database') + parser['free-add'].set_defaults(func=cls.commandline_free_add) + + parser['free-list'] = parser['sub'].add_parser('free-list', parents=parents, + help="List free mac addresses") + parser['free-list'].set_defaults(func=cls.commandline_free_list) + + parser['generate'] = parser['sub'].add_parser('generate', parents=parents) + parser['generate'].set_defaults(func=cls.commandline_generate) + + parser['prefix-get'] = parser['sub'].add_parser('prefix-get', parents=parents) + parser['prefix-get'].set_defaults(func=cls.commandline_prefix_get) + + parser['prefix-set'] = parser['sub'].add_parser('prefix-set', parents=parents) + parser['prefix-set'].add_argument('prefix', help="3 Byte address prefix (f.i. '00:16:3e')") + parser['prefix-set'].set_defaults(func=cls.commandline_prefix_set) From 53c6a14d608c5c7d564a14d4ca31eac9c413c930 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:03:04 +0100 Subject: [PATCH 117/163] mac: begin to downstrip --- uncloud/hack/hackcloud/mac-gen.py | 56 ++++++++++++------------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/uncloud/hack/hackcloud/mac-gen.py b/uncloud/hack/hackcloud/mac-gen.py index 9f23854..f2a5db0 100644 --- a/uncloud/hack/hackcloud/mac-gen.py +++ b/uncloud/hack/hackcloud/mac-gen.py @@ -26,29 +26,32 @@ import os.path import os import re -import cinv -from cinv import fsproperty - log = logging.getLogger(__name__) -class Error(cinv.Error): +class Error(Exception): pass class Mac(object): - def __init__(self): - self.base_dir = self.get_base_dir() + self.base_dir = "." - _prefix = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_dir, "prefix")) - free = fsproperty.FileListProperty(lambda obj: os.path.join(obj.base_dir, "free")) - last = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_dir, "last")) + self._prefix = "02:00" + + self.free = self.read_file("mac-free") + self.last = self.read_file("mac-last") + + def read_file(self, filename): + fname = os.path.join(self.base_dir, filename) + ret = [] - def _init_base_dir(self): try: - os.makedirs(self.base_dir, exist_ok=True) - except OSError as e: - raise Error(e) + with open(fname, "r") as fd: + ret = fd.readlines() + except Exception as e: + pass + + return ret @staticmethod def validate_mac(mac): @@ -146,26 +149,11 @@ class Mac(object): host.host_type = args.type @classmethod - def commandline_args(cls, parent_parser, parents): - """Add us to the parent parser and add all parents to our parsers""" + def commandline(cls): + pass - parser = {} - parser['sub'] = parent_parser.add_subparsers(title="Mac Commands") - parser['free-add'] = parser['sub'].add_parser('free-add', parents=parents) - parser['free-add'].add_argument('address', help='Address to add to free database') - parser['free-add'].set_defaults(func=cls.commandline_free_add) - - parser['free-list'] = parser['sub'].add_parser('free-list', parents=parents, - help="List free mac addresses") - parser['free-list'].set_defaults(func=cls.commandline_free_list) - - parser['generate'] = parser['sub'].add_parser('generate', parents=parents) - parser['generate'].set_defaults(func=cls.commandline_generate) - - parser['prefix-get'] = parser['sub'].add_parser('prefix-get', parents=parents) - parser['prefix-get'].set_defaults(func=cls.commandline_prefix_get) - - parser['prefix-set'] = parser['sub'].add_parser('prefix-set', parents=parents) - parser['prefix-set'].add_argument('prefix', help="3 Byte address prefix (f.i. '00:16:3e')") - parser['prefix-set'].set_defaults(func=cls.commandline_prefix_set) +if __name__ == '__main__': + m = Mac() + m.commandline() + print(m.free) From 94dad7c9b6ae8bd5e5bf4b035b582ce7d4a44a01 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:35:59 +0100 Subject: [PATCH 118/163] Add script to generate mac addresses --- uncloud/hack/hackcloud/mac-gen.py | 96 +++++++++---------------------- uncloud/hack/hackcloud/mac-last | 1 + 2 files changed, 28 insertions(+), 69 deletions(-) mode change 100644 => 100755 uncloud/hack/hackcloud/mac-gen.py create mode 100644 uncloud/hack/hackcloud/mac-last diff --git a/uncloud/hack/hackcloud/mac-gen.py b/uncloud/hack/hackcloud/mac-gen.py old mode 100644 new mode 100755 index f2a5db0..e2b4bc5 --- a/uncloud/hack/hackcloud/mac-gen.py +++ b/uncloud/hack/hackcloud/mac-gen.py @@ -36,7 +36,8 @@ class Mac(object): def __init__(self): self.base_dir = "." - self._prefix = "02:00" + self.prefix = 0x002000000000 + #self.prefix = "{:012x}".format(self._prefix) self.free = self.read_file("mac-free") self.last = self.read_file("mac-last") @@ -53,6 +54,11 @@ class Mac(object): return ret + def append_to_file(self, text, filename): + fname = os.path.join(self.base_dir, filename) + with open(fname, "a+") as fd: + fd.write("{}\n".format(text)) + @staticmethod def validate_mac(mac): if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): @@ -62,30 +68,24 @@ class Mac(object): if mac in self.free: raise Error("Mac already in free database: %s" % mac) - self._init_base_dir() - self.free.append(mac) + self.append_to_file(mac, "mac-free") + self.free = self.read_file("mac-free") + @staticmethod - def get_base_dir(): - return cinv.get_base_dir("db/mac") + def int_to_mac(number): + b = number.to_bytes(6, byteorder="big") + return ':'.join(format(s, '02x') for s in b) - @classmethod - def exists(cls): - return os.path.exists(cls.get_base_dir()) + def getnext(self): +# if self.free: +# return self.free.pop() - def get_next(self): - self._init_base_dir() - - if self.free: - return self.free.pop() - - if not self.prefix: - raise Error("Cannot generate address without prefix - use prefix-set") +# if not self.prefix: +# raise Error("Cannot generate address without prefix - use prefix-set") if self.last: - suffix = re.search(r'([0-9A-F]{2}[-:]){2}[0-9A-F]{2}$', self.last, re.I) - last_number_hex = "0x%s" % suffix.group().replace(":", "") - last_number = int(last_number_hex, 16) + last_number = int(self.last[0], 16) if last_number == int('0xffffff', 16): raise Error("Exhausted all possible mac addresses - try to free some") @@ -94,60 +94,16 @@ class Mac(object): else: next_number = 0 - next_number_hex = "%0.6x" % next_number - next_suffix = "%s:%s:%s" % (next_number_hex[0:2], next_number_hex[2:4], next_number_hex[4:6]) + next_number_string = "{:012x}".format(next_number) - next_mac = "%s:%s" % (self.prefix, next_suffix) + next_mac_number = self.prefix + next_number + next_mac = self.int_to_mac(next_mac_number) - self.last = next_mac + with open(os.path.join(self.base_dir, "mac-last"), "w+") as fd: + fd.write("{}\n".format(next_number_string)) return next_mac - - @property - def prefix(self): - return self._prefix - - @prefix.setter - def prefix(self, prefix): - if not re.match(r'([0-9A-F]{2}[-:]){2}[0-9A-F]{2}$', prefix, re.I): - raise Error("Wrong mac address format - use 00:11:22") - - self._init_base_dir() - self._prefix = prefix - - @classmethod - def commandline_generate(cls, args): - mac = Mac() - print(mac.get_next()) - - @classmethod - def commandline_free_add(cls, args): - mac = Mac() - mac.validate_mac(args.address) - mac.free_append(args.address) - - @classmethod - def commandline_free_list(cls, args): - mac = Mac() - for mac in mac.free: - print(mac) - - @classmethod - def commandline_prefix_set(cls, args): - mac = Mac() - mac.prefix = args.prefix - - @classmethod - def commandline_prefix_get(cls, args): - mac = cls() - print(mac.prefix) - - @classmethod - def commandline_add(cls, args): - host = cls(fqdn=args.fqdn) - host.host_type = args.type - @classmethod def commandline(cls): pass @@ -156,4 +112,6 @@ class Mac(object): if __name__ == '__main__': m = Mac() m.commandline() - print(m.free) + # print(m.free) + #print(m.last) + print(m.getnext()) diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last new file mode 100644 index 0000000..df32b47 --- /dev/null +++ b/uncloud/hack/hackcloud/mac-last @@ -0,0 +1 @@ +000000000006 From 3188787c2a9f4ad9bb7c4b0f66818d21ea5d8579 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:38:01 +0100 Subject: [PATCH 119/163] ++mac change --- uncloud/hack/hackcloud/mac-last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last index df32b47..90a4264 100644 --- a/uncloud/hack/hackcloud/mac-last +++ b/uncloud/hack/hackcloud/mac-last @@ -1 +1 @@ -000000000006 +000000000245 From 02526baaf979783e3f1ed661f8f16a256b3b9f5a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:43:06 +0100 Subject: [PATCH 120/163] add ifdown support --- uncloud/hack/hackcloud/ifdown.sh | 3 +++ uncloud/hack/hackcloud/vm.sh | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 uncloud/hack/hackcloud/ifdown.sh diff --git a/uncloud/hack/hackcloud/ifdown.sh b/uncloud/hack/hackcloud/ifdown.sh new file mode 100644 index 0000000..70fe1db --- /dev/null +++ b/uncloud/hack/hackcloud/ifdown.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo $@! diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index dfef8cc..56956ea 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -1,7 +1,5 @@ #!/bin/sh -vmid=$1; shift - qemu=/usr/bin/qemu-system-x86_64 accel=kvm @@ -9,15 +7,16 @@ accel=kvm memory=1024 cores=2 -uuid=732e08c7-84f8-4d43-9571-263db4f80080 +uuid=$(uuidgen) +mac=$(./mac-gen.py) export bridge=br100 -$qemu -name uc${vmid} \ +$qemu -name "uncloud-!${uuid}" \ -machine pc,accel=${accel} \ -m ${memory} \ -smp ${cores} \ -uuid ${uuid} \ -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ - -netdev tap,id=netmain,script=./ifup.sh \ - -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e + -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac} From e6d22a73c5efbe9c91d316cf1c7ee576bd239e92 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:44:53 +0100 Subject: [PATCH 121/163] ++ cleanup --- uncloud/hack/hackcloud/ifdown.sh | 2 +- uncloud/hack/hackcloud/mac-last | 2 +- uncloud/hack/hackcloud/vm.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) mode change 100644 => 100755 uncloud/hack/hackcloud/ifdown.sh diff --git a/uncloud/hack/hackcloud/ifdown.sh b/uncloud/hack/hackcloud/ifdown.sh old mode 100644 new mode 100755 index 70fe1db..5753099 --- a/uncloud/hack/hackcloud/ifdown.sh +++ b/uncloud/hack/hackcloud/ifdown.sh @@ -1,3 +1,3 @@ #!/bin/sh -echo $@! +echo $@ diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last index 90a4264..59f6410 100644 --- a/uncloud/hack/hackcloud/mac-last +++ b/uncloud/hack/hackcloud/mac-last @@ -1 +1 @@ -000000000245 +000000000251 diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index 56956ea..a0e111b 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -12,7 +12,8 @@ mac=$(./mac-gen.py) export bridge=br100 -$qemu -name "uncloud-!${uuid}" \ +set -x +$qemu -name "uncloud-${uuid}" \ -machine pc,accel=${accel} \ -m ${memory} \ -smp ${cores} \ From c3b42aabc626f4cca1617dfcba9b52fbf587502a Mon Sep 17 00:00:00 2001 From: Ahmed Bilal <49-ahmedbilal@users.noreply.code.ungleich.ch> Date: Mon, 13 Jan 2020 05:57:41 +0100 Subject: [PATCH 122/163] Added --conf-dir, --etcd-{host,port,ca_cert,cert_cert,cert_key} parameters to cli and settings is now accessbile through uncloud.shared.shared.settings --- scripts/uncloud | 59 ++++++++++++++++++----------- uncloud/api/common_fields.py | 7 +--- uncloud/api/create_image_store.py | 3 +- uncloud/api/helper.py | 18 ++++----- uncloud/api/main.py | 61 +++++++++++++++--------------- uncloud/api/schemas.py | 13 +++---- uncloud/cli/helper.py | 29 ++++++-------- uncloud/common/cli.py | 26 +++++++++++++ uncloud/common/settings.py | 19 ++++++---- uncloud/common/shared.py | 22 +++++------ uncloud/common/storage_handlers.py | 15 ++++---- uncloud/configure/main.py | 11 +++--- uncloud/filescanner/main.py | 7 ++-- uncloud/host/main.py | 11 +++--- uncloud/host/virtualmachine.py | 11 +++--- uncloud/imagescanner/main.py | 7 ++-- uncloud/metadata/main.py | 3 +- uncloud/scheduler/helper.py | 3 +- uncloud/scheduler/main.py | 5 +-- 19 files changed, 176 insertions(+), 154 deletions(-) create mode 100644 uncloud/common/cli.py diff --git a/scripts/uncloud b/scripts/uncloud index a6e61aa..1a6483b 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -3,14 +3,13 @@ import logging import sys import importlib import argparse +import os +from etcd3.exceptions import ConnectionFailedError + +from uncloud.common import settings from uncloud import UncloudException - -# the components that use etcd -ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] - -ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('cli') +from uncloud.common.cli import resolve_otp_credentials def exception_hook(exc_type, exc_value, exc_traceback): @@ -22,6 +21,13 @@ def exception_hook(exc_type, exc_value, exc_traceback): sys.excepthook = exception_hook +# the components that use etcd +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('cli') + + if __name__ == '__main__': # Setting up root logger logger = logging.getLogger() @@ -31,15 +37,13 @@ if __name__ == '__main__': subparsers = arg_parser.add_subparsers(dest='command') parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument('--debug', '-d', - action='store_true', - default=False, + parent_parser.add_argument('--debug', '-d', action='store_true', default=False, help='More verbose logging') - parent_parser.add_argument('--conf-dir', '-c', - help='Configuration directory') + parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory', + default=os.path.expanduser('~/uncloud')) etcd_parser = argparse.ArgumentParser(add_help=False) - etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-host', dest='etcd_url') etcd_parser.add_argument('--etcd-port') etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') @@ -54,25 +58,36 @@ if __name__ == '__main__': else: subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - args = arg_parser.parse_args() - if not args.command: + arguments = vars(arg_parser.parse_args()) + etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value] + etcd_arguments = { + 'etcd': { + key.replace('etcd_', ''): arguments[key] + for key in etcd_arguments + } + } + if not arguments['command']: arg_parser.print_help() else: - arguments = vars(args) + # Initializing Settings and resolving otp_credentials + # It is neccessary to resolve_otp_credentials after argument parsing is done because + # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and + # providing the default values for --name, --realm and --seed arguments from the values + # we read from file. But, now we are asking user about where the config file lives. So, + # to providing default value is not possible before parsing arguments. So, we are doing + # it after.. + settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) + resolve_otp_credentials(arguments) + name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') - # If the component requires etcd3, we import it and catch the - # etcd3.exceptions.ConnectionFailedError - if name in ETCD_COMPONENTS: - import etcd3 - try: main(arguments) except UncloudException as err: logger.error(err) - sys.exit(1) + except ConnectionFailedError: + logger.error('Cannot connect to etcd') except Exception as err: logger.exception(err) - sys.exit(1) diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py index d1fcb64..ba9fb37 100755 --- a/uncloud/api/common_fields.py +++ b/uncloud/api/common_fields.py @@ -1,7 +1,6 @@ import os from uncloud.common.shared import shared -from uncloud.common.settings import settings class Optional: @@ -54,9 +53,7 @@ class VmUUIDField(Field): def vm_uuid_validation(self): r = shared.etcd_client.get( - os.path.join(settings["etcd"]["vm_prefix"], self.uuid) + os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid) ) if not r: - self.add_error( - "VM with uuid {} does not exists".format(self.uuid) - ) + self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 1040e97..90e0f92 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -4,7 +4,6 @@ import os from uuid import uuid4 from uncloud.common.shared import shared -from uncloud.common.settings import settings data = { 'is_public': True, @@ -15,6 +14,6 @@ data = { } shared.etcd_client.put( - os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), + os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data), ) diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index 0805280..8ceb3a6 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -7,7 +7,6 @@ import requests from pyotp import TOTP from uncloud.common.shared import shared -from uncloud.common.settings import settings logger = logging.getLogger(__name__) @@ -15,9 +14,9 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": settings["otp"]["auth_name"], - "auth_token": TOTP(settings["otp"]["auth_seed"]).now(), - "auth_realm": settings["otp"]["auth_realm"], + "auth_name": shared.settings["otp"]["auth_name"], + "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), + "auth_realm": shared.settings["otp"]["auth_realm"], "name": name, "realm": realm, "token": token, @@ -25,13 +24,13 @@ def check_otp(name, realm, token): except binascii.Error as err: logger.error( "Cannot compute OTP for seed: {}".format( - settings["otp"]["auth_seed"] + shared.settings["otp"]["auth_seed"] ) ) return 400 response = requests.post( - settings["otp"]["verification_controller_url"], json=data + shared.settings["otp"]["verification_controller_url"], json=data ) return response.status_code @@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client): ) images = etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) # Try to find image with name == image_name and store_name == store_name @@ -111,9 +110,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix): lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) prefix = ipaddress.IPv6Address(prefix) return str(prefix + int(lower_part)) + diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 34e1dd1..73e8e21 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -15,9 +15,8 @@ from uncloud.common.shared import shared from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType -from uncloud.common.settings import settings -from . import schemas -from .helper import generate_mac, mac2ipv6 +from uncloud.api import schemas +from uncloud.api.helper import generate_mac, mac2ipv6 from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -50,7 +49,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) + vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) specs = { 'cpu': validator.specs['cpu'], 'ram': validator.specs['ram'], @@ -60,7 +59,7 @@ class CreateVM(Resource): macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['tap_counter'] + shared.etcd_client, shared.settings['etcd']['tap_counter'] ) for _ in range(len(data['network'])) ] @@ -84,7 +83,7 @@ class CreateVM(Resource): r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) @@ -99,7 +98,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) vm_value = vm.value.copy() vm_value['ip'] = [] @@ -107,7 +106,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( join_path( - settings['etcd']['network_prefix'], + shared.settings['etcd']['network_prefix'], data['name'], network_name, ), @@ -130,7 +129,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = shared.etcd_client.get( - join_path(settings['etcd']['file_prefix'], data['uuid']) + join_path(shared.settings['etcd']['file_prefix'], data['uuid']) ) file_entry_value = json.loads(file_entry.value) @@ -144,7 +143,7 @@ class CreateImage(Resource): } shared.etcd_client.put( join_path( - settings['etcd']['image_prefix'], data['uuid'] + shared.settings['etcd']['image_prefix'], data['uuid'] ), json.dumps(image_entry_json), ) @@ -157,7 +156,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = shared.etcd_client.get_prefix( - settings['etcd']['image_prefix'], value_in_json=True + shared.settings['etcd']['image_prefix'], value_in_json=True ) r = {'images': []} for image in images: @@ -178,7 +177,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) action = data['action'] @@ -208,7 +207,7 @@ class VMAction(Resource): type='{}VM'.format(action.title()), uuid=data['uuid'], hostname=vm_entry.hostname, - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( @@ -231,10 +230,10 @@ class VMMigration(Resource): type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( - settings['etcd']['host_prefix'], + shared.settings['etcd']['host_prefix'], validator.destination.value, ), - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) @@ -254,7 +253,7 @@ class ListUserVM(Resource): if validator.is_valid(): vms = shared.etcd_client.get_prefix( - settings['etcd']['vm_prefix'], value_in_json=True + shared.settings['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter( @@ -287,7 +286,7 @@ class ListUserFiles(Resource): if validator.is_valid(): files = shared.etcd_client.get_prefix( - settings['etcd']['file_prefix'], value_in_json=True + shared.settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] user_files = [f for f in files if f.value['owner'] == data['name']] @@ -312,7 +311,7 @@ class CreateHost(Resource): validator = schemas.CreateHostSchema(data) if validator.is_valid(): host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': data['specs'], @@ -354,7 +353,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -372,7 +371,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -405,7 +404,7 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -439,7 +438,7 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -471,23 +470,23 @@ class CreateNetwork(Resource): network_entry = { 'id': counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['vxlan_counter'] + shared.etcd_client, shared.settings['etcd']['vxlan_counter'] ), 'type': data['type'], } if validator.user.value: try: nb = pynetbox.api( - url=settings['netbox']['url'], - token=settings['netbox']['token'], + url=shared.settings['netbox']['url'], + token=shared.settings['netbox']['token'], ) nb_prefix = nb.ipam.prefixes.get( - prefix=settings['network']['prefix'] + prefix=shared.settings['network']['prefix'] ) prefix = nb_prefix.available_prefixes.create( data={ 'prefix_length': int( - settings['network']['prefix_length'] + shared.settings['network']['prefix_length'] ), 'description': '{}\'s network "{}"'.format( data['name'], data['network_name'] @@ -506,7 +505,7 @@ class CreateNetwork(Resource): network_entry['ipv6'] = 'fd00::/64' network_key = join_path( - settings['etcd']['network_prefix'], + shared.settings['etcd']['network_prefix'], data['name'], data['network_name'], ) @@ -526,7 +525,7 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - settings['etcd']['network_prefix'], data['name'] + shared.settings['etcd']['network_prefix'], data['name'] ) networks = shared.etcd_client.get_prefix( prefix, value_in_json=True @@ -570,7 +569,7 @@ def main(arguments): try: image_stores = list( shared.etcd_client.get_prefix( - settings['etcd']['image_store_prefix'], value_in_json=True + shared.settings['etcd']['image_store_prefix'], value_in_json=True ) ) except KeyError: @@ -590,7 +589,7 @@ def main(arguments): # shared.etcd_client.put( # join_path( - # settings['etcd']['image_store_prefix'], uuid4().hex + # shared.settings['etcd']['image_store_prefix'], uuid4().hex # ), # json.dumps(data), # ) diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index e4de9a8..87f20c9 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -22,7 +22,6 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from uncloud.common.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema): def file_uuid_validation(self): file_entry = shared.etcd_client.get( os.path.join( - settings["etcd"]["file_prefix"], self.uuid.value + shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value ) ) if file_entry is None: @@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema): def image_store_name_validation(self): image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"] + shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] ) ) @@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema): for net in _network: network = shared.etcd_client.get( os.path.join( - settings["etcd"]["network_prefix"], + shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, net, ), @@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema): self.add_error("Can't migrate non-running VM") if vm.hostname == os.path.join( - settings["etcd"]["host_prefix"], self.destination.value + shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value ): self.add_error( "Destination host couldn't be same as Source Host" @@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - print(self.name.value, self.network_name.value) - key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) - print(key) + key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) network = shared.etcd_client.get(key, value_in_json=True) if network: self.add_error( diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py index 3c63073..51a4355 100644 --- a/uncloud/cli/helper.py +++ b/uncloud/cli/helper.py @@ -5,23 +5,14 @@ import binascii from pyotp import TOTP from os.path import join as join_path -from uncloud.common.settings import settings +from uncloud.common.shared import shared def get_otp_parser(): otp_parser = argparse.ArgumentParser('otp') - try: - name = settings['client']['name'] - realm = settings['client']['realm'] - seed = settings['client']['seed'] - except Exception: - otp_parser.add_argument('--name', required=True) - otp_parser.add_argument('--realm', required=True) - otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED') - else: - otp_parser.add_argument('--name', default=name) - otp_parser.add_argument('--realm', default=realm) - otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED') + otp_parser.add_argument('--name') + otp_parser.add_argument('--realm') + otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') return otp_parser @@ -34,11 +25,15 @@ def load_dump_pretty(content): def make_request(*args, data=None, request_method=requests.post): - r = request_method(join_path(settings['client']['api_server'], *args), json=data) try: - print(load_dump_pretty(r.content)) - except Exception: - print('Error occurred while getting output from api server.') + r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) + except requests.exceptions.RequestException: + print('Error occurred while connecting to API server.') + else: + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') def get_token(seed): diff --git a/uncloud/common/cli.py b/uncloud/common/cli.py new file mode 100644 index 0000000..3d3c248 --- /dev/null +++ b/uncloud/common/cli.py @@ -0,0 +1,26 @@ +from uncloud.common.shared import shared +from pyotp import TOTP + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except Exception: + raise Exception('Invalid seed') + else: + return token + + +def resolve_otp_credentials(kwargs): + d = { + 'name': shared.settings['client']['name'], + 'realm': shared.settings['client']['realm'], + 'token': get_token(shared.settings['client']['seed']) + } + + for k, v in d.items(): + if k in kwargs and kwargs[k] is None: + kwargs.update({k: v}) + + return d diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 0d524a7..8503f42 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper from os.path import join as join_path logger = logging.getLogger(__name__) +settings = None class CustomConfigParser(configparser.RawConfigParser): @@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): - def __init__(self): + def __init__(self, conf_dir, seed_value=None): conf_name = 'uncloud.conf' - conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/')) self.config_file = join_path(conf_dir, conf_name) # this is used to cache config from etcd for 1 minutes. Without this we @@ -38,15 +38,19 @@ class Settings(object): self.config_parser.add_section('etcd') self.config_parser.set('etcd', 'base_prefix', '/') - try: + if os.access(self.config_file, os.R_OK): self.config_parser.read(self.config_file) - except Exception as err: - logger.error('%s', err) - + else: + raise FileNotFoundError('Config file %s not found!', self.config_file) self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') self.read_internal_values() + if seed_value is None: + seed_value = dict() + + self.config_parser.read_dict(seed_value) + def get_etcd_client(self): args = tuple() try: @@ -128,4 +132,5 @@ class Settings(object): return self.config_parser[key] -settings = Settings() +def get_settings(): + return settings diff --git a/uncloud/common/shared.py b/uncloud/common/shared.py index 918dd0c..aea7cbc 100644 --- a/uncloud/common/shared.py +++ b/uncloud/common/shared.py @@ -1,34 +1,34 @@ -from uncloud.common.settings import settings +from uncloud.common.settings import get_settings from uncloud.common.vm import VmPool from uncloud.common.host import HostPool from uncloud.common.request import RequestPool -from uncloud.common.storage_handlers import get_storage_handler +import uncloud.common.storage_handlers as storage_handlers class Shared: + @property + def settings(self): + return get_settings() + @property def etcd_client(self): - return settings.get_etcd_client() + return self.settings.get_etcd_client() @property def host_pool(self): - return HostPool( - self.etcd_client, settings["etcd"]["host_prefix"] - ) + return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) @property def vm_pool(self): - return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) + return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) @property def request_pool(self): - return RequestPool( - self.etcd_client, settings["etcd"]["request_prefix"] - ) + return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) @property def storage_handler(self): - return get_storage_handler() + return storage_handlers.get_storage_handler() shared = Shared() diff --git a/uncloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py index 6f9b29e..58c2dc2 100644 --- a/uncloud/common/storage_handlers.py +++ b/uncloud/common/storage_handlers.py @@ -6,8 +6,7 @@ import stat from abc import ABC from . import logger from os.path import join as join_path - -from uncloud.common.settings import settings as config +import uncloud.common.shared as shared class ImageStorageHandler(ABC): @@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): def get_storage_handler(): - __storage_backend = config["storage"]["storage_backend"] + __storage_backend = shared.shared.settings["storage"]["storage_backend"] if __storage_backend == "filesystem": return FileSystemBasedImageStorageHandler( - vm_base=config["storage"]["vm_dir"], - image_base=config["storage"]["image_dir"], + vm_base=shared.shared.settings["storage"]["vm_dir"], + image_base=shared.shared.settings["storage"]["image_dir"], ) elif __storage_backend == "ceph": return CEPHBasedImageStorageHandler( - vm_base=config["storage"]["ceph_vm_pool"], - image_base=config["storage"]["ceph_image_pool"], + vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], + image_base=shared.shared.settings["storage"]["ceph_image_pool"], ) else: - raise Exception("Unknown Image Storage Handler") + raise Exception("Unknown Image Storage Handler") \ No newline at end of file diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index e190460..87f5752 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,7 +1,6 @@ import os import argparse -from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('configure', add_help=False) @@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True) def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) + uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) + shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) -def main(**kwargs): - subcommand = kwargs.pop('subcommand') +def main(arguments): + subcommand = arguments['subcommand'] if not subcommand: arg_parser.print_help() else: - update_config(subcommand, kwargs) + update_config(subcommand, arguments) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index c5660dd..046f915 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -9,7 +9,6 @@ import bitmath from uuid import uuid4 from . import logger -from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('filescanner', add_help=False) @@ -53,7 +52,7 @@ def track_file(file, base_dir, host): file_path = file_path.relative_to(owner) creation_date = time.ctime(os.stat(file_str).st_ctime) - entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) + entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) entry_value = { 'filename': str(file_path), 'owner': owner, @@ -70,7 +69,7 @@ def track_file(file, base_dir, host): def main(arguments): hostname = arguments['hostname'] - base_dir = settings['storage']['file_dir'] + base_dir = shared.settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] @@ -78,7 +77,7 @@ def main(arguments): # Files that are already tracked tracked_files = [ pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) - for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) + for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) if f.value['host'] == hostname ] untracked_files = set(files) - set(tracked_files) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index ccffd77..f680991 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -6,7 +6,6 @@ from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared -from uncloud.common.settings import settings from uncloud.common.vm import VMStatus from uncloud.vmm import VMM from os.path import join as join_path @@ -36,7 +35,7 @@ def maintenance(host): if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], vm_uuid) + join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) ) vm.status = VMStatus.running vm.vnc_socket = vmm.get_vnc(vm_uuid) @@ -52,7 +51,7 @@ def main(arguments): # Does not yet exist, create it if not host: host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': '', @@ -80,9 +79,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for events_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False) ]: for request_event in events_iterator: @@ -95,7 +94,7 @@ def main(arguments): shared.request_pool.client.client.delete(request_event.key) vm_entry = shared.etcd_client.get( - join_path(settings['etcd']['vm_prefix'], request_event.uuid) + join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) ) logger.debug('VM hostname: {}'.format(vm_entry.value)) diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index 2f6a5e3..a592efc 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -17,7 +17,6 @@ from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger from uncloud.common.shared import shared -from uncloud.common.settings import settings from uncloud.vmm import VMM from marshmallow import ValidationError @@ -91,7 +90,7 @@ class VM: self.vmm.socket_dir, self.uuid ), destination_host_key=destination_host_key, # Where source host transfer VM - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) else: @@ -119,7 +118,7 @@ class VM: network_name, mac, tap = network_mac_and_tap _key = os.path.join( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], self.vm["owner"], network_name, ) @@ -133,13 +132,13 @@ class VM: if network["type"] == "vxlan": tap = create_vxlan_br_tap( _id=network["id"], - _dev=settings["network"]["vxlan_phy_dev"], + _dev=shared.settings["network"]["vxlan_phy_dev"], tap_id=tap, ip=network["ipv6"], ) all_networks = shared.etcd_client.get_prefix( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], value_in_json=True, ) @@ -229,7 +228,7 @@ class VM: def resolve_network(network_name, network_owner): network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], network_owner, network_name, ), diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index 1803213..ee9da2e 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -4,7 +4,6 @@ import argparse import subprocess as sp from os.path import join as join_path -from uncloud.common.settings import settings from uncloud.common.shared import shared from uncloud.imagescanner import logger @@ -33,7 +32,7 @@ def qemu_img_type(path): def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) images_to_be_created = list( filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) @@ -46,13 +45,13 @@ def main(arguments): image_filename = image.value["filename"] image_store_name = image.value["store_name"] image_full_path = join_path( - settings["storage"]["file_dir"], + shared.settings["storage"]["file_dir"], image_owner, image_filename, ) image_stores = shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], + shared.settings["etcd"]["image_store_prefix"], value_in_json=True, ) user_image_store = next( diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index c47364e..374260e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -5,7 +5,6 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from uncloud.common.settings import settings from uncloud.common.shared import shared app = Flask(__name__) @@ -74,7 +73,7 @@ class Root(Resource): ) else: etcd_key = os.path.join( - settings["etcd"]["user_prefix"], + shared.settings["etcd"]["user_prefix"], data.value["owner_realm"], data.value["owner"], "key", diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index 108d126..79db322 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -7,7 +7,6 @@ from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from uncloud.common.settings import settings def accumulated_specs(vms_specs): @@ -130,7 +129,7 @@ def assign_host(vm): type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index c25700b..38c07bf 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -6,7 +6,6 @@ import argparse -from uncloud.common.settings import settings from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared from uncloud.scheduler import logger @@ -24,9 +23,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for request_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), ]: for request_event in request_iterator: From 091131d3509ecae41aedacc3788e7c166f623b99 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 13 Jan 2020 11:52:40 +0100 Subject: [PATCH 123/163] dummy --- uncloud/hack/hackcloud/mac-last | 2 +- uncloud/vmm/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last index 59f6410..8c5f254 100644 --- a/uncloud/hack/hackcloud/mac-last +++ b/uncloud/hack/hackcloud/mac-last @@ -1 +1 @@ -000000000251 +000000000252 diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 4c893f6..719bdbe 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -100,9 +100,9 @@ class TransferVM(Process): class VMM: # Virtual Machine Manager def __init__( - self, - qemu_path="/usr/bin/qemu-system-x86_64", - vmm_backend=os.path.expanduser("~/uncloud/vmm/"), + self, + qemu_path="/usr/bin/qemu-system-x86_64", + vmm_backend=os.path.expanduser("~/uncloud/vmm/"), ): self.qemu_path = qemu_path self.vmm_backend = vmm_backend From 10c8dc85ba58398203046c4c303689c8d3e45bd5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 13 Jan 2020 12:14:30 +0100 Subject: [PATCH 124/163] Begin hacky database handling --- uncloud/hack/hackcloud/db.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 uncloud/hack/hackcloud/db.py diff --git a/uncloud/hack/hackcloud/db.py b/uncloud/hack/hackcloud/db.py new file mode 100644 index 0000000..3d885e9 --- /dev/null +++ b/uncloud/hack/hackcloud/db.py @@ -0,0 +1,17 @@ +import etcd3 + + +if __name__ == '__main__': + endpoints = [ "https://etcd1.ungleich.ch:2379", + "!https://etcd2.ungleich.ch:2379", + "https://etcd3.ungleich.ch:2379" ] + + clients = [] + + for endpoint in endpoints: + client = etcd3.client(host=endpoint, + ca_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem", + cert_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem", + cert_key="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem") + + clients.append(client) From 9f02b31b1b2035cda0fb663781add044990d942b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 13 Jan 2020 12:54:02 +0100 Subject: [PATCH 125/163] Add hacky etcd client --- uncloud/hack/hackcloud/etcd-client.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 uncloud/hack/hackcloud/etcd-client.sh diff --git a/uncloud/hack/hackcloud/etcd-client.sh b/uncloud/hack/hackcloud/etcd-client.sh new file mode 100644 index 0000000..ab102a5 --- /dev/null +++ b/uncloud/hack/hackcloud/etcd-client.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \ + --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \ + --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \ + --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@" From b96e56b453bff53898bb105560881c88ffd63218 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 11:05:42 +0100 Subject: [PATCH 126/163] Begin to integrate hack into the main script --- scripts/uncloud | 3 +- uncloud/hack/__init__.py | 0 uncloud/hack/hackcloud/__init__.py | 1 + uncloud/hack/hackcloud/db.py | 55 ++++++++++++++++++++++++------ uncloud/hack/hackcloud/vm.py | 53 ++++++++++++++++++++++++++++ uncloud/hack/hackcloud/vm.sh | 6 ++++ uncloud/hack/main.py | 10 ++++++ 7 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 uncloud/hack/__init__.py create mode 100644 uncloud/hack/hackcloud/__init__.py create mode 100755 uncloud/hack/hackcloud/vm.py create mode 100644 uncloud/hack/main.py diff --git a/scripts/uncloud b/scripts/uncloud index 1a6483b..70cb535 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -22,7 +22,8 @@ def exception_hook(exc_type, exc_value, exc_traceback): sys.excepthook = exception_hook # the components that use etcd -ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', + 'imagescanner', 'metadata', 'configure', 'hack'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() ALL_COMPONENTS.append('cli') diff --git a/uncloud/hack/__init__.py b/uncloud/hack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/hack/hackcloud/__init__.py b/uncloud/hack/hackcloud/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/uncloud/hack/hackcloud/__init__.py @@ -0,0 +1 @@ + diff --git a/uncloud/hack/hackcloud/db.py b/uncloud/hack/hackcloud/db.py index 3d885e9..0e6bd0a 100644 --- a/uncloud/hack/hackcloud/db.py +++ b/uncloud/hack/hackcloud/db.py @@ -1,17 +1,52 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + import etcd3 +class DB(object): + def __init__(self, urls): + self.urls = urls + self.prefix = "/nicohack/" + + def connect(self): + self.clients = [] + for endpoint in self.urls: + client = etcd3.client(host=endpoint, + ca_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem", + cert_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem", + cert_key="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem") + + clients.append(client) + + def get_value(self, key): + pass + + def set_value(self, key, val): + pass + if __name__ == '__main__': endpoints = [ "https://etcd1.ungleich.ch:2379", - "!https://etcd2.ungleich.ch:2379", + "https://etcd2.ungleich.ch:2379", "https://etcd3.ungleich.ch:2379" ] - clients = [] - - for endpoint in endpoints: - client = etcd3.client(host=endpoint, - ca_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem", - cert_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem", - cert_key="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem") - - clients.append(client) + db = DB(url=endpoints) diff --git a/uncloud/hack/hackcloud/vm.py b/uncloud/hack/hackcloud/vm.py new file mode 100755 index 0000000..9dd80bf --- /dev/null +++ b/uncloud/hack/hackcloud/vm.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +import subprocess +import uuid + +from . import db + +qemu="/usr/bin/qemu-system-x86_64" +accel="kvm" +memory=1024 +cores=2 +uuid=uuid.uuid4() + +#mac=$(./mac-gen.py) +mac="" + +owner="nico" + +bridge="br100" + +if __name__ == '__main__': + p = ["qemu", + "-name", "uncloud-{}".format(uuid), + "-machine", "pc,accel={}".format(accel), + "-m", "{}".format(memory), + "-smp", "{}".format(cores), + "-uuid", "{}".format(uuid), + "-drive", "file=alpine-virt-3.11.2-x86_64.iso,media=cdrom", + "-netdev", "tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh", + "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(mac) + ] + print(" ".join(p)) + subprocess.run(p) diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index a0e111b..dd9be84 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -1,5 +1,10 @@ #!/bin/sh +# if [ $# -ne 1 ]; then +# echo "$0: owner" +# exit 1 +# fi + qemu=/usr/bin/qemu-system-x86_64 accel=kvm @@ -9,6 +14,7 @@ memory=1024 cores=2 uuid=$(uuidgen) mac=$(./mac-gen.py) +owner=nico export bridge=br100 diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py new file mode 100644 index 0000000..ce105e8 --- /dev/null +++ b/uncloud/hack/main.py @@ -0,0 +1,10 @@ +import argparse + +arg_parser = argparse.ArgumentParser('hack', add_help=False) +arg_parser.add_argument('--create-vm') + + +def main(arguments): + print(arguments)! + debug = arguments['debug'] + port = arguments['port'] From 22531a7459e1b1b3da7de9b765daddf2d483f5bd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 11:09:45 +0100 Subject: [PATCH 127/163] Disable cli / otp reading for the moment Imho this should clearly not leak into scripts/uncloud and additionally it is broken at the moment --- scripts/uncloud | 6 +++--- uncloud/hack/main.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 70cb535..263d99e 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -26,7 +26,7 @@ ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure', 'hack'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('cli') +#ALL_COMPONENTS.append('cli') if __name__ == '__main__': @@ -77,8 +77,8 @@ if __name__ == '__main__': # we read from file. But, now we are asking user about where the config file lives. So, # to providing default value is not possible before parsing arguments. So, we are doing # it after.. - settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) - resolve_otp_credentials(arguments) +# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) +# resolve_otp_credentials(arguments) name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index ce105e8..2ce19da 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -5,6 +5,6 @@ arg_parser.add_argument('--create-vm') def main(arguments): - print(arguments)! - debug = arguments['debug'] - port = arguments['port'] + print(arguments) + #debug = arguments['debug'] + #port = arguments['port'] From 083ba439183cbedb1baf30a5dfcc0f4da5e65d24 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 11:22:04 +0100 Subject: [PATCH 128/163] Integrate hack + vm create into python code --- uncloud/hack/hackcloud/vm.py | 53 --------------------------------- uncloud/hack/main.py | 9 +++++- uncloud/hack/vm.py | 57 ++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 54 deletions(-) delete mode 100755 uncloud/hack/hackcloud/vm.py create mode 100755 uncloud/hack/vm.py diff --git a/uncloud/hack/hackcloud/vm.py b/uncloud/hack/hackcloud/vm.py deleted file mode 100755 index 9dd80bf..0000000 --- a/uncloud/hack/hackcloud/vm.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) -# -# This file is part of uncloud. -# -# uncloud is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# uncloud is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with uncloud. If not, see . -# -# - -import subprocess -import uuid - -from . import db - -qemu="/usr/bin/qemu-system-x86_64" -accel="kvm" -memory=1024 -cores=2 -uuid=uuid.uuid4() - -#mac=$(./mac-gen.py) -mac="" - -owner="nico" - -bridge="br100" - -if __name__ == '__main__': - p = ["qemu", - "-name", "uncloud-{}".format(uuid), - "-machine", "pc,accel={}".format(accel), - "-m", "{}".format(memory), - "-smp", "{}".format(cores), - "-uuid", "{}".format(uuid), - "-drive", "file=alpine-virt-3.11.2-x86_64.iso,media=cdrom", - "-netdev", "tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh", - "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(mac) - ] - print(" ".join(p)) - subprocess.run(p) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 2ce19da..4baed98 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -1,10 +1,17 @@ import argparse +from uncloud.hack.vm import VM + arg_parser = argparse.ArgumentParser('hack', add_help=False) -arg_parser.add_argument('--create-vm') +arg_parser.add_argument('--create-vm', action='store_true') def main(arguments): print(arguments) + if arguments['create_vm']: + print("Creating VM") + vm = VM() + vm.create() + #debug = arguments['debug'] #port = arguments['port'] diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py new file mode 100755 index 0000000..988ea2b --- /dev/null +++ b/uncloud/hack/vm.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +import subprocess +import uuid +import os + +class VM(object): + def __init__(self): + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" + + self.qemu="/usr/bin/qemu-system-x86_64" + self.accel="kvm" + self.memory=1024 + self.cores=2 + self.uuid=uuid.uuid4() +# self.mac=$(./mac-gen.py) + self.mac="42:00:00:00:00:42" + self.owner="nico" + self.bridge="br100" + self.os_image = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + + def create(self): + p = [ "sudo", + "{}".format(self.qemu), + "-name", "uncloud-{}".format(self.uuid), + "-machine", "pc,accel={}".format(self.accel), + "-m", "{}".format(self.memory), + "-smp", "{}".format(self.cores), + "-uuid", "{}".format(self.uuid), + "-drive", "file={},media=cdrom".format(self.os_image), + "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), + "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) + ] + print(" ".join(p)) + subprocess.run(p) From c0e6d6a0d85dddecccd93f4f20b47b2a3c62f177 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 11:25:06 +0100 Subject: [PATCH 129/163] Begin further integration of code into hack --- uncloud/hack/{hackcloud => }/db.py | 0 uncloud/hack/{hackcloud/mac-gen.py => mac.py} | 0 uncloud/hack/main.py | 1 + 3 files changed, 1 insertion(+) rename uncloud/hack/{hackcloud => }/db.py (100%) rename uncloud/hack/{hackcloud/mac-gen.py => mac.py} (100%) diff --git a/uncloud/hack/hackcloud/db.py b/uncloud/hack/db.py similarity index 100% rename from uncloud/hack/hackcloud/db.py rename to uncloud/hack/db.py diff --git a/uncloud/hack/hackcloud/mac-gen.py b/uncloud/hack/mac.py similarity index 100% rename from uncloud/hack/hackcloud/mac-gen.py rename to uncloud/hack/mac.py diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 4baed98..2e1e9d5 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -3,6 +3,7 @@ import argparse from uncloud.hack.vm import VM arg_parser = argparse.ArgumentParser('hack', add_help=False) + #description="Commands that are unfinished - use at own risk") arg_parser.add_argument('--create-vm', action='store_true') From 1b36c2f96f945e317e5ef2cec2a5b00d6194ab35 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 14:23:26 +0100 Subject: [PATCH 130/163] Write VM to etcd --- scripts/uncloud | 6 +++--- uncloud/hack/config.py | 39 +++++++++++++++++++++++++++++++++++ uncloud/hack/db.py | 29 ++++++++++++++------------ uncloud/hack/main.py | 8 +++---- uncloud/hack/vm.py | 47 ++++++++++++++++++++++++++++-------------- 5 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 uncloud/hack/config.py diff --git a/scripts/uncloud b/scripts/uncloud index 263d99e..ab5b40d 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -44,7 +44,7 @@ if __name__ == '__main__': default=os.path.expanduser('~/uncloud')) etcd_parser = argparse.ArgumentParser(add_help=False) - etcd_parser.add_argument('--etcd-host', dest='etcd_url') + etcd_parser.add_argument('--etcd-host') etcd_parser.add_argument('--etcd-port') etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') @@ -88,7 +88,7 @@ if __name__ == '__main__': main(arguments) except UncloudException as err: logger.error(err) - except ConnectionFailedError: - logger.error('Cannot connect to etcd') +# except ConnectionFailedError as err: +# logger.error('Cannot connect to etcd: {}'.format(err)) except Exception as err: logger.exception(err) diff --git a/uncloud/hack/config.py b/uncloud/hack/config.py new file mode 100644 index 0000000..7e2655d --- /dev/null +++ b/uncloud/hack/config.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +class Config(object): + def __init__(self, arguments): + """ read arguments dicts as a base """ + + self.arguments = arguments + + # Split them so *etcd_args can be used and we can + # iterate over etcd_hosts + self.etcd_hosts = [ arguments['etcd_host'] ] + self.etcd_args = { + 'ca_cert': arguments['etcd_ca_cert'], + 'cert_cert': arguments['etcd_cert_cert'], + 'cert_key': arguments['etcd_cert_key'], +# 'user': None, +# 'password': None + } + self.etcd_prefix = '/nicohack/' diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py index 0e6bd0a..be0342a 100644 --- a/uncloud/hack/db.py +++ b/uncloud/hack/db.py @@ -21,28 +21,31 @@ # import etcd3 +import json class DB(object): - def __init__(self, urls): - self.urls = urls - self.prefix = "/nicohack/" + def __init__(self, config): + self.config = config + self.prefix= '/nicohack/' + self.connect() def connect(self): - self.clients = [] - for endpoint in self.urls: - client = etcd3.client(host=endpoint, - ca_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem", - cert_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem", - cert_key="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem") - - clients.append(client) + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) def get_value(self, key): pass - def set_value(self, key, val): - pass + def set(self, key, value, store_as_json=False, **kwargs): + if store_as_json: + value = json.dumps(value) + key = "{}/{}".format(self.prefix, key) + + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(key, value, **kwargs) if __name__ == '__main__': endpoints = [ "https://etcd1.ungleich.ch:2379", diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 2e1e9d5..df618c6 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -1,6 +1,7 @@ import argparse from uncloud.hack.vm import VM +from uncloud.hack.config import Config arg_parser = argparse.ArgumentParser('hack', add_help=False) #description="Commands that are unfinished - use at own risk") @@ -9,10 +10,9 @@ arg_parser.add_argument('--create-vm', action='store_true') def main(arguments): print(arguments) + config = Config(arguments) + if arguments['create_vm']: print("Creating VM") - vm = VM() + vm = VM(config) vm.create() - - #debug = arguments['debug'] - #port = arguments['port'] diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 988ea2b..e33e473 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -24,34 +24,49 @@ import subprocess import uuid import os -class VM(object): - def __init__(self): - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" +from uncloud.hack.db import DB +class VM(object): + def __init__(self, config): + self.config = config + self.db = DB(config) + + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" self.qemu="/usr/bin/qemu-system-x86_64" + + self.vm = {} + self.accel="kvm" - self.memory=1024 - self.cores=2 - self.uuid=uuid.uuid4() + # self.mac=$(./mac-gen.py) self.mac="42:00:00:00:00:42" self.owner="nico" self.bridge="br100" - self.os_image = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + self.ifup = os.path.join(self.hackprefix, "ifup.sh") self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") - def create(self): - p = [ "sudo", + self.uuid = uuid.uuid4() + self.vm['uuid'] = str(self.uuid) + self.vm['memory']=1024 + self.vm['cores']=2 + self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + + self.vm['commandline' ] = [ "sudo", "{}".format(self.qemu), - "-name", "uncloud-{}".format(self.uuid), + "-name", "uncloud-{}".format(self.vm['uuid']), "-machine", "pc,accel={}".format(self.accel), - "-m", "{}".format(self.memory), - "-smp", "{}".format(self.cores), - "-uuid", "{}".format(self.uuid), - "-drive", "file={},media=cdrom".format(self.os_image), + "-m", "{}".format(self.vm['memory']), + "-smp", "{}".format(self.vm['cores']), + "-uuid", "{}".format(self.vm['uuid']), + "-drive", "file={},media=cdrom".format(self.vm['os_image']), "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] - print(" ".join(p)) - subprocess.run(p) + + def create(self): + self.db.set("vm/{}".format(str(self.vm['uuid'])), + self.vm, store_as_json=True) + + print(" ".join(self.vm['commandline'])) + subprocess.run(self.vm['commandline']) From 8078ffae5a379f338c1e65f4acbb0832a73454f5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 19:02:15 +0100 Subject: [PATCH 131/163] Add working --last-used-mac {'create_vm': False, 'last_used_mac': True, 'get_new_mac': False, 'debug': False, 'conf_dir': '/home/nico/uncloud', 'etcd_host': 'etcd1.ungleich.ch', 'etcd_port': None, 'etcd_ca_cert': '/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem', 'etcd_cert_cert': '/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem', 'etcd_cert_key': '/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem'} 00:20:00:00:00:00 (venv) [19:02] diamond:uncloud% ./bin/uncloud-run-reinstall hack --etcd-host etcd1.ungleich.ch --etcd-ca-cert /home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem --etcd-cert-cert /home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem --etcd-cert-key /home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem --last-used-mac --- uncloud/hack/db.py | 63 ++++++++++++++++++++++++++----- uncloud/hack/mac.py | 90 +++++++++++++++----------------------------- uncloud/hack/main.py | 11 ++++++ uncloud/hack/vm.py | 70 +++++++++++++++++----------------- 4 files changed, 132 insertions(+), 102 deletions(-) diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py index be0342a..ac643bd 100644 --- a/uncloud/hack/db.py +++ b/uncloud/hack/db.py @@ -22,30 +22,75 @@ import etcd3 import json +import logging + +from functools import wraps +from uncloud import UncloudException + +log = logging.getLogger(__name__) + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + class DB(object): - def __init__(self, config): + def __init__(self, config, prefix="/"): self.config = config - self.prefix= '/nicohack/' + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + self.connect() + @readable_errors def connect(self): self._db_clients = [] for endpoint in self.config.etcd_hosts: client = etcd3.client(host=endpoint, **self.config.etcd_args) self._db_clients.append(client) - def get_value(self, key): - pass + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) - def set(self, key, value, store_as_json=False, **kwargs): - if store_as_json: + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: value = json.dumps(value) - key = "{}/{}".format(self.prefix, key) - # FIXME: iterate over clients in case of failure ? - return self._db_clients[0].put(key, value, **kwargs) + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + @readable_errors + def increment(key, **kwargs): + with self._db_clients[0].lock(key) as lock: + value = int(self.get(self.realkey(key), **kwargs)) + self.set(self.realkey(key), str(value + 1), **kwargs) + if __name__ == '__main__': endpoints = [ "https://etcd1.ungleich.ch:2379", diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index e2b4bc5..4ac05f2 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -25,93 +25,65 @@ import logging import os.path import os import re +import json + +from uncloud import UncloudException +from uncloud.hack.db import DB log = logging.getLogger(__name__) -class Error(Exception): - pass - -class Mac(object): - def __init__(self): - self.base_dir = "." +class MAC(object): + def __init__(self, config): + self.config = config + self.db = DB(config, prefix="/mac") self.prefix = 0x002000000000 - #self.prefix = "{:012x}".format(self._prefix) - - self.free = self.read_file("mac-free") - self.last = self.read_file("mac-last") - - def read_file(self, filename): - fname = os.path.join(self.base_dir, filename) - ret = [] - - try: - with open(fname, "r") as fd: - ret = fd.readlines() - except Exception as e: - pass - - return ret - - def append_to_file(self, text, filename): - fname = os.path.join(self.base_dir, filename) - with open(fname, "a+") as fd: - fd.write("{}\n".format(text)) @staticmethod def validate_mac(mac): if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): raise Error("Not a valid mac address: %s" % mac) - def free_append(self, mac): - if mac in self.free: - raise Error("Mac already in free database: %s" % mac) - - self.append_to_file(mac, "mac-free") - self.free = self.read_file("mac-free") + def last_used_index(self): + value = self.db.get("last_used_index") + if not value: + return 0 + return int(value) + def last_used_mac(self): + return self.int_to_mac(self.prefix + self.last_used_index()) @staticmethod def int_to_mac(number): b = number.to_bytes(6, byteorder="big") return ':'.join(format(s, '02x') for s in b) - def getnext(self): + def get_next(self, vmuuid=None, as_int=False): # if self.free: # return self.free.pop() -# if not self.prefix: -# raise Error("Cannot generate address without prefix - use prefix-set") + last_number = self.last_used_index() - if self.last: - last_number = int(self.last[0], 16) - - if last_number == int('0xffffff', 16): - raise Error("Exhausted all possible mac addresses - try to free some") - - next_number = last_number + 1 - else: - next_number = 0 + # FIXME: compare to 48bit minus prefix length + if last_number == int('0xffffff', 16): + raise UncloudException("Exhausted all possible mac addresses - try to free some") + next_number = last_number + 1 next_number_string = "{:012x}".format(next_number) next_mac_number = self.prefix + next_number next_mac = self.int_to_mac(next_mac_number) - with open(os.path.join(self.base_dir, "mac-last"), "w+") as fd: - fd.write("{}\n".format(next_number_string)) + db_entry = {} + db_entry['vm_uuid'] = vmuuid + db_entry['index'] = next_number + db_entry['mac_address'] = next_mac - return next_mac + self.db.set("used/{}".format(next_mac), + db_entry) - @classmethod - def commandline(cls): - pass - - -if __name__ == '__main__': - m = Mac() - m.commandline() - # print(m.free) - #print(m.last) - print(m.getnext()) + if as_int: + return next_mac_number + else: + return next_mac diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index df618c6..ffd0374 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -2,10 +2,13 @@ import argparse from uncloud.hack.vm import VM from uncloud.hack.config import Config +from uncloud.hack.mac import MAC arg_parser = argparse.ArgumentParser('hack', add_help=False) #description="Commands that are unfinished - use at own risk") arg_parser.add_argument('--create-vm', action='store_true') +arg_parser.add_argument('--last-used-mac', action='store_true') +arg_parser.add_argument('--get-new-mac', action='store_true') def main(arguments): @@ -16,3 +19,11 @@ def main(arguments): print("Creating VM") vm = VM(config) vm.create() + + if arguments['last_used_mac']: + m = MAC(config) + print(m.last_used_mac()) + + if arguments['get_new_mac']: + m = MAC(config).get_next() + print(m.last_used()) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index e33e473..eb75902 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -25,48 +25,50 @@ import uuid import os from uncloud.hack.db import DB +from uncloud.hack.mac import MAC class VM(object): - def __init__(self, config): - self.config = config - self.db = DB(config) + def __init__(self, config): + self.config = config + self.db = DB(config, prefix="/vm") - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" - self.qemu="/usr/bin/qemu-system-x86_64" + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" + self.qemu="/usr/bin/qemu-system-x86_64" + self.accel="kvm" - self.vm = {} + self.vm = {} - self.accel="kvm" -# self.mac=$(./mac-gen.py) - self.mac="42:00:00:00:00:42" - self.owner="nico" - self.bridge="br100" + self.owner="nico" + self.bridge="br100" - self.ifup = os.path.join(self.hackprefix, "ifup.sh") - self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") - self.uuid = uuid.uuid4() - self.vm['uuid'] = str(self.uuid) - self.vm['memory']=1024 - self.vm['cores']=2 - self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + def create(self): + self.uuid = uuid.uuid4() + self.vm['uuid'] = str(self.uuid) + self.vm['memory'] = 1024 + self.vm['cores'] = 2 + self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") - self.vm['commandline' ] = [ "sudo", - "{}".format(self.qemu), - "-name", "uncloud-{}".format(self.vm['uuid']), - "-machine", "pc,accel={}".format(self.accel), - "-m", "{}".format(self.vm['memory']), - "-smp", "{}".format(self.vm['cores']), - "-uuid", "{}".format(self.vm['uuid']), - "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), - "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) - ] + self.mac=MAC().next() - def create(self): - self.db.set("vm/{}".format(str(self.vm['uuid'])), - self.vm, store_as_json=True) + self.vm['commandline' ] = [ "sudo", + "{}".format(self.qemu), + "-name", "uncloud-{}".format(self.vm['uuid']), + "-machine", "pc,accel={}".format(self.accel), + "-m", "{}".format(self.vm['memory']), + "-smp", "{}".format(self.vm['cores']), + "-uuid", "{}".format(self.vm['uuid']), + "-drive", "file={},media=cdrom".format(self.vm['os_image']), + "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), + "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) + ] - print(" ".join(self.vm['commandline'])) - subprocess.run(self.vm['commandline']) + self.db.set(str(self.vm['uuid']), + self.vm, + as_json=True) + + print(" ".join(self.vm['commandline'])) + subprocess.run(self.vm['commandline']) From 12e8ccd01c62b8dab8f20cb8ff624c5d1d8aac1c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 19:10:59 +0100 Subject: [PATCH 132/163] Cleanups for mac handling --- uncloud/hack/mac.py | 5 +---- uncloud/hack/main.py | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index 4ac05f2..a408103 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -60,9 +60,6 @@ class MAC(object): return ':'.join(format(s, '02x') for s in b) def get_next(self, vmuuid=None, as_int=False): -# if self.free: -# return self.free.pop() - last_number = self.last_used_index() # FIXME: compare to 48bit minus prefix length @@ -81,7 +78,7 @@ class MAC(object): db_entry['mac_address'] = next_mac self.db.set("used/{}".format(next_mac), - db_entry) + db_entry, as_json=True) if as_int: return next_mac_number diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index ffd0374..2980516 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -25,5 +25,4 @@ def main(arguments): print(m.last_used_mac()) if arguments['get_new_mac']: - m = MAC(config).get_next() - print(m.last_used()) + print(MAC(config).get_next()) From b877ab13b34b058540fc10fabde67501be1b79f8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 10:02:37 +0100 Subject: [PATCH 133/163] add hack code --- uncloud/hack/db.py | 21 +++++++++++++++++---- uncloud/hack/mac.py | 8 ++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py index ac643bd..cb5e490 100644 --- a/uncloud/hack/db.py +++ b/uncloud/hack/db.py @@ -86,10 +86,23 @@ class DB(object): return self._db_clients[0].put(self.realkey(key), value, **kwargs) @readable_errors - def increment(key, **kwargs): - with self._db_clients[0].lock(key) as lock: - value = int(self.get(self.realkey(key), **kwargs)) - self.set(self.realkey(key), str(value + 1), **kwargs) + def increment(self, key, **kwargs): + print(self.realkey(key)) + + + print("prelock") + lock = self._db_clients[0].lock('/nicohack/foo') + print("prelockacq") + lock.acquire() + print("prelockrelease") + lock.release() + + with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock: + print("in lock") + pass + +# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs)) +# self.set(self.realkey(key), str(value + 1), **kwargs) if __name__ == '__main__': diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index a408103..e7f41a2 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -48,7 +48,9 @@ class MAC(object): def last_used_index(self): value = self.db.get("last_used_index") if not value: - return 0 + self.db.set("last_used_index", "0") + value = self.db.get("last_used_index") + return int(value) def last_used_mac(self): @@ -62,7 +64,7 @@ class MAC(object): def get_next(self, vmuuid=None, as_int=False): last_number = self.last_used_index() - # FIXME: compare to 48bit minus prefix length + # FIXME: compare to 48bit minus prefix length to the power of 2 if last_number == int('0xffffff', 16): raise UncloudException("Exhausted all possible mac addresses - try to free some") @@ -77,6 +79,8 @@ class MAC(object): db_entry['index'] = next_number db_entry['mac_address'] = next_mac + # should be one transaction + self.db.increment("last_used_index") self.db.set("used/{}".format(next_mac), db_entry, as_json=True) From 26d5c916256ccf91c99de92ae6a80353d58b4720 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 10:53:22 +0100 Subject: [PATCH 134/163] Update hacking docs --- uncloud/docs/source/hacking.rst | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/uncloud/docs/source/hacking.rst b/uncloud/docs/source/hacking.rst index 2df42a7..d198126 100644 --- a/uncloud/docs/source/hacking.rst +++ b/uncloud/docs/source/hacking.rst @@ -1,17 +1,25 @@ Hacking ======= -How to hack on the code. +Using uncloud in hacking (aka development) mode. -[ to be done by Balazs: -* make nice -* indent with shell script mode +Get the code +------------ +.. code-block:: sh + :linenos: -] + git clone https://code.ungleich.ch/uncloud/uncloud.git -* git clone the repo -* cd to the repo -* Setup your venv: python -m venv venv -* . ./venv/bin/activate # you need the leading dot for sourcing! -* Run ./bin/ucloud-run-reinstall - it should print you an error - message on how to use ucloud + + +Install python requirements +--------------------------- +You need to have python3 installed. + +.. code-block:: sh + :linenos: + + cd uncloud! + python -m venv venv + . ./venv/bin/activate + ./bin/uncloud-run-reinstall From bd03f95e9925589375d30e60b0dc4b1960dae6ff Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 11:32:23 +0100 Subject: [PATCH 135/163] [docs] move one level higher --- {uncloud/docs => docs}/Makefile | 2 +- {uncloud/docs => docs}/README.md | 0 {uncloud/docs => docs}/__init__.py | 0 {uncloud/docs => docs}/source/__init__.py | 0 {uncloud/docs => docs}/source/admin-guide | 0 {uncloud/docs => docs}/source/conf.py | 0 {uncloud/docs => docs}/source/diagram-code/ucloud | 0 {uncloud/docs => docs}/source/hacking.rst | 0 {uncloud/docs => docs}/source/images/ucloud.svg | 0 {uncloud/docs => docs}/source/index.rst | 0 {uncloud/docs => docs}/source/introduction.rst | 0 {uncloud/docs => docs}/source/misc/todo.rst | 0 {uncloud/docs => docs}/source/setup-install.rst | 0 {uncloud/docs => docs}/source/theory/summary.rst | 0 {uncloud/docs => docs}/source/troubleshooting.rst | 0 {uncloud/docs => docs}/source/user-guide.rst | 0 .../source/user-guide/how-to-create-an-os-image-for-ucloud.rst | 0 17 files changed, 1 insertion(+), 1 deletion(-) rename {uncloud/docs => docs}/Makefile (93%) rename {uncloud/docs => docs}/README.md (100%) rename {uncloud/docs => docs}/__init__.py (100%) rename {uncloud/docs => docs}/source/__init__.py (100%) rename {uncloud/docs => docs}/source/admin-guide (100%) rename {uncloud/docs => docs}/source/conf.py (100%) rename {uncloud/docs => docs}/source/diagram-code/ucloud (100%) rename {uncloud/docs => docs}/source/hacking.rst (100%) rename {uncloud/docs => docs}/source/images/ucloud.svg (100%) rename {uncloud/docs => docs}/source/index.rst (100%) rename {uncloud/docs => docs}/source/introduction.rst (100%) rename {uncloud/docs => docs}/source/misc/todo.rst (100%) rename {uncloud/docs => docs}/source/setup-install.rst (100%) rename {uncloud/docs => docs}/source/theory/summary.rst (100%) rename {uncloud/docs => docs}/source/troubleshooting.rst (100%) rename {uncloud/docs => docs}/source/user-guide.rst (100%) rename {uncloud/docs => docs}/source/user-guide/how-to-create-an-os-image-for-ucloud.rst (100%) diff --git a/uncloud/docs/Makefile b/docs/Makefile similarity index 93% rename from uncloud/docs/Makefile rename to docs/Makefile index 5e7ea85..246b56c 100644 --- a/uncloud/docs/Makefile +++ b/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source/ BUILDDIR = build/ -DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/ +DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/ .PHONY: all build clean diff --git a/uncloud/docs/README.md b/docs/README.md similarity index 100% rename from uncloud/docs/README.md rename to docs/README.md diff --git a/uncloud/docs/__init__.py b/docs/__init__.py similarity index 100% rename from uncloud/docs/__init__.py rename to docs/__init__.py diff --git a/uncloud/docs/source/__init__.py b/docs/source/__init__.py similarity index 100% rename from uncloud/docs/source/__init__.py rename to docs/source/__init__.py diff --git a/uncloud/docs/source/admin-guide b/docs/source/admin-guide similarity index 100% rename from uncloud/docs/source/admin-guide rename to docs/source/admin-guide diff --git a/uncloud/docs/source/conf.py b/docs/source/conf.py similarity index 100% rename from uncloud/docs/source/conf.py rename to docs/source/conf.py diff --git a/uncloud/docs/source/diagram-code/ucloud b/docs/source/diagram-code/ucloud similarity index 100% rename from uncloud/docs/source/diagram-code/ucloud rename to docs/source/diagram-code/ucloud diff --git a/uncloud/docs/source/hacking.rst b/docs/source/hacking.rst similarity index 100% rename from uncloud/docs/source/hacking.rst rename to docs/source/hacking.rst diff --git a/uncloud/docs/source/images/ucloud.svg b/docs/source/images/ucloud.svg similarity index 100% rename from uncloud/docs/source/images/ucloud.svg rename to docs/source/images/ucloud.svg diff --git a/uncloud/docs/source/index.rst b/docs/source/index.rst similarity index 100% rename from uncloud/docs/source/index.rst rename to docs/source/index.rst diff --git a/uncloud/docs/source/introduction.rst b/docs/source/introduction.rst similarity index 100% rename from uncloud/docs/source/introduction.rst rename to docs/source/introduction.rst diff --git a/uncloud/docs/source/misc/todo.rst b/docs/source/misc/todo.rst similarity index 100% rename from uncloud/docs/source/misc/todo.rst rename to docs/source/misc/todo.rst diff --git a/uncloud/docs/source/setup-install.rst b/docs/source/setup-install.rst similarity index 100% rename from uncloud/docs/source/setup-install.rst rename to docs/source/setup-install.rst diff --git a/uncloud/docs/source/theory/summary.rst b/docs/source/theory/summary.rst similarity index 100% rename from uncloud/docs/source/theory/summary.rst rename to docs/source/theory/summary.rst diff --git a/uncloud/docs/source/troubleshooting.rst b/docs/source/troubleshooting.rst similarity index 100% rename from uncloud/docs/source/troubleshooting.rst rename to docs/source/troubleshooting.rst diff --git a/uncloud/docs/source/user-guide.rst b/docs/source/user-guide.rst similarity index 100% rename from uncloud/docs/source/user-guide.rst rename to docs/source/user-guide.rst diff --git a/uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst b/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst similarity index 100% rename from uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst rename to docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst From 8a451ff4ffd82ab382183eb1017704c8d4ea25d2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 12:40:37 +0100 Subject: [PATCH 136/163] [hack] phase in networking --- uncloud/hack/net.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 uncloud/hack/net.py diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py new file mode 100644 index 0000000..142eeb7 --- /dev/null +++ b/uncloud/hack/net.py @@ -0,0 +1,21 @@ +import subprocess + +class VXLANBridge(object): + def __init__(self, bridgedev=None, uplinkdev=None): + self.management_vni = 1 + + cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {netid} dstport 4789 group ff05::{netid} dev {uplinkdev} ttl 5" + cmd_up_dev = "ip link set {dev} up" + cmd_create_bridge="ip link add {bridgedev} type bridge" + cmd_add_to_bridge="ip link set {vxlandev} master {bridgedev} up" + cmd_add_addr="ip addr add {ip} dev {bridgedev}" + + def setup_networking(dev=wlan0, v6net): + ip=2a0a:e5c1:111:888::48/64 + vxlandev=vxlan${netid} + bridgedev=br${netid} + + +class DNSRA(object): + def __init__(self): + pass From 1b5a3f6d2ee71e75bdef9540ff204940b72a1f5c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 13:26:05 +0100 Subject: [PATCH 137/163] Progress with networking --- docs/source/hacking.rst | 11 +++++++++++ uncloud/hack/main.py | 10 ++++++++++ uncloud/hack/net.py | 4 ++++ 3 files changed, 25 insertions(+) diff --git a/docs/source/hacking.rst b/docs/source/hacking.rst index d198126..1c750d6 100644 --- a/docs/source/hacking.rst +++ b/docs/source/hacking.rst @@ -23,3 +23,14 @@ You need to have python3 installed. python -m venv venv . ./venv/bin/activate ./bin/uncloud-run-reinstall + + + +Install os requirements +----------------------- +Install the following software packages: **dnsmasq**. + +If you already have a working IPv6 SLAAC and DNS setup, +this step can be skipped. + +Note that you need at least one /64 IPv6 network to run uncloud. diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 2980516..d7a4714 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -3,12 +3,18 @@ import argparse from uncloud.hack.vm import VM from uncloud.hack.config import Config from uncloud.hack.mac import MAC +from uncloud import UncloudException arg_parser = argparse.ArgumentParser('hack', add_help=False) #description="Commands that are unfinished - use at own risk") arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--last-used-mac', action='store_true') arg_parser.add_argument('--get-new-mac', action='store_true') +arg_parser.add_argument('--management-network', help="IPv6 management network") +arg_parser.add_argument('--run-dns-ra', action='store_true', + help="Provide router advertisements and DNS resolution via dnsmasq") + + def main(arguments): @@ -26,3 +32,7 @@ def main(arguments): if arguments['get_new_mac']: print(MAC(config).get_next()) + + if arguments['run_dns_ra']: + if not arguments['management_network']: + raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network. You can use fd00::/64 for testing (non production!)") diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 142eeb7..0a7819b 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -19,3 +19,7 @@ class VXLANBridge(object): class DNSRA(object): def __init__(self): pass + + +class Firewall(object): + pass From b8472607684a7ca9c73f86296144fe83a6d5e4f4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 09:16:29 +0100 Subject: [PATCH 138/163] ++network --- uncloud/hack/main.py | 13 +++++++++++++ uncloud/hack/net.py | 6 ++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index d7a4714..4ccb74a 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -3,6 +3,8 @@ import argparse from uncloud.hack.vm import VM from uncloud.hack.config import Config from uncloud.hack.mac import MAC +from uncloud.hack.net import VXLANBridge, DNSRA + from uncloud import UncloudException arg_parser = argparse.ArgumentParser('hack', add_help=False) @@ -10,6 +12,8 @@ arg_parser = argparse.ArgumentParser('hack', add_help=False) arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--last-used-mac', action='store_true') arg_parser.add_argument('--get-new-mac', action='store_true') + +arg_parser.add_argument('--init-network', help="Initialise networking") arg_parser.add_argument('--management-network', help="IPv6 management network") arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") @@ -33,6 +37,15 @@ def main(arguments): if arguments['get_new_mac']: print(MAC(config).get_next()) + if arguments['init_networking!']: + if not arguments['management_network']: + raise UncloudException("Initialising the network requires an IPv6 network. You can use fd00::/64 for testing (non production!)") + vb = VXLANBridge(arguments['management_network']) + vb.setup() + if arguments['run_dns_ra']: if not arguments['management_network']: raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network. You can use fd00::/64 for testing (non production!)") + + dnsra = DNSRA(arguments['management_network']) + dnsra.setup() diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 0a7819b..11649b8 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -1,7 +1,10 @@ import subprocess +class ManagementBridge(VXLANBridge): + pass + class VXLANBridge(object): - def __init__(self, bridgedev=None, uplinkdev=None): + def __init__(self, vni, bridgedev=None, uplinkdev=None): self.management_vni = 1 cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {netid} dstport 4789 group ff05::{netid} dev {uplinkdev} ttl 5" @@ -11,7 +14,6 @@ class VXLANBridge(object): cmd_add_addr="ip addr add {ip} dev {bridgedev}" def setup_networking(dev=wlan0, v6net): - ip=2a0a:e5c1:111:888::48/64 vxlandev=vxlan${netid} bridgedev=br${netid} From 2b8831784a4d22ec8f20216ccb54139e3da98aeb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 11:30:16 +0100 Subject: [PATCH 139/163] [pep440] improve versioning name for python --- bin/gen-version | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/gen-version b/bin/gen-version index a2e2882..06c3e22 100755 --- a/bin/gen-version +++ b/bin/gen-version @@ -1,22 +1,22 @@ #!/bin/sh # -*- coding: utf-8 -*- # -# 2019 Nico Schottelius (nico-ucloud at schottelius.org) +# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org) # -# This file is part of ucloud. +# This file is part of uncloud. # -# ucloud is free software: you can redistribute it and/or modify +# uncloud is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# ucloud is distributed in the hope that it will be useful, +# uncloud is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with ucloud. If not, see . +# along with uncloud. If not, see . # # @@ -26,4 +26,4 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py +printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py From 30be79131212cefb844d79afc86ffbb20ac921ab Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 11:30:30 +0100 Subject: [PATCH 140/163] Be less verbose when reinstalling --- bin/uncloud-run-reinstall | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/uncloud-run-reinstall b/bin/uncloud-run-reinstall index 18e95c0..b211613 100755 --- a/bin/uncloud-run-reinstall +++ b/bin/uncloud-run-reinstall @@ -24,6 +24,6 @@ dir=${0%/*} ${dir}/gen-version; -pip uninstall -y uncloud -python setup.py install +pip uninstall -y uncloud >/dev/null +python setup.py install >/dev/null ${dir}/uncloud "$@" From bd9dbb12b798a1bfe0651cfb7bcae22058ae456b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 11:30:41 +0100 Subject: [PATCH 141/163] Cleanup networking --- uncloud/hack/main.py | 23 +++++++++++++--------- uncloud/hack/net.py | 45 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 4ccb74a..cb9fd7b 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -13,8 +13,11 @@ arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--last-used-mac', action='store_true') arg_parser.add_argument('--get-new-mac', action='store_true') -arg_parser.add_argument('--init-network', help="Initialise networking") -arg_parser.add_argument('--management-network', help="IPv6 management network") +arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true') +arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true') +arg_parser.add_argument('--network', help="/64 IPv6 network") +arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0") +arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") @@ -37,15 +40,17 @@ def main(arguments): if arguments['get_new_mac']: print(MAC(config).get_next()) - if arguments['init_networking!']: - if not arguments['management_network']: - raise UncloudException("Initialising the network requires an IPv6 network. You can use fd00::/64 for testing (non production!)") - vb = VXLANBridge(arguments['management_network']) - vb.setup() + #if arguments['init_network']: + if arguments['create_vxlan']: + if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: + raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + vb = VXLANBridge(vni=arguments['vni'], + uplinkdev=arguments['vxlan_uplink_device']) + vb._setup_vxlan() if arguments['run_dns_ra']: - if not arguments['management_network']: + if not arguments['network']: raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network. You can use fd00::/64 for testing (non production!)") - dnsra = DNSRA(arguments['management_network']) + dnsra = DNSRA(arguments['network']) dnsra.setup() diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 11649b8..170e7b9 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -1,21 +1,48 @@ import subprocess +import ipaddress + +from uncloud import UncloudException -class ManagementBridge(VXLANBridge): - pass class VXLANBridge(object): - def __init__(self, vni, bridgedev=None, uplinkdev=None): - self.management_vni = 1 - - cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {netid} dstport 4789 group ff05::{netid} dev {uplinkdev} ttl 5" + cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" cmd_up_dev = "ip link set {dev} up" cmd_create_bridge="ip link add {bridgedev} type bridge" cmd_add_to_bridge="ip link set {vxlandev} master {bridgedev} up" cmd_add_addr="ip addr add {ip} dev {bridgedev}" - def setup_networking(dev=wlan0, v6net): - vxlandev=vxlan${netid} - bridgedev=br${netid} + # VXLAN ids are at maximum 24 bit - use a /104 + multicast_network = ipaddress.IPv6Network("ff05::/104") + max_vni = (2**24)-1 + + def __init__(self, + vni, + uplinkdev): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + self.config['multicast_address'] = self.multicast_network[vni] + + self.config['uplinkdev'] = uplinkdev + self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + + def setup_networking(self): + pass + + def _setup_vxlan(self): + # check for device first (?) + cmd = self.cmd_create_vxlan.format(**self.config) + print(cmd) + subprocess.run(cmd.split()) + +class ManagementBridge(VXLANBridge): + pass class DNSRA(object): From 8888f5d9f7aaaf20d10f1657bb2df60df4a6f912 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 12:55:06 +0100 Subject: [PATCH 142/163] add logging --- scripts/uncloud | 29 ++++++++++---------------- uncloud/hack/main.py | 7 ++++++- uncloud/hack/net.py | 49 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index ab5b40d..d565954 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -11,17 +11,7 @@ from uncloud.common import settings from uncloud import UncloudException from uncloud.common.cli import resolve_otp_credentials - -def exception_hook(exc_type, exc_value, exc_traceback): - logging.getLogger(__name__).error( - 'Uncaught exception', - exc_info=(exc_type, exc_value, exc_traceback) - ) - - -sys.excepthook = exception_hook - -# the components that use etcd +# Components that use etcd ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure', 'hack'] @@ -30,10 +20,6 @@ ALL_COMPONENTS = ETCD_COMPONENTS.copy() if __name__ == '__main__': - # Setting up root logger - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - arg_parser = argparse.ArgumentParser() subparsers = arg_parser.add_subparsers(dest='command') @@ -84,11 +70,18 @@ if __name__ == '__main__': mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') + if arguments['debug']: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + log = logging.getLogger() + try: main(arguments) except UncloudException as err: - logger.error(err) + log.error(err) # except ConnectionFailedError as err: -# logger.error('Cannot connect to etcd: {}'.format(err)) +# log.error('Cannot connect to etcd: {}'.format(err)) except Exception as err: - logger.exception(err) + log.exception(err) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index cb9fd7b..f275e62 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -20,6 +20,7 @@ arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") +arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') @@ -45,8 +46,12 @@ def main(arguments): if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") vb = VXLANBridge(vni=arguments['vni'], - uplinkdev=arguments['vxlan_uplink_device']) + route=arguments['network'], + uplinkdev=arguments['vxlan_uplink_device'], + use_sudo=arguments['use_sudo']) vb._setup_vxlan() + vb._setup_bridge() + vb._route_network() if arguments['run_dns_ra']: if not arguments['network']: diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 170e7b9..e18b36a 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -1,15 +1,20 @@ import subprocess import ipaddress +import logging + from uncloud import UncloudException +log = logging.getLogger(__name__) + class VXLANBridge(object): - cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" - cmd_up_dev = "ip link set {dev} up" - cmd_create_bridge="ip link add {bridgedev} type bridge" - cmd_add_to_bridge="ip link set {vxlandev} master {bridgedev} up" - cmd_add_addr="ip addr add {ip} dev {bridgedev}" + cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" + cmd_up_dev = "{sudo}ip link set {dev} up" + cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge" + cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up" + cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}" + cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}" # VXLAN ids are at maximum 24 bit - use a /104 multicast_network = ipaddress.IPv6Network("ff05::/104") @@ -17,16 +22,28 @@ class VXLANBridge(object): def __init__(self, vni, - uplinkdev): + uplinkdev, + route=None, + use_sudo=False): self.config = {} if vni > self.max_vni: raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + if use_sudo: + self.config['sudo'] = 'sudo ' + self.config['vni_dec'] = vni self.config['vni_hex'] = "{:x}".format(vni) self.config['multicast_address'] = self.multicast_network[vni] + #try: + self.config['route_network'] = ipaddress.IPv6Network(route) + #except Exception as e: + # print("Ahhhhhhhhhhhhhhhhh, die: {}".format(e)) + + self.config['route'] = route + self.config['uplinkdev'] = uplinkdev self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) @@ -36,9 +53,23 @@ class VXLANBridge(object): pass def _setup_vxlan(self): - # check for device first (?) - cmd = self.cmd_create_vxlan.format(**self.config) - print(cmd) + self._execute_cmd(self.cmd_create_vxlan) + self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev']) + + def _setup_bridge(self): + self._execute_cmd(self.cmd_create_bridge) + self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev']) + + def _route_network(self): + self._execute_cmd(self.cmd_add_route_dev) + + def _add_vxlan_to_bridge(self): + self._execute_cmd(self.cmd_add_to_bridge) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + print("Executing: {}".format(cmd)) subprocess.run(cmd.split()) class ManagementBridge(VXLANBridge): From 8e839aeb44ec47e82d446b3545cbe283f35c80ea Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 23 Jan 2020 18:41:59 +0100 Subject: [PATCH 143/163] commit stuff before dominique does --- uncloud/hack/main.py | 5 +++-- uncloud/hack/net.py | 5 ----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index f275e62..1e38c8a 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -1,4 +1,5 @@ import argparse +import logging from uncloud.hack.vm import VM from uncloud.hack.config import Config @@ -22,11 +23,11 @@ arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') - +log = logging.getLogger(__name__) def main(arguments): - print(arguments) + log.debug("args={}".format(arguments)) config = Config(arguments) if arguments['create_vm']: diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index e18b36a..e695dc8 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -37,11 +37,7 @@ class VXLANBridge(object): self.config['vni_hex'] = "{:x}".format(vni) self.config['multicast_address'] = self.multicast_network[vni] - #try: self.config['route_network'] = ipaddress.IPv6Network(route) - #except Exception as e: - # print("Ahhhhhhhhhhhhhhhhh, die: {}".format(e)) - self.config['route'] = route self.config['uplinkdev'] = uplinkdev @@ -69,7 +65,6 @@ class VXLANBridge(object): def _execute_cmd(self, cmd_string, **kwargs): cmd = cmd_string.format(**self.config, **kwargs) log.info("Executing: {}".format(cmd)) - print("Executing: {}".format(cmd)) subprocess.run(cmd.split()) class ManagementBridge(VXLANBridge): From 0982927c1bfe2a91b8244e148e3d7098b7c44ede Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Thu, 23 Jan 2020 18:43:41 +0100 Subject: [PATCH 144/163] Added DNSmasq ability for RA --- uncloud/hack/main.py | 10 ++++++---- uncloud/hack/net.py | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index f275e62..281c251 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -54,8 +54,10 @@ def main(arguments): vb._route_network() if arguments['run_dns_ra']: - if not arguments['network']: - raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network. You can use fd00::/64 for testing (non production!)") + if not arguments['network'] or not arguments['vni']: + raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") - dnsra = DNSRA(arguments['network']) - dnsra.setup() + dnsra = DNSRA(route=arguments['network'], + vni=arguments['vni'], + use_sudo=arguments['use_sudo']) + dnsra._setup_dnsmasq() diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index e18b36a..b036198 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -77,9 +77,41 @@ class ManagementBridge(VXLANBridge): class DNSRA(object): - def __init__(self): - pass + # VXLAN ids are at maximum 24 bit + max_vni = (2**24)-1 + # Command to start dnsmasq + cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra" + + def __init__(self, + vni, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + + #TODO: remove if not needed + #self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + + # dnsmasq only wants the network without the prefix, therefore, cut it off + self.config['route'] = ipaddress.IPv6Network(route).network_address + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + def _setup_dnsmasq(self): + self._execute_cmd(self.cmd_start_dnsmasq) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + print("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + class Firewall(object): pass From c881c7ce4d65044f0b8bc63de981680f2bab9a1e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 23 Jan 2020 21:15:26 +0100 Subject: [PATCH 145/163] hack mac: be a proper python class --- uncloud/hack/mac.py | 50 +++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index e7f41a2..084df13 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -38,7 +38,8 @@ class MAC(object): self.config = config self.db = DB(config, prefix="/mac") - self.prefix = 0x002000000000 + self.prefix = 0x420000000000 + self._number = 0 # Not set by default @staticmethod def validate_mac(mac): @@ -56,35 +57,40 @@ class MAC(object): def last_used_mac(self): return self.int_to_mac(self.prefix + self.last_used_index()) - @staticmethod - def int_to_mac(number): - b = number.to_bytes(6, byteorder="big") + def to_colon_format(self): + b = self._number.to_bytes(6, byteorder="big") return ':'.join(format(s, '02x') for s in b) - def get_next(self, vmuuid=None, as_int=False): + def to_str_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ''.join(format(s, '02x') for s in b) + + def create(self): last_number = self.last_used_index() - # FIXME: compare to 48bit minus prefix length to the power of 2 - if last_number == int('0xffffff', 16): + if last_number == int('0xffffffff', 16): raise UncloudException("Exhausted all possible mac addresses - try to free some") next_number = last_number + 1 - next_number_string = "{:012x}".format(next_number) + self._number = self.prefix + next_number - next_mac_number = self.prefix + next_number - next_mac = self.int_to_mac(next_mac_number) - - db_entry = {} - db_entry['vm_uuid'] = vmuuid - db_entry['index'] = next_number - db_entry['mac_address'] = next_mac + #next_number_string = "{:012x}".format(next_number) + #next_mac = self.int_to_mac(next_mac_number) + # db_entry = {} + # db_entry['vm_uuid'] = vmuuid + # db_entry['index'] = next_number + # db_entry['mac_address'] = next_mac # should be one transaction - self.db.increment("last_used_index") - self.db.set("used/{}".format(next_mac), - db_entry, as_json=True) + # self.db.increment("last_used_index") + # self.db.set("used/{}".format(next_mac), + # db_entry, as_json=True) - if as_int: - return next_mac_number - else: - return next_mac + def __int__(self): + return self._number + + def __repr__(self): + return self.to_str_format() + + def __str__(self): + return self.to_colon_format() From 46a04048b54dc148d8b5538c0674d56038e00017 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Thu, 23 Jan 2020 21:17:09 +0100 Subject: [PATCH 146/163] small changes in vm.py to make it more generic --- uncloud/hack/vm.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index eb75902..e8038cc 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -32,27 +32,35 @@ class VM(object): self.config = config self.db = DB(config, prefix="/vm") - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" - self.qemu="/usr/bin/qemu-system-x86_64" - self.accel="kvm" + #TODO: Select generic + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm + self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config + self.accel="kvm" #TODO: should be config self.vm = {} - self.owner="nico" - self.bridge="br100" + #TODO: this should be generic + self.vm['owner']="nico" #TODO: Should in config.arguments + #self.config['vni_hex'] = "{:x}".format(self.config.vni) + #self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + self.vni_hex = "{:x}".format(self.config.arguments['vni']) + self.bridgedev = "br{}".format(self.vni_hex) + + #TODO: Touch later! (when necessary) self.ifup = os.path.join(self.hackprefix, "ifup.sh") self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") def create(self): self.uuid = uuid.uuid4() + #TODO: This all should be generic self.vm['uuid'] = str(self.uuid) self.vm['memory'] = 1024 self.vm['cores'] = 2 self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") - self.mac=MAC().next() + self.mac=MAC(self.config).get_next() self.vm['commandline' ] = [ "sudo", "{}".format(self.qemu), @@ -62,7 +70,7 @@ class VM(object): "-smp", "{}".format(self.vm['cores']), "-uuid", "{}".format(self.vm['uuid']), "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), + "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown),self.mac, "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] From b5409552d8765afadada6c26793162beb8a5eda3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 23 Jan 2020 21:20:16 +0100 Subject: [PATCH 147/163] prepare vm.py for dominique --- uncloud/hack/vm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index e8038cc..1a531e0 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -32,7 +32,7 @@ class VM(object): self.config = config self.db = DB(config, prefix="/vm") - #TODO: Select generic + #TODO: Select generic self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config self.accel="kvm" #TODO: should be config @@ -60,7 +60,9 @@ class VM(object): self.vm['cores'] = 2 self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") - self.mac=MAC(self.config).get_next() + self.mac=MAC(self.config) + self.mac.create() + self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) self.vm['commandline' ] = [ "sudo", "{}".format(self.qemu), @@ -74,6 +76,8 @@ class VM(object): "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] + # TODO: Add ip link command afterwards (rouxdo) + self.db.set(str(self.vm['uuid']), self.vm, as_json=True) From 58daf8191e3c5d48a96b745a906fb4d9fa2a72e0 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 24 Jan 2020 13:56:08 +0100 Subject: [PATCH 148/163] refactored vm.py to create a VM --- uncloud/hack/mac.py | 14 ++++-- uncloud/hack/main.py | 6 ++- uncloud/hack/vm.py | 106 ++++++++++++++++++++++++------------------- 3 files changed, 75 insertions(+), 51 deletions(-) diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index 084df13..66286dd 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -36,7 +36,9 @@ log = logging.getLogger(__name__) class MAC(object): def __init__(self, config): self.config = config - self.db = DB(config, prefix="/mac") + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(config, prefix="/mac") self.prefix = 0x420000000000 self._number = 0 # Not set by default @@ -47,10 +49,14 @@ class MAC(object): raise Error("Not a valid mac address: %s" % mac) def last_used_index(self): - value = self.db.get("last_used_index") - if not value: - self.db.set("last_used_index", "0") + if not self.no_db: value = self.db.get("last_used_index") + if not value: + self.db.set("last_used_index", "0") + value = self.db.get("last_used_index") + + else: + value = "0" return int(value) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 2981184..4778ef6 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -22,6 +22,10 @@ arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') +arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) +arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) +arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') + log = logging.getLogger(__name__) @@ -33,7 +37,7 @@ def main(arguments): if arguments['create_vm']: print("Creating VM") vm = VM(config) - vm.create() + vm.commandline() if arguments['last_used_mac']: m = MAC(config) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 1a531e0..8e20e2e 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -29,58 +29,72 @@ from uncloud.hack.mac import MAC class VM(object): def __init__(self, config): - self.config = config - self.db = DB(config, prefix="/vm") + self.config = config + #TODO: Enable etcd lookup + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(self.config, prefix="/vm") - #TODO: Select generic - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm - self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config - self.accel="kvm" #TODO: should be config + #TODO: Select generic + #self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm + self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing + self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config + self.accel="kvm" #TODO: should be config - self.vm = {} + self.vm = {} + #TODO: Touch later! (when necessary) + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") - #TODO: this should be generic - self.vm['owner']="nico" #TODO: Should in config.arguments - #self.config['vni_hex'] = "{:x}".format(self.config.vni) - #self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) - self.vni_hex = "{:x}".format(self.config.arguments['vni']) - self.bridgedev = "br{}".format(self.vni_hex) + def commandline(self): + """This method is used to trigger / create a vm from the cli""" + #TODO: read arguments from cli + #TODO: create etcd json object + self.vm['owner']= "nico" + self.vm['memory'] = self.config.arguments['memory'] + self.vm['cores'] = self.config.arguments['cores'] + self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + self.create_template() + # mimics api call = this will already be in etcd + #self.vm['os_image'] = self.db.get("os_image") + self.create() + def create_template(self): + self.uuid = uuid.uuid4() + #TODO: This all should be generic + self.vm['uuid'] = str(self.uuid) + #self.vni_hex = "{:x}".format(self.config.arguments['vni']) + self.bridgedev = "br{}".format("{:x}".format(self.config.arguments['vni'])) + + #TODO: Enable sudo + if self.config.arguments['use_sudo']: + self.sudo = "sudo" + + self.mac=MAC(self.config) + self.mac.create() + self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) + + #self.vm['commandline'] = [ "{}".format(self.sudo), + self.vm['commandline'] = [ "{}".format(self.sudo), + "{}".format(self.qemu), + "-name", "uncloud-{}".format(self.vm['uuid']), + "-machine", "pc,accel={}".format(self.accel), + "-m", "{}".format(self.vm['memory']), + "-smp", "{}".format(self.vm['cores']), + "-uuid", "{}".format(self.vm['uuid']), + "-drive", "file={},media=cdrom".format(self.vm['os_image']), + "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.mac), + "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) + ] - #TODO: Touch later! (when necessary) - self.ifup = os.path.join(self.hackprefix, "ifup.sh") - self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") def create(self): - self.uuid = uuid.uuid4() - #TODO: This all should be generic - self.vm['uuid'] = str(self.uuid) - self.vm['memory'] = 1024 - self.vm['cores'] = 2 - self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + if not self.no_db: + self.db.set(str(self.vm['uuid']), + self.vm, + as_json=True) - self.mac=MAC(self.config) - self.mac.create() - self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) - - self.vm['commandline' ] = [ "sudo", - "{}".format(self.qemu), - "-name", "uncloud-{}".format(self.vm['uuid']), - "-machine", "pc,accel={}".format(self.accel), - "-m", "{}".format(self.vm['memory']), - "-smp", "{}".format(self.vm['cores']), - "-uuid", "{}".format(self.vm['uuid']), - "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown),self.mac, - "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) - ] - - # TODO: Add ip link command afterwards (rouxdo) - - self.db.set(str(self.vm['uuid']), - self.vm, - as_json=True) - - print(" ".join(self.vm['commandline'])) - subprocess.run(self.vm['commandline']) + print(" ".join(self.vm['commandline'])) + subprocess.run(self.vm['commandline']) #TODO: run in background + #TODO: Add interface ifname to bridge brXX (via net.py: public function add iface to bridge) From 7e91f60c0acf75c8d7bae75a8e1068cdbf4784cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 24 Jan 2020 14:10:08 +0100 Subject: [PATCH 149/163] sudo fix --- uncloud/hack/net.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 6e2a6ee..30d0c03 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -32,6 +32,8 @@ class VXLANBridge(object): if use_sudo: self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' self.config['vni_dec'] = vni self.config['vni_hex'] = "{:x}".format(vni) From 93d7a409b12e2cf7ab3c06a312cc6b4901816db0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 24 Jan 2020 14:10:49 +0100 Subject: [PATCH 150/163] Fix Dominique's sudo bug Totally not related to my previous commit --- uncloud/hack/net.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 30d0c03..f28ab7f 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -92,6 +92,8 @@ class DNSRA(object): if use_sudo: self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' #TODO: remove if not needed #self.config['vni_dec'] = vni From b1319d654af20cd14c5f8f9b82a67a5e58d93098 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 24 Jan 2020 14:15:48 +0100 Subject: [PATCH 151/163] Make me and Dominique happy (aka add vxlan to bridge) --- uncloud/hack/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 4778ef6..fc54da1 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -56,13 +56,14 @@ def main(arguments): use_sudo=arguments['use_sudo']) vb._setup_vxlan() vb._setup_bridge() + vb._add_vxlan_to_bridge() vb._route_network() if arguments['run_dns_ra']: if not arguments['network'] or not arguments['vni']: raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") - dnsra = DNSRA(route=arguments['network'], + dnsra = DNSRA(route=arguments['network'], vni=arguments['vni'], use_sudo=arguments['use_sudo']) dnsra._setup_dnsmasq() From ae3482cc71350b8fc85a578798b73f500df45bd7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 24 Jan 2020 14:21:38 +0100 Subject: [PATCH 152/163] Fix and break some VM stuff --- uncloud/hack/vm.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 8e20e2e..c41fddc 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -36,8 +36,8 @@ class VM(object): self.db = DB(self.config, prefix="/vm") #TODO: Select generic - #self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm - self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm + #self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config self.accel="kvm" #TODO: should be config @@ -67,14 +67,19 @@ class VM(object): #self.vni_hex = "{:x}".format(self.config.arguments['vni']) self.bridgedev = "br{}".format("{:x}".format(self.config.arguments['vni'])) - #TODO: Enable sudo + #TODO: Enable sudo -- FIXME! if self.config.arguments['use_sudo']: self.sudo = "sudo" + else: + self.sudo = "" + self.mac=MAC(self.config) self.mac.create() self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) + # FIXME: TODO: turn this into a string and THEN + # .split() it later -- easier for using .format() #self.vm['commandline'] = [ "{}".format(self.sudo), self.vm['commandline'] = [ "{}".format(self.sudo), "{}".format(self.qemu), @@ -84,7 +89,7 @@ class VM(object): "-smp", "{}".format(self.vm['cores']), "-uuid", "{}".format(self.vm['uuid']), "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.mac), + "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] From 5711bf4770159821a50e5ef0b677bdba860780c8 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 24 Jan 2020 14:34:34 +0100 Subject: [PATCH 153/163] bugfixes in vm --- uncloud/hack/vm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 8e20e2e..24eb108 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -54,7 +54,7 @@ class VM(object): self.vm['owner']= "nico" self.vm['memory'] = self.config.arguments['memory'] self.vm['cores'] = self.config.arguments['cores'] - self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.3-x86_64.iso") self.create_template() # mimics api call = this will already be in etcd #self.vm['os_image'] = self.db.get("os_image") @@ -84,7 +84,7 @@ class VM(object): "-smp", "{}".format(self.vm['cores']), "-uuid", "{}".format(self.vm['uuid']), "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.mac), + "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] From 5d05e91335925def4ec4342874bef352db205cbd Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 24 Jan 2020 17:12:50 +0100 Subject: [PATCH 154/163] added hackerprefix argument, changed the commandline structure of vm to work better with sudo --- uncloud/hack/main.py | 1 + uncloud/hack/vm.py | 40 +++++++++++++++++++++++++--------------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index fc54da1..b6d8fad 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -25,6 +25,7 @@ arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root! arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') +arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") log = logging.getLogger(__name__) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index bb35348..4caa2fe 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -23,10 +23,14 @@ import subprocess import uuid import os +import logging from uncloud.hack.db import DB from uncloud.hack.mac import MAC + +log = logging.getLogger(__name__) + class VM(object): def __init__(self, config): self.config = config @@ -36,8 +40,9 @@ class VM(object): self.db = DB(self.config, prefix="/vm") #TODO: Select generic - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm + #self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm #self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing + self.hackprefix=self.config.arguments['hackprefix'] self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config self.accel="kvm" #TODO: should be config @@ -69,30 +74,36 @@ class VM(object): #TODO: Enable sudo -- FIXME! if self.config.arguments['use_sudo']: - self.sudo = "sudo" + self.sudo = "sudo " else: self.sudo = "" self.mac=MAC(self.config) self.mac.create() + self.vm['mac'] = self.mac self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) # FIXME: TODO: turn this into a string and THEN # .split() it later -- easier for using .format() #self.vm['commandline'] = [ "{}".format(self.sudo), - self.vm['commandline'] = [ "{}".format(self.sudo), - "{}".format(self.qemu), - "-name", "uncloud-{}".format(self.vm['uuid']), - "-machine", "pc,accel={}".format(self.accel), - "-m", "{}".format(self.vm['memory']), - "-smp", "{}".format(self.vm['cores']), - "-uuid", "{}".format(self.vm['uuid']), - "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), - "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) - ] + self.vm['commandline'] = "{sudo}{qemu} -name uncloud-{uuid} -machine pc,accel={accel} -m {memory} -smp {cores} -uuid {uuid} -drive file={os_image},media=cdrom -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname} -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" +# self.vm['commandline'] = [ "{}".format(self.sudo), +# "{}".format(self.qemu), +# "-name", "uncloud-{}".format(self.vm['uuid']), +# "-machine", "pc,accel={}".format(self.accel), +# "-m", "{}".format(self.vm['memory']), +# "-smp", "{}".format(self.vm['cores']), +# "-uuid", "{}".format(self.vm['uuid']), +# "-drive", "file={},media=cdrom".format(self.vm['os_image']), +# "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), +# "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.vm['mac']) +# ] + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.vm, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) def create(self): if not self.no_db: @@ -100,6 +111,5 @@ class VM(object): self.vm, as_json=True) - print(" ".join(self.vm['commandline'])) - subprocess.run(self.vm['commandline']) #TODO: run in background + self._execute_cmd(self.vm['commandline'], sudo=self.sudo, qemu=self.qemu, accel=self.accel, ifup=self.ifup, ifdown=self.ifdown) #TODO: Add interface ifname to bridge brXX (via net.py: public function add iface to bridge) From cbcaf636506e138542e1580098d29057b9558b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 26 Jan 2020 12:04:37 +0100 Subject: [PATCH 155/163] Update VM images documentation (upstream images, uncloud-init) --- docs/source/{admin-guide => admin-guide.rst} | 39 ++---------- docs/source/index.rst | 5 +- docs/source/vm-images.rst | 66 ++++++++++++++++++++ 3 files changed, 74 insertions(+), 36 deletions(-) rename docs/source/{admin-guide => admin-guide.rst} (72%) create mode 100644 docs/source/vm-images.rst diff --git a/docs/source/admin-guide b/docs/source/admin-guide.rst similarity index 72% rename from docs/source/admin-guide rename to docs/source/admin-guide.rst index ec6597d..b62808d 100644 --- a/docs/source/admin-guide +++ b/docs/source/admin-guide.rst @@ -56,40 +56,13 @@ To start host we created earlier, execute the following command ucloud host ungleich.ch -Create OS Image ---------------- +File & image scanners +-------------------------- -Create ucloud-init ready OS image (Optional) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This step is optional if you just want to test ucloud. However, sooner or later -you want to create OS images with ucloud-init to properly -contexualize VMs. - -1. Start a VM with OS image on which you want to install ucloud-init -2. Execute the following command on the started VM - - .. code-block:: sh - - apk add git - git clone https://code.ungleich.ch/ucloud/ucloud-init.git - cd ucloud-init - sh ./install.sh -3. Congratulations. Your image is now ucloud-init ready. - - -Upload Sample OS Image -~~~~~~~~~~~~~~~~~~~~~~ -Execute the following to get the sample OS image file. - -.. code-block:: sh - - mkdir /var/www/admin - (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download) - -Run File Scanner and Image Scanner -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make -images from tracked files. So, we need to track the file by running File Scanner +Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our +uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by +ucloud. We can only make images from tracked files. So, we need to track the +file by running File Scanner .. code-block:: sh diff --git a/docs/source/index.rst b/docs/source/index.rst index b31cff3..fad1f88 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,14 +11,13 @@ Welcome to ucloud's documentation! :caption: Contents: introduction - user-guide setup-install + vm-images + user-guide admin-guide - user-guide/how-to-create-an-os-image-for-ucloud troubleshooting hacking - Indices and tables ================== diff --git a/docs/source/vm-images.rst b/docs/source/vm-images.rst new file mode 100644 index 0000000..4b2758a --- /dev/null +++ b/docs/source/vm-images.rst @@ -0,0 +1,66 @@ +VM images +================================== + +Overview +--------- + +ucloud tries to be least invasise towards VMs and only require +strictly necessary changes for running in a virtualised +environment. This includes configurations for: + +* Configuring the network +* Managing access via ssh keys +* Resizing the attached disk(s) + +Upstream images +--------------- + +The 'official' uncloud images are defined in the `uncloud/images +`_ repository. + +How to make you own Uncloud images +---------------------------------- + +.. note:: + It is fairly easy to create your own images for uncloud, as the common + operations (which are detailed below) can be automatically handled by the + `uncloud/uncloud-init `_ tool. + +Network configuration +~~~~~~~~~~~~~~~~~~~~~ +All VMs in ucloud are required to support IPv6. The primary network +configuration is always done using SLAAC. A VM thus needs only to be +configured to + +* accept router advertisements on all network interfaces +* use the router advertisements to configure the network interfaces +* accept the DNS entries from the router advertisements + + +Configuring SSH keys +~~~~~~~~~~~~~~~~~~~~ + +To be able to access the VM, ucloud support provisioning SSH keys. + +To accept ssh keys in your VM, request the URL +*http://metadata/ssh_keys*. Add the content to the appropriate user's +**authorized_keys** file. Below you find sample code to accomplish +this task: + +.. code-block:: sh + + tmp=$(mktemp) + curl -s http://metadata/ssk_keys > "$tmp" + touch ~/.ssh/authorized_keys # ensure it exists + cat ~/.ssh/authorized_keys >> "$tmp" + sort "$tmp" | uniq > ~/.ssh/authorized_keys + + +Disk resize +~~~~~~~~~~~ +In virtualised environments, the disk sizes might grow. The operating +system should detect disks that are bigger than the existing partition +table and resize accordingly. This task is os specific. + +ucloud does not support shrinking disks due to the complexity and +intra OS dependencies. From 2b71c1807de324be6a9bb707c561621548e3a48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 09:25:25 +0100 Subject: [PATCH 156/163] Wire uncloud-hack vm module to VMM --- uncloud/hack/main.py | 20 +++++-- uncloud/hack/vm.py | 130 +++++++++++++++++++++---------------------- 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index b6d8fad..351f582 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -10,7 +10,6 @@ from uncloud import UncloudException arg_parser = argparse.ArgumentParser('hack', add_help=False) #description="Commands that are unfinished - use at own risk") -arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--last-used-mac', action='store_true') arg_parser.add_argument('--get-new-mac', action='store_true') @@ -22,8 +21,15 @@ arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') + +arg_parser.add_argument('--create-vm', action='store_true') +arg_parser.add_argument('--destroy-vm', action='store_true') +arg_parser.add_argument('--get-vm-status', action='store_true') arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) +arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") +arg_parser.add_argument('--uuid', help="VM UUID") + arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") @@ -32,13 +38,19 @@ log = logging.getLogger(__name__) def main(arguments): - log.debug("args={}".format(arguments)) config = Config(arguments) if arguments['create_vm']: - print("Creating VM") vm = VM(config) - vm.commandline() + vm.create() + + if arguments['destroy_vm']: + vm = VM(config) + vm.stop() + + if arguments['get_vm_status']: + vm = VM(config) + vm.status() if arguments['last_used_mac']: m = MAC(config) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 4caa2fe..ce96fbf 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -27,89 +27,87 @@ import logging from uncloud.hack.db import DB from uncloud.hack.mac import MAC - +from uncloud.vmm import VMM log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) class VM(object): def __init__(self, config): self.config = config + #TODO: Enable etcd lookup self.no_db = self.config.arguments['no_db'] if not self.no_db: self.db = DB(self.config, prefix="/vm") - #TODO: Select generic - #self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm - #self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing - self.hackprefix=self.config.arguments['hackprefix'] - self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config - self.accel="kvm" #TODO: should be config + # General CLI arguments. + self.hackprefix = self.config.arguments['hackprefix'] + self.uuid = self.config.arguments['uuid'] + self.memory = self.config.arguments['memory'] or '1024M' + self.cores = self.config.arguments['cores'] or 1 + if self.config.arguments['image']: + self.image = os.path.join(self.hackprefix, self.config.arguments['image']) + else: + self.image = None - self.vm = {} + # External components. + self.vmm = VMM(vmm_backend=self.hackprefix) + self.mac = MAC(self.config) - #TODO: Touch later! (when necessary) + # Harcoded & generated values. + self.owner = 'uncoud' + self.image_format='qcow2' + self.accel = 'kvm' + self.threads = 1 self.ifup = os.path.join(self.hackprefix, "ifup.sh") self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + self.ifname = "uc{}".format(self.mac.to_str_format()) - def commandline(self): - """This method is used to trigger / create a vm from the cli""" - #TODO: read arguments from cli - #TODO: create etcd json object - self.vm['owner']= "nico" - self.vm['memory'] = self.config.arguments['memory'] - self.vm['cores'] = self.config.arguments['cores'] - self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.3-x86_64.iso") - self.create_template() - # mimics api call = this will already be in etcd - #self.vm['os_image'] = self.db.get("os_image") - self.create() + def get_qemu_args(self): + command = ( + "-name {owner}-{name}" + " -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}" + " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" + ).format( + owner=self.owner, name=self.uuid, + accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname, + mac=self.mac + ) - def create_template(self): - self.uuid = uuid.uuid4() - #TODO: This all should be generic - self.vm['uuid'] = str(self.uuid) - #self.vni_hex = "{:x}".format(self.config.arguments['vni']) - self.bridgedev = "br{}".format("{:x}".format(self.config.arguments['vni'])) - - #TODO: Enable sudo -- FIXME! - if self.config.arguments['use_sudo']: - self.sudo = "sudo " - else: - self.sudo = "" - - - self.mac=MAC(self.config) - self.mac.create() - self.vm['mac'] = self.mac - self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) - - # FIXME: TODO: turn this into a string and THEN - # .split() it later -- easier for using .format() - #self.vm['commandline'] = [ "{}".format(self.sudo), - self.vm['commandline'] = "{sudo}{qemu} -name uncloud-{uuid} -machine pc,accel={accel} -m {memory} -smp {cores} -uuid {uuid} -drive file={os_image},media=cdrom -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname} -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" -# self.vm['commandline'] = [ "{}".format(self.sudo), -# "{}".format(self.qemu), -# "-name", "uncloud-{}".format(self.vm['uuid']), -# "-machine", "pc,accel={}".format(self.accel), -# "-m", "{}".format(self.vm['memory']), -# "-smp", "{}".format(self.vm['cores']), -# "-uuid", "{}".format(self.vm['uuid']), -# "-drive", "file={},media=cdrom".format(self.vm['os_image']), -# "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), -# "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.vm['mac']) -# ] - - def _execute_cmd(self, cmd_string, **kwargs): - cmd = cmd_string.format(**self.vm, **kwargs) - log.info("Executing: {}".format(cmd)) - subprocess.run(cmd.split()) + return command.split(" ") def create(self): - if not self.no_db: - self.db.set(str(self.vm['uuid']), - self.vm, - as_json=True) + # New VM: new UUID, new MAC. + self.uuid = str(uuid.uuid4()) + self.mac.create() + + qemu_args = self.get_qemu_args() + log.debug("QEMU args passed to VMM: {}".format(qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + self.vmm.stop(self.uuid) + + def status(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_status(self.uuid)) - self._execute_cmd(self.vm['commandline'], sudo=self.sudo, qemu=self.qemu, accel=self.accel, ifup=self.ifup, ifdown=self.ifdown) - #TODO: Add interface ifname to bridge brXX (via net.py: public function add iface to bridge) From 4c6a126d8b0a59a454ec69cbbc867786f0b7b04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 11:02:18 +0100 Subject: [PATCH 157/163] Hack/VM: wire get_vnc and list_vms --- uncloud/hack/main.py | 10 ++++++++++ uncloud/hack/vm.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 351f582..9607ec2 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -25,6 +25,8 @@ arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root! arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--destroy-vm', action='store_true') arg_parser.add_argument('--get-vm-status', action='store_true') +arg_parser.add_argument('--get-vm-vnc', action='store_true') +arg_parser.add_argument('--list-vms', action='store_true') arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") @@ -52,6 +54,14 @@ def main(arguments): vm = VM(config) vm.status() + if arguments['get_vm_vnc']: + vm = VM(config) + vm.vnc_addr() + + if arguments['list_vms']: + vm = VM(config) + vm.list() + if arguments['last_used_mac']: m = MAC(config) print(m.last_used_mac()) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index ce96fbf..e9b7719 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -111,3 +111,13 @@ class VM(object): print(self.vmm.get_status(self.uuid)) + def vnc_addr(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_vnc(self.uuid)) + + def list(self): + print(self.vmm.discover()) + From a759b8aa39ae96a08904119b15c5306048c34c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 12:24:26 +0100 Subject: [PATCH 158/163] VMM: make use of socket_dir --- uncloud/vmm/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 719bdbe..6db61eb 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -125,7 +125,7 @@ class VMM: os.makedirs(self.socket_dir, exist_ok=True) def is_running(self, uuid): - sock_path = os.path.join(self.vmm_backend, uuid) + sock_path = os.path.join(self.socket_dir, uuid) try: sock = socket.socket(socket.AF_UNIX) sock.connect(sock_path) @@ -163,7 +163,7 @@ class VMM: qmp_arg = ( "-qmp", "unix:{},server,nowait".format( - join_path(self.vmm_backend, uuid) + join_path(self.socket_dir, uuid) ), ) vnc_arg = ( @@ -212,7 +212,7 @@ class VMM: def execute_command(self, uuid, command, **kwargs): # execute_command -> sucess?, output try: - with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as ( + with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as ( sock_handle, file_handle, ): @@ -255,8 +255,8 @@ class VMM: def discover(self): vms = [ uuid - for uuid in os.listdir(self.vmm_backend) - if not isdir(join_path(self.vmm_backend, uuid)) + for uuid in os.listdir(self.socket_dir) + if not isdir(join_path(self.socket_dir, uuid)) ] return vms From 1758629ca1b861c0406e80591ff35073d7d6331f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 12:33:36 +0100 Subject: [PATCH 159/163] Add minimal doc to hack/vm.py --- uncloud/hack/vm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index e9b7719..f9cd31a 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -17,8 +17,21 @@ # # You should have received a copy of the GNU General Public License # along with uncloud. If not, see . + +# This module is directly called from the hack module, and can be used as follow: # +# Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix. +# `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2` # +# List running VMs (returns a list of UUIDs). +# `uncloud hack --hackprefix /tmp/hackcloud --list-vms +# +# Get VM status: +# `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid` +# +# Stop a VM: +# `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` + `` import subprocess import uuid From e2cd44826b9c307f816d170ed93b3a172edcf712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 13:45:20 +0100 Subject: [PATCH 160/163] Fix typo in hack/vm.py --- uncloud/hack/vm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index f9cd31a..ac403d8 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -31,7 +31,7 @@ # # Stop a VM: # `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` - `` +# `` import subprocess import uuid From 618fecb73fe3bc77f43d567219a88f3c5cb19b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 14:38:07 +0100 Subject: [PATCH 161/163] Initial implementation (no networking) of uncloud-oneshot --- scripts/uncloud | 1 + uncloud/oneshot/__init__.py | 3 ++ uncloud/oneshot/main.py | 65 ++++++++++++++++++++++++++++ uncloud/oneshot/virtualmachine.py | 70 +++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 uncloud/oneshot/__init__.py create mode 100644 uncloud/oneshot/main.py create mode 100644 uncloud/oneshot/virtualmachine.py diff --git a/scripts/uncloud b/scripts/uncloud index d565954..7d38e42 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -16,6 +16,7 @@ ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure', 'hack'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('oneshot') #ALL_COMPONENTS.append('cli') diff --git a/uncloud/oneshot/__init__.py b/uncloud/oneshot/__init__.py new file mode 100644 index 0000000..eea436a --- /dev/null +++ b/uncloud/oneshot/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/uncloud/oneshot/main.py b/uncloud/oneshot/main.py new file mode 100644 index 0000000..20f22e4 --- /dev/null +++ b/uncloud/oneshot/main.py @@ -0,0 +1,65 @@ +import argparse +import os + +from pathlib import Path +from uncloud.vmm import VMM + +from . import virtualmachine, logger + +arg_parser = argparse.ArgumentParser('oneshot', add_help=False) +arg_parser.add_argument('--workdir', default=Path.home()) +arg_parser.add_argument('--list-vms', action='store_true') +arg_parser.add_argument('--start-vm', action='store_true') +arg_parser.add_argument('--stop-vm', action='store_true') +arg_parser.add_argument('--name') +arg_parser.add_argument('--image') +arg_parser.add_argument('--uuid') +arg_parser.add_argument('--mac') +arg_parser.add_argument('--get_vm_status', action='store_true') +arg_parser.add_argument('--setup-network') + +def setup_network(): + print("Not implemented yet.") + exit(1) + +def require_with(arguments, required, mode): + if not arguments[required]: + print("--{} is required with the {} flag. Exiting.".format(required, mode)) + exit(1) + +def main(arguments): + # Initialize VMM + workdir = arguments['workdir'] + vmm = VMM(vmm_backend=workdir) + + # Initialize workdir directory. + # TODO: copy ifup, ifdown. + + # Build VM configuration. + vm_config = {} + for spec in ['uuid', 'memory', 'cores', 'threads', 'image', 'image_format', 'name']: + if arguments.get(spec): + vm_config[spec] = arguments[spec] + + # Execute requested VM action. + vm = virtualmachine.VM(vmm, vm_config) + if arguments['setup_network']: + setup_network() + elif arguments['start_vm']: + require_with(arguments, 'image', 'start_vm') + vm.start() + logger.info("Created VM {}".format(vm.get_uuid)) + elif arguments['get_vm_status']: + require_with(arguments, 'uuid', 'get_vm_status') + print("VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status())) + elif arguments['stop_vm']: + require_with(arguments, 'uuid', 'stop_vm') + vm.stop() + elif arguments['list_vms']: + discovered = vmm.discover() + print("Found {} VMs.".format(len(discovered))) + for uuid in vmm.discover(): + vmi = virtualmachine.VM(vmm, {'uuid': uuid}) + print("VM: {} {} {}".format(vmi.get_uuid, vmi.get_name, vmi.get_status)) + else: + print('No action requested. Exiting.') diff --git a/uncloud/oneshot/virtualmachine.py b/uncloud/oneshot/virtualmachine.py new file mode 100644 index 0000000..47365d5 --- /dev/null +++ b/uncloud/oneshot/virtualmachine.py @@ -0,0 +1,70 @@ +import uuid +import os + +from uncloud.oneshot import logger + +class VM(object): + def __init__(self, vmm, config): + self.config = config + self.vmm = vmm + + # Extract VM specs/metadata from configuration. + self.name = config.get('name') + self.memory = config.get('memory', 1024) + self.cores = config.get('cores', 1) + self.threads = config.get('threads', 1) + self.image_format = config.get('image_format', 'qcow2') + self.image = config.get('image') + self.uuid = config.get('uuid', uuid.uuid4()) + self.mac = config.get('mac', 'spuik') + + # Harcoded & generated values. + self.image_format='qcow2' + self.accel = 'kvm' + + def get_qemu_args(self): + command = ( + "-uuid {uuid} -name {name}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + ).format( + uuid=self.uuid, name=self.name, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + ) + + return command.split(" ") + + def start(self): + # Check that VM image is available. + if not os.path.isfile(self.image): + logger.error("Image {} does not exist. Aborting.".format(self.image)) + + # Generate config for and run QEMU. + qemu_args = self.get_qemu_args() + logger.warning("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + self.vmm.stop(self.uuid) + + def get_status(self): + return self.vmm.get_status(self.uuid) + + def get_vnc_addr(self): + return self.vmm.get_vnc(self.uuid) + + def get_uuid(self): + return self.uuid + + def get_name(self): + success, json = self.vmm.execute_command(uuid, 'query-name') + if success: + return json['return']['name'] + + return None From 3e69fb275fb152cc842582e3a173cdbea8e2e155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 17:44:53 +0100 Subject: [PATCH 162/163] Oneshot: cleanup CLI, initial networking support --- uncloud/oneshot/main.py | 140 +++++++++++++++++++++--------- uncloud/oneshot/virtualmachine.py | 27 ++++-- 2 files changed, 119 insertions(+), 48 deletions(-) diff --git a/uncloud/oneshot/main.py b/uncloud/oneshot/main.py index 20f22e4..0e94a81 100644 --- a/uncloud/oneshot/main.py +++ b/uncloud/oneshot/main.py @@ -1,65 +1,123 @@ import argparse import os + from pathlib import Path from uncloud.vmm import VMM +from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap from . import virtualmachine, logger +### +# Argument parser loaded by scripts/uncloud. arg_parser = argparse.ArgumentParser('oneshot', add_help=False) -arg_parser.add_argument('--workdir', default=Path.home()) -arg_parser.add_argument('--list-vms', action='store_true') -arg_parser.add_argument('--start-vm', action='store_true') -arg_parser.add_argument('--stop-vm', action='store_true') -arg_parser.add_argument('--name') -arg_parser.add_argument('--image') -arg_parser.add_argument('--uuid') -arg_parser.add_argument('--mac') -arg_parser.add_argument('--get_vm_status', action='store_true') -arg_parser.add_argument('--setup-network') -def setup_network(): - print("Not implemented yet.") - exit(1) +# Actions. +arg_parser.add_argument('--list', action='store_true', + help='list UUID and name of running VMs') +arg_parser.add_argument('--start', nargs=3, + metavar=('IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'), + help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix') +arg_parser.add_argument('--stop', metavar='UUID', + help='stop a VM') +arg_parser.add_argument('--get-status', metavar='UUID', + help='return the status of the VM') +arg_parser.add_argument('--get-vnc', metavar='UUID', + help='return the path of the VNC socket of the VM') +arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK', + help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix') -def require_with(arguments, required, mode): - if not arguments[required]: - print("--{} is required with the {} flag. Exiting.".format(required, mode)) - exit(1) +# Arguments. +arg_parser.add_argument('--workdir', default=Path.home(), + help='Working directory, defaulting to $HOME') +arg_parser.add_argument('--mac', + help='MAC address of the VM to create (--start)') +arg_parser.add_argument('--memory', type=int, + help='Memory (MB) to allocate (--start)') +arg_parser.add_argument('--cores', type=int, + help='Number of cores to allocate (--start)') +arg_parser.add_argument('--threads', type=int, + help='Number of threads to allocate (--start)') +arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], + help='Format of OS image (--start)') +arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='tcg', + help='QEMU acceleration to use (--start)') +arg_parser.add_argument('--upstream-interface', default='eth0', + help='Name of upstream interface (--start)') + +### +# Helpers. + +# XXX: check if it is possible to use the type returned by ETCD queries. +class UncloudEntryWrapper: + def __init__(self, value): + self.value = value + + def value(self): + return self.value + +def status_line(vm): + return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status()) + +### +# Entrypoint. def main(arguments): - # Initialize VMM + # Initialize VMM. workdir = arguments['workdir'] vmm = VMM(vmm_backend=workdir) - # Initialize workdir directory. - # TODO: copy ifup, ifdown. + # Harcoded debug values. + net_id = 0 # Build VM configuration. vm_config = {} - for spec in ['uuid', 'memory', 'cores', 'threads', 'image', 'image_format', 'name']: - if arguments.get(spec): - vm_config[spec] = arguments[spec] + vm_options = [ + 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', + '--upstream_interface', 'upstream_interface', 'network' + ] + for option in vm_options: + if arguments.get(option): + vm_config[option] = arguments[option] + + vm_config['net_id'] = net_id # Execute requested VM action. - vm = virtualmachine.VM(vmm, vm_config) - if arguments['setup_network']: - setup_network() - elif arguments['start_vm']: - require_with(arguments, 'image', 'start_vm') + if arguments['reconfigure_radvd']: + # TODO: check that RADVD is available. + prefix = arguments['reconfigure_radvd'] + network = UncloudEntryWrapper({ + 'id': net_id, + 'ipv6': prefix + }) + + # Make use of uncloud.host.virtualmachine for network configuration. + update_radvd_conf([network]) + elif arguments['start']: + # Extract from --start positional arguments. Quite fragile. + vm_config['image'] = arguments['start'][0] + vm_config['network'] = arguments['start'][1] + vm_config['upstream_interface'] = arguments['start'][2] + + vm_config['tap_interface'] = "uc{}".format(len(vmm.discover())) + vm = virtualmachine.VM(vmm, vm_config) vm.start() - logger.info("Created VM {}".format(vm.get_uuid)) - elif arguments['get_vm_status']: - require_with(arguments, 'uuid', 'get_vm_status') - print("VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status())) - elif arguments['stop_vm']: - require_with(arguments, 'uuid', 'stop_vm') + elif arguments['stop']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']}) + vm = virtualmachine.VM(vmm, vm_config) vm.stop() - elif arguments['list_vms']: - discovered = vmm.discover() - print("Found {} VMs.".format(len(discovered))) - for uuid in vmm.discover(): - vmi = virtualmachine.VM(vmm, {'uuid': uuid}) - print("VM: {} {} {}".format(vmi.get_uuid, vmi.get_name, vmi.get_status)) + elif arguments['get_status']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']}) + print(status_line(vm)) + elif arguments['get_vnc']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']}) + print(vm.get_vnc_addr()) + elif arguments['list']: + vms = vmm.discover() + print("Found {} VMs.".format(len(vms))) + for uuid in vms: + vm = virtualmachine.VM(vmm, {'uuid': uuid}) + print(status_line(vm)) else: - print('No action requested. Exiting.') + print('Please specify an action: --start, --stop, --list,\ +--get-status, --get-vnc, --reconfigure-radvd') diff --git a/uncloud/oneshot/virtualmachine.py b/uncloud/oneshot/virtualmachine.py index 47365d5..1388d49 100644 --- a/uncloud/oneshot/virtualmachine.py +++ b/uncloud/oneshot/virtualmachine.py @@ -1,6 +1,7 @@ import uuid import os +from uncloud.host.virtualmachine import create_vxlan_br_tap from uncloud.oneshot import logger class VM(object): @@ -9,29 +10,36 @@ class VM(object): self.vmm = vmm # Extract VM specs/metadata from configuration. - self.name = config.get('name') + self.name = config.get('name', 'no-name') self.memory = config.get('memory', 1024) self.cores = config.get('cores', 1) self.threads = config.get('threads', 1) self.image_format = config.get('image_format', 'qcow2') self.image = config.get('image') - self.uuid = config.get('uuid', uuid.uuid4()) - self.mac = config.get('mac', 'spuik') + self.uuid = config.get('uuid', str(uuid.uuid4())) + self.mac = config.get('mac') + + self.net_id = config.get('net_id', 0) + self.upstream_interface = config.get('upstream_interface', 'eth0') + self.tap_interface = config.get('tap_interface', 'uc0') + self.network = config.get('network') # Harcoded & generated values. - self.image_format='qcow2' self.accel = 'kvm' def get_qemu_args(self): command = ( - "-uuid {uuid} -name {name}" + "-uuid {uuid} -name {name} -machine pc,accel={accel}" " -drive file={image},format={image_format},if=virtio" " -device virtio-rng-pci" " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" ).format( - uuid=self.uuid, name=self.name, + uuid=self.uuid, name=self.name, accel=self.accel, image=self.image, image_format=self.image_format, memory=self.memory, cores=self.cores, threads=self.threads, + net_id=self.net_id, tap=self.tap_interface, mac=self.mac ) return command.split(" ") @@ -41,9 +49,14 @@ class VM(object): if not os.path.isfile(self.image): logger.error("Image {} does not exist. Aborting.".format(self.image)) + # Create Bridge, VXLAN and tap interface for VM. + create_vxlan_br_tap( + self.net_id, self.upstream_interface, self.tap_interface, self.network + ) + # Generate config for and run QEMU. qemu_args = self.get_qemu_args() - logger.warning("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) + logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) self.vmm.start( uuid=self.uuid, migration=False, From e14ac947c14bbad0cfeef66b5397a2feb2a3b043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 30 Jan 2020 08:44:41 +0100 Subject: [PATCH 163/163] Fix --accel parameter for oneshot --- uncloud/oneshot/main.py | 4 ++-- uncloud/oneshot/virtualmachine.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/uncloud/oneshot/main.py b/uncloud/oneshot/main.py index 0e94a81..5b9b61c 100644 --- a/uncloud/oneshot/main.py +++ b/uncloud/oneshot/main.py @@ -40,7 +40,7 @@ arg_parser.add_argument('--threads', type=int, help='Number of threads to allocate (--start)') arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], help='Format of OS image (--start)') -arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='tcg', +arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm', help='QEMU acceleration to use (--start)') arg_parser.add_argument('--upstream-interface', default='eth0', help='Name of upstream interface (--start)') @@ -74,7 +74,7 @@ def main(arguments): vm_config = {} vm_options = [ 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', - '--upstream_interface', 'upstream_interface', 'network' + '--upstream_interface', 'upstream_interface', 'network', 'accel' ] for option in vm_options: if arguments.get(option): diff --git a/uncloud/oneshot/virtualmachine.py b/uncloud/oneshot/virtualmachine.py index 1388d49..c8c2909 100644 --- a/uncloud/oneshot/virtualmachine.py +++ b/uncloud/oneshot/virtualmachine.py @@ -18,15 +18,13 @@ class VM(object): self.image = config.get('image') self.uuid = config.get('uuid', str(uuid.uuid4())) self.mac = config.get('mac') + self.accel = config.get('accel', 'kvm') self.net_id = config.get('net_id', 0) self.upstream_interface = config.get('upstream_interface', 'eth0') self.tap_interface = config.get('tap_interface', 'uc0') self.network = config.get('network') - # Harcoded & generated values. - self.accel = 'kvm' - def get_qemu_args(self): command = ( "-uuid {uuid} -name {name} -machine pc,accel={accel}"