diff --git a/conf/uncloud.conf b/conf/uncloud.conf index 9d4358d..6a1b500 100644 --- a/conf/uncloud.conf +++ b/conf/uncloud.conf @@ -1,6 +1,7 @@ [etcd] url = localhost port = 2379 +base_prefix = / ca_cert cert_cert cert_key @@ -9,3 +10,4 @@ cert_key name = replace_me realm = replace_me seed = replace_me +api_server = http://localhost:5000 \ No newline at end of file diff --git a/scripts/uncloud b/scripts/uncloud index 968ace6..a6e61aa 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -3,10 +3,14 @@ import logging import sys import importlib import argparse -import multiprocessing as mp from uncloud import UncloudException -from contextlib import suppress + +# the components that use etcd +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('cli') def exception_hook(exc_type, exc_value, exc_traceback): @@ -27,32 +31,48 @@ if __name__ == '__main__': subparsers = arg_parser.add_subparsers(dest='command') parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument('--debug', '-d', action='store_true', default=False, + parent_parser.add_argument('--debug', '-d', + action='store_true', + default=False, help='More verbose logging') + parent_parser.add_argument('--conf-dir', '-c', + help='Configuration directory') - for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', - 'metadata', 'configure', 'cli']: + etcd_parser = argparse.ArgumentParser(add_help=False) + etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-port') + etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') + etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') + etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') + + for component in ALL_COMPONENTS: mod = importlib.import_module('uncloud.{}.main'.format(component)) parser = getattr(mod, 'arg_parser') - subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) + + if component in ETCD_COMPONENTS: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) + else: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) args = arg_parser.parse_args() if not args.command: arg_parser.print_help() else: - - # if we start etcd in seperate process with default settings - # i.e inheriting few things from parent process etcd3 module - # errors out, so the following command configure multiprocessing - # module to not inherit anything from parent. - mp.set_start_method('spawn') arguments = vars(args) + name = arguments.pop('command') + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') + + # If the component requires etcd3, we import it and catch the + # etcd3.exceptions.ConnectionFailedError + if name in ETCD_COMPONENTS: + import etcd3 + try: - name = arguments.pop('command') - mod = importlib.import_module('uncloud.{}.main'.format(name)) - main = getattr(mod, 'main') - main(**arguments) + main(arguments) except UncloudException as err: logger.error(err) + sys.exit(1) except Exception as err: logger.exception(err) + sys.exit(1) diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py index 8bcf777..d1fcb64 100755 --- a/uncloud/api/common_fields.py +++ b/uncloud/api/common_fields.py @@ -1,7 +1,7 @@ import os -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared +from uncloud.common.settings import settings class Optional: diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 73b92f1..1040e97 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -3,18 +3,18 @@ import os from uuid import uuid4 -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared +from uncloud.common.settings import settings data = { - "is_public": True, - "type": "ceph", - "name": "images", - "description": "first ever public image-store", - "attributes": {"list": [], "key": [], "pool": "images"}, + 'is_public': True, + 'type': 'ceph', + 'name': 'images', + 'description': 'first ever public image-store', + 'attributes': {'list': [], 'key': [], 'pool': 'images'}, } shared.etcd_client.put( - os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex), + os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data), ) diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index c806814..0805280 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -1,14 +1,13 @@ import binascii import ipaddress import random -import subprocess as sp import logging import requests from pyotp import TOTP -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared +from uncloud.common.settings import settings logger = logging.getLogger(__name__) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 47e7003..34e1dd1 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -10,11 +10,12 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException +from uncloud.common.shared import shared + from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.settings import settings from . import schemas from .helper import generate_mac, mac2ipv6 from uncloud import UncloudException @@ -41,7 +42,7 @@ def handle_exception(e): class CreateVM(Resource): - '''API Request to Handle Creation of VM''' + """API Request to Handle Creation of VM""" @staticmethod def post(): @@ -59,7 +60,7 @@ class CreateVM(Resource): macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, '/v1/counter/tap' + shared.etcd_client, settings['etcd']['tap_counter'] ) for _ in range(len(data['network'])) ] @@ -289,18 +290,16 @@ class ListUserFiles(Resource): settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] - user_files = list( - filter( - lambda f: f.value['owner'] == data['name'], files - ) - ) + user_files = [f for f in files if f.value['owner'] == data['name']] for file in user_files: - return_files.append( - { - 'filename': file.value['filename'], - 'uuid': file.key.split('/')[-1], - } - ) + file_uuid = file.key.split('/')[-1] + file = file.value + file['uuid'] = file_uuid + + file.pop('sha512sum', None) + file.pop('owner', None) + + return_files.append(file) return {'message': return_files}, 200 else: return validator.get_errors(), 400 @@ -472,7 +471,7 @@ class CreateNetwork(Resource): network_entry = { 'id': counters.increment_etcd_counter( - shared.etcd_client, '/v1/counter/vxlan' + shared.etcd_client, settings['etcd']['vxlan_counter'] ), 'type': data['type'], } @@ -564,7 +563,10 @@ api.add_resource(ListHost, '/host/list') api.add_resource(CreateNetwork, '/network/create') -def main(debug=False, port=None): +def main(arguments): + debug = arguments['debug'] + port = arguments['port'] + try: image_stores = list( shared.etcd_client.get_prefix( @@ -594,12 +596,6 @@ def main(debug=False, port=None): # ) try: - app.run(host='::', - port=port, - debug=debug) + app.run(host='::', port=port, debug=debug) except OSError as e: raise UncloudException('Failed to start Flask: {}'.format(e)) - - -if __name__ == '__main__': - main() diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index 8e06e8d..e4de9a8 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -21,8 +21,8 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared +from uncloud.common.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py index bdcce78..3c63073 100644 --- a/uncloud/cli/helper.py +++ b/uncloud/cli/helper.py @@ -5,15 +5,24 @@ import binascii from pyotp import TOTP from os.path import join as join_path -from uncloud.settings import settings +from uncloud.common.settings import settings def get_otp_parser(): otp_parser = argparse.ArgumentParser('otp') - otp_parser.add_argument('--name', default=settings['client']['name']) - otp_parser.add_argument('--realm', default=settings['client']['realm']) - otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'], - dest='token', metavar='SEED') + try: + name = settings['client']['name'] + realm = settings['client']['realm'] + seed = settings['client']['seed'] + except Exception: + otp_parser.add_argument('--name', required=True) + otp_parser.add_argument('--realm', required=True) + otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED') + else: + otp_parser.add_argument('--name', default=name) + otp_parser.add_argument('--realm', default=realm) + otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED') + return otp_parser diff --git a/uncloud/cli/image.py b/uncloud/cli/image.py index 641a00f..2f59c32 100644 --- a/uncloud/cli/image.py +++ b/uncloud/cli/image.py @@ -12,7 +12,7 @@ class ImageParser(BaseParser): p = self.subparser.add_parser('create', **kwargs) p.add_argument('--name', required=True) p.add_argument('--uuid', required=True) - p.add_argument('--image-store-name', default='image_store') + p.add_argument('--image-store', required=True, dest='image_store') def list(self, **kwargs): self.subparser.add_parser('list', **kwargs) diff --git a/uncloud/cli/main.py b/uncloud/cli/main.py index 7f5e367..9a42497 100644 --- a/uncloud/cli/main.py +++ b/uncloud/cli/main.py @@ -12,12 +12,12 @@ for component in ['user', 'host', 'image', 'network', 'vm']: subparser.add_parser(name=parser.prog, parents=[parser]) -def main(**kwargs): - if not kwargs['subcommand']: +def main(arguments): + if not arguments['subcommand']: arg_parser.print_help() else: - name = kwargs.pop('subcommand') - kwargs.pop('debug') + name = arguments.pop('subcommand') + arguments.pop('debug') mod = importlib.import_module('uncloud.cli.{}'.format(name)) _main = getattr(mod, 'main') - _main(**kwargs) + _main(**arguments) diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index 6a979ba..38471ab 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -1,24 +1,21 @@ import etcd3 import json -import queue -import copy -from uncloud import UncloudException -from collections import namedtuple from functools import wraps -from . import logger - -PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) +from uncloud import UncloudException +from uncloud.common import logger class EtcdEntry: - # key: str - # value: str - - def __init__(self, meta, value, value_in_json=False): - self.key = meta.key.decode("utf-8") - self.value = value.decode("utf-8") + def __init__(self, meta_or_key, value, value_in_json=False): + if hasattr(meta_or_key, 'key'): + # if meta has attr 'key' then get it + self.key = meta_or_key.key.decode('utf-8') + else: + # otherwise meta is the 'key' + self.key = meta_or_key + self.value = value.decode('utf-8') if value_in_json: self.value = json.loads(self.value) @@ -29,18 +26,12 @@ def readable_errors(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) - except etcd3.exceptions.ConnectionFailedError as err: - raise UncloudException( - "Cannot connect to etcd: is etcd running as configured in uncloud.conf?" - ) + except etcd3.exceptions.ConnectionFailedError: + raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') except etcd3.exceptions.ConnectionTimeoutError as err: - raise etcd3.exceptions.ConnectionTimeoutError( - "etcd connection timeout." - ) from err + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err except Exception: - logger.exception( - "Some etcd error occured. See syslog for details." - ) + logger.exception('Some etcd error occured. See syslog for details.') return wrapper @@ -64,55 +55,21 @@ class Etcd3Wrapper: _value = json.dumps(_value) if not isinstance(_key, str): - _key = _key.decode("utf-8") + _key = _key.decode('utf-8') return self.client.put(_key, _value, **kwargs) @readable_errors - def get_prefix(self, *args, value_in_json=False, **kwargs): - r = self.client.get_prefix(*args, **kwargs) - for entry in r: - e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) - if e.value: - yield e + def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) @readable_errors - def watch_prefix(self, key, timeout=0, value_in_json=False): - timeout_event = EtcdEntry( - PseudoEtcdMeta(key=b"TIMEOUT"), - value=str.encode( - json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"}) - ), - value_in_json=value_in_json, - ) - - event_queue = queue.Queue() - - def add_event_to_queue(event): - if hasattr(event, "events"): - for e in event.events: - if e.value: - event_queue.put( - EtcdEntry( - e, e.value, value_in_json=value_in_json - ) - ) - - self.client.add_watch_prefix_callback(key, add_event_to_queue) - - while True: - try: - while True: - v = event_queue.get(timeout=timeout) - yield v - except queue.Empty: - event_queue.put(copy.deepcopy(timeout_event)) - - -class PsuedoEtcdEntry(EtcdEntry): - def __init__(self, key, value, value_in_json=False): - super().__init__( - PseudoEtcdMeta(key=key.encode("utf-8")), - value, - value_in_json=value_in_json, - ) + def watch_prefix(self, key, raise_exception=True, value_in_json=False): + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = e._event + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) diff --git a/uncloud/common/network.py b/uncloud/common/network.py index adba108..32f6951 100644 --- a/uncloud/common/network.py +++ b/uncloud/common/network.py @@ -1,8 +1,6 @@ import subprocess as sp import random import logging -import socket -from contextlib import closing logger = logging.getLogger(__name__) diff --git a/uncloud/common/request.py b/uncloud/common/request.py index a8c2d0a..cb0add5 100644 --- a/uncloud/common/request.py +++ b/uncloud/common/request.py @@ -2,8 +2,8 @@ import json from os.path import join from uuid import uuid4 -from .etcd_wrapper import PsuedoEtcdEntry -from .classes import SpecificEtcdEntryBase +from uncloud.common.etcd_wrapper import EtcdEntry +from uncloud.common.classes import SpecificEtcdEntryBase class RequestType: @@ -29,11 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase): @classmethod def from_scratch(cls, request_prefix, **kwargs): - e = PsuedoEtcdEntry( - join(request_prefix, uuid4().hex), - value=json.dumps(kwargs).encode("utf-8"), - value_in_json=True, - ) + e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex), + value=json.dumps(kwargs).encode('utf-8'), value_in_json=True) return cls(e) diff --git a/uncloud/settings/__init__.py b/uncloud/common/settings.py similarity index 54% rename from uncloud/settings/__init__.py rename to uncloud/common/settings.py index f6da61c..0d524a7 100644 --- a/uncloud/settings/__init__.py +++ b/uncloud/common/settings.py @@ -4,8 +4,8 @@ import sys import os from datetime import datetime - from uncloud.common.etcd_wrapper import Etcd3Wrapper +from os.path import join as join_path logger = logging.getLogger(__name__) @@ -16,7 +16,7 @@ class CustomConfigParser(configparser.RawConfigParser): result = super().__getitem__(key) except KeyError as err: raise KeyError( - "Key '{}' not found in configuration. Make sure you configure uncloud.".format( + 'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format( key ) ) from err @@ -25,40 +25,41 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): - def __init__(self, config_key="/uncloud/config/"): - conf_name = "uncloud.conf" - conf_dir = os.environ.get( - "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/") - ) - self.config_file = os.path.join(conf_dir, conf_name) - self.config_parser = CustomConfigParser(allow_no_value=True) - self.config_key = config_key + def __init__(self): + conf_name = 'uncloud.conf' + conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/')) + self.config_file = join_path(conf_dir, conf_name) # this is used to cache config from etcd for 1 minutes. Without this we # would make a lot of requests to etcd which slows down everything. self.last_config_update = datetime.fromtimestamp(0) - self.read_internal_values() + self.config_parser = CustomConfigParser(allow_no_value=True) + self.config_parser.add_section('etcd') + self.config_parser.set('etcd', 'base_prefix', '/') + try: self.config_parser.read(self.config_file) except Exception as err: - logger.error("%s", err) + logger.error('%s', err) + + self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') + + self.read_internal_values() def get_etcd_client(self): args = tuple() try: kwargs = { - "host": self.config_parser.get("etcd", "url"), - "port": self.config_parser.get("etcd", "port"), - "ca_cert": self.config_parser.get("etcd", "ca_cert"), - "cert_cert": self.config_parser.get( - "etcd", "cert_cert" - ), - "cert_key": self.config_parser.get("etcd", "cert_key"), + 'host': self.config_parser.get('etcd', 'url'), + 'port': self.config_parser.get('etcd', 'port'), + 'ca_cert': self.config_parser.get('etcd', 'ca_cert'), + 'cert_cert': self.config_parser.get('etcd', 'cert_cert'), + 'cert_key': self.config_parser.get('etcd', 'cert_key'), } except configparser.Error as err: raise configparser.Error( - "{} in config file {}".format( + '{} in config file {}'.format( err.message, self.config_file ) ) from err @@ -67,8 +68,8 @@ class Settings(object): wrapper = Etcd3Wrapper(*args, **kwargs) except Exception as err: logger.error( - "etcd connection not successfull. Please check your config file." - "\nDetails: %s\netcd connection parameters: %s", + 'etcd connection not successfull. Please check your config file.' + '\nDetails: %s\netcd connection parameters: %s', err, kwargs, ) @@ -77,17 +78,20 @@ class Settings(object): return wrapper def read_internal_values(self): + base_prefix = self['etcd']['base_prefix'] self.config_parser.read_dict( { - "etcd": { - "file_prefix": "/files/", - "host_prefix": "/hosts/", - "image_prefix": "/images/", - "image_store_prefix": "/imagestore/", - "network_prefix": "/networks/", - "request_prefix": "/requests/", - "user_prefix": "/users/", - "vm_prefix": "/vms/", + 'etcd': { + 'file_prefix': join_path(base_prefix, 'files/'), + 'host_prefix': join_path(base_prefix, 'hosts/'), + 'image_prefix': join_path(base_prefix, 'images/'), + 'image_store_prefix': join_path(base_prefix, 'imagestore/'), + 'network_prefix': join_path(base_prefix, 'networks/'), + 'request_prefix': join_path(base_prefix, 'requests/'), + 'user_prefix': join_path(base_prefix, 'users/'), + 'vm_prefix': join_path(base_prefix, 'vms/'), + 'vxlan_counter': join_path(base_prefix, 'counters/vxlan'), + 'tap_counter': join_path(base_prefix, 'counters/tap') } } ) @@ -95,15 +99,13 @@ class Settings(object): def read_config_file_values(self, config_file): try: # Trying to read configuration file - with open(config_file, "r") as config_file_handle: + with open(config_file) as config_file_handle: self.config_parser.read_file(config_file_handle) except FileNotFoundError: - sys.exit( - "Configuration file {} not found!".format(config_file) - ) + sys.exit('Configuration file {} not found!'.format(config_file)) except Exception as err: logger.exception(err) - sys.exit("Error occurred while reading configuration file") + sys.exit('Error occurred while reading configuration file') def read_values_from_etcd(self): etcd_client = self.get_etcd_client() @@ -113,7 +115,7 @@ class Settings(object): self.config_parser.read_dict(config_from_etcd.value) self.last_config_update = datetime.utcnow() else: - raise KeyError("Key '{}' not found in etcd. Please configure uncloud.".format(self.config_key)) + raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key)) def __getitem__(self, key): # Allow failing to read from etcd if we have @@ -121,9 +123,8 @@ class Settings(object): if key not in self.config_parser.sections(): try: self.read_values_from_etcd() - except KeyError as e: + except KeyError: pass - return self.config_parser[key] diff --git a/uncloud/shared/__init__.py b/uncloud/common/shared.py similarity index 94% rename from uncloud/shared/__init__.py rename to uncloud/common/shared.py index db2093f..918dd0c 100644 --- a/uncloud/shared/__init__.py +++ b/uncloud/common/shared.py @@ -1,4 +1,4 @@ -from uncloud.settings import settings +from uncloud.common.settings import settings from uncloud.common.vm import VmPool from uncloud.common.host import HostPool from uncloud.common.request import RequestPool diff --git a/uncloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py index 06751c4..6f9b29e 100644 --- a/uncloud/common/storage_handlers.py +++ b/uncloud/common/storage_handlers.py @@ -7,7 +7,7 @@ from abc import ABC from . import logger from os.path import join as join_path -from uncloud.settings import settings as config +from uncloud.common.settings import settings as config class ImageStorageHandler(ABC): diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index f89a30c..e190460 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,8 +1,8 @@ import os import argparse -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.settings import settings +from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('configure', add_help=False) configure_subparsers = arg_parser.add_subparsers(dest='subcommand') @@ -40,18 +40,14 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True) def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get( - settings.config_key, value_in_json=True - ) + uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - shared.etcd_client.put( - settings.config_key, uncloud_config, value_in_json=True - ) + shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) def main(**kwargs): diff --git a/uncloud/docs/README.md b/uncloud/docs/README.md new file mode 100644 index 0000000..a5afbaa --- /dev/null +++ b/uncloud/docs/README.md @@ -0,0 +1,12 @@ +# uncloud docs + +## Requirements +1. Python3 +2. Sphinx + +## Usage +Run `make build` to build docs. + +Run `make clean` to remove build directory. + +Run `make publish` to push build dir to https://ungleich.ch/ucloud/ \ No newline at end of file diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index c81fbbe..c5660dd 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -4,15 +4,16 @@ import pathlib import subprocess as sp import time import argparse +import bitmath from uuid import uuid4 from . import logger -from uncloud.settings import settings -from uncloud.shared import shared - +from uncloud.common.settings import settings +from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('filescanner', add_help=False) +arg_parser.add_argument('--hostname', required=True) def sha512sum(file: str): @@ -28,66 +29,58 @@ def sha512sum(file: str): if not isinstance(file, str): raise TypeError try: - output = sp.check_output(["sha512sum", file], stderr=sp.PIPE) + output = sp.check_output(['sha512sum', file], stderr=sp.PIPE) except sp.CalledProcessError as e: - error = e.stderr.decode("utf-8") - if "No such file or directory" in error: + error = e.stderr.decode('utf-8') + if 'No such file or directory' in error: raise FileNotFoundError from None else: - output = output.decode("utf-8").strip() - output = output.split(" ") + output = output.decode('utf-8').strip() + output = output.split(' ') return output[0] return None -def track_file(file, base_dir): - file_id = uuid4() - +def track_file(file, base_dir, host): + file_path = file.relative_to(base_dir) + file_str = str(file) # Get Username - owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] + try: + owner = file_path.parts[0] + except IndexError: + pass + else: + file_path = file_path.relative_to(owner) + creation_date = time.ctime(os.stat(file_str).st_ctime) - # Get Creation Date of File - # Here, we are assuming that ctime is creation time - # which is mostly not true. - creation_date = time.ctime(os.stat(file).st_ctime) + entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) + entry_value = { + 'filename': str(file_path), + 'owner': owner, + 'sha512sum': sha512sum(file_str), + 'creation_date': creation_date, + 'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()), + 'host': host + } - file_path = pathlib.Path(file).parts[-1] + logger.info('Tracking %s', file_str) - # Create Entry - entry_key = os.path.join( - settings["etcd"]["file_prefix"], str(file_id) - ) - entry_value = { - "filename": file_path, - "owner": owner, - "sha512sum": sha512sum(file), - "creation_date": creation_date, - "size": os.path.getsize(file), - } - - logger.info("Tracking %s", file) - - shared.etcd_client.put(entry_key, entry_value, value_in_json=True) - os.setxattr(file, "user.utracked", b"True") + shared.etcd_client.put(entry_key, entry_value, value_in_json=True) -def main(debug=False): - base_dir = settings["storage"]["file_dir"] - +def main(arguments): + hostname = arguments['hostname'] + base_dir = settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR - files = glob.glob("{}/**".format(base_dir), recursive=True) + files = glob.glob('{}/**'.format(base_dir), recursive=True) + files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] - # Retain only Files - files = [file for file in files if os.path.isfile(file)] - - untracked_files = [] - for file in files: - try: - os.getxattr(file, "user.utracked") - except OSError: - track_file(file, base_dir) - untracked_files.append(file) - - -if __name__ == "__main__": - main() + # Files that are already tracked + tracked_files = [ + pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) + for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) + if f.value['host'] == hostname + ] + untracked_files = set(files) - set(tracked_files) + for file in untracked_files: + track_file(file, base_dir, hostname) diff --git a/uncloud/hack/hackcloud/.gitignore b/uncloud/hack/hackcloud/.gitignore new file mode 100644 index 0000000..0ad647b --- /dev/null +++ b/uncloud/hack/hackcloud/.gitignore @@ -0,0 +1,3 @@ +*.iso +radvdpid +foo diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh new file mode 100755 index 0000000..99e8690 --- /dev/null +++ b/uncloud/hack/hackcloud/ifup.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +echo $@ >> foo + +dev=$1; shift + +# bridge is setup from outside +ip link set dev "$dev" master ${bridge} +ip link set dev "$dev" up diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh new file mode 100755 index 0000000..7d4b88f --- /dev/null +++ b/uncloud/hack/hackcloud/net.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +netid=100 +dev=wlp2s0 +dev=wlp0s20f3 +dev=wlan0 + +ip=2a0a:e5c1:111:888::42/64 +vxlandev=vxlan${netid} +bridgedev=br${netid} + +ip -6 link add ${vxlandev} type vxlan \ + id ${netid} \ + dstport 4789 \ + group ff05::${netid} \ + dev ${dev} \ + ttl 5 + +ip link set ${vxlandev} up + +ip link add ${bridgedev} type bridge +ip link set ${bridgedev} up + +ip addr add ${ip} dev ${bridgedev} diff --git a/uncloud/hack/hackcloud/radvd.conf b/uncloud/hack/hackcloud/radvd.conf new file mode 100644 index 0000000..3d8ce4d --- /dev/null +++ b/uncloud/hack/hackcloud/radvd.conf @@ -0,0 +1,13 @@ +interface br100 +{ + AdvSendAdvert on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 5; + AdvDefaultLifetime 3600; + + prefix 2a0a:e5c1:111:888::/64 { + }; + + RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; }; + DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ; +}; diff --git a/uncloud/hack/hackcloud/radvd.sh b/uncloud/hack/hackcloud/radvd.sh new file mode 100644 index 0000000..9d0e7d1 --- /dev/null +++ b/uncloud/hack/hackcloud/radvd.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +radvd -C ./radvd.conf -n -p ./radvdpid diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh new file mode 100755 index 0000000..2a8b794 --- /dev/null +++ b/uncloud/hack/hackcloud/vm.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +vmid=$1; shift + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +accel=tcg + +memory=1024 +cores=2 +uuid=732e08c7-84f8-4d43-9571-263db4f80080 + +export bridge=br100 + +$qemu -name uc${vmid} \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e + + + +exit 0 + +-S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-17-one-24992/master-key.aes +-machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off + +-m 2048 +-realtime mlock=off +-smp 1,sockets=1,cores=1,threads=1 +-uuid 732e08c7-84f8-4d43-9571-263db4f80080 -no-user-config \ + -nodefaults +-chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-17-one-24992/monitor.sock,server,nowait +-mon chardev=charmonitor,id=monitor,mode=control +-rtc base=utc -no-shutdown +-boot strict=on +-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 +-drive file=rbd:ssd/one-292-24992-0:id=libvirt:auth_supported=cephx\;none:mon_host=ceph1\:6789\;ceph2\:6789\;ceph3\:6789,format=raw,if=none,id=drive-virtio-disk0,cache=none +-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 +-drive file=/var/lib/one//datastores/104/24992/disk.1,format=raw,if=none,id=drive-ide0-0-0,readonly=on +-device ide-cd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0 +-netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38 +-device virtio-net-pci,netdev=hostnet0,id=net0,mac=02:00:f0:a9:c4:4e,bus=pci.0,addr=0x3 +-vnc [::]:4414 -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on diff --git a/uncloud/host/main.py b/uncloud/host/main.py index ec2ef4d..ccffd77 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -5,8 +5,8 @@ import time from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared +from uncloud.common.settings import settings from uncloud.common.vm import VMStatus from uncloud.vmm import VMM from os.path import join as join_path @@ -33,10 +33,10 @@ def maintenance(host): vmm = VMM() running_vms = vmm.discover() for vm_uuid in running_vms: - if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running": + if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], vm_uuid) + join_path(settings['etcd']['vm_prefix'], vm_uuid) ) vm.status = VMStatus.running vm.vnc_socket = vmm.get_vnc(vm_uuid) @@ -44,20 +44,21 @@ def maintenance(host): shared.vm_pool.put(vm) -def main(hostname, debug=False): +def main(arguments): + hostname = arguments['hostname'] host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) # Does not yet exist, create it if not host: host_key = join_path( - settings["etcd"]["host_prefix"], uuid4().hex + settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { - "specs": "", - "hostname": hostname, - "status": "DEAD", - "last_heartbeat": "", + 'specs': '', + 'hostname': hostname, + 'status': 'DEAD', + 'last_heartbeat': '', } shared.etcd_client.put( host_key, host_entry, value_in_json=True @@ -70,54 +71,54 @@ def main(hostname, debug=False): heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) heartbeat_updating_process.start() except Exception as e: - raise Exception("uncloud-host heartbeat updating mechanism is not working") from e + raise Exception('uncloud-host heartbeat updating mechanism is not working') from e - for events_iterator in [ - shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True), - shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True) - ]: - for request_event in events_iterator: - request_event = RequestEntry(request_event) + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for events_iterator in [ + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False) + ]: + for request_event in events_iterator: + request_event = RequestEntry(request_event) - if request_event.type == "TIMEOUT": maintenance(host.key) - elif request_event.hostname == host.key: - logger.debug("VM Request: %s on Host %s", request_event, host.hostname) - shared.request_pool.client.client.delete(request_event.key) - vm_entry = shared.etcd_client.get( - join_path(settings["etcd"]["vm_prefix"], request_event.uuid) - ) - logger.debug("VM hostname: {}".format(vm_entry.value)) - vm = virtualmachine.VM(vm_entry) - if request_event.type == RequestType.StartVM: - vm.start() + if request_event.hostname == host.key: + logger.debug('VM Request: %s on Host %s', request_event, host.hostname) - elif request_event.type == RequestType.StopVM: - vm.stop() + shared.request_pool.client.client.delete(request_event.key) + vm_entry = shared.etcd_client.get( + join_path(settings['etcd']['vm_prefix'], request_event.uuid) + ) - elif request_event.type == RequestType.DeleteVM: - vm.delete() + logger.debug('VM hostname: {}'.format(vm_entry.value)) - elif request_event.type == RequestType.InitVMMigration: - vm.start(destination_host_key=host.key) + vm = virtualmachine.VM(vm_entry) + if request_event.type == RequestType.StartVM: + vm.start() - elif request_event.type == RequestType.TransferVM: - destination_host = host_pool.get(request_event.destination_host_key) - if destination_host: - vm.migrate( - destination_host=destination_host.hostname, - destination_sock_path=request_event.destination_sock_path, - ) - else: - logger.error("Host %s not found!", request_event.destination_host_key) + elif request_event.type == RequestType.StopVM: + vm.stop() + elif request_event.type == RequestType.DeleteVM: + vm.delete() -if __name__ == "__main__": - argparser = argparse.ArgumentParser() - argparser.add_argument( - "hostname", help="Name of this host. e.g uncloud1.ungleich.ch" - ) - args = argparser.parse_args() - mp.set_start_method("spawn") - main(args.hostname) + elif request_event.type == RequestType.InitVMMigration: + vm.start(destination_host_key=host.key) + + elif request_event.type == RequestType.TransferVM: + destination_host = host_pool.get(request_event.destination_host_key) + if destination_host: + vm.migrate( + destination_host=destination_host.hostname, + destination_sock_path=request_event.destination_sock_path, + ) + else: + logger.error('Host %s not found!', request_event.destination_host_key) diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index 0bd20bf..2f6a5e3 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -16,8 +16,8 @@ from uncloud.common.vm import VMStatus, declare_stopped from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared +from uncloud.common.settings import settings from uncloud.vmm import VMM from marshmallow import ValidationError @@ -42,7 +42,7 @@ class VM: def get_qemu_args(self): command = ( - "-drive file={file},format=raw,if=virtio,cache=none" + "-drive file={file},format=raw,if=virtio" " -device virtio-rng-pci" " -m {memory} -smp cores={cores},threads={threads}" " -name {owner}_{name}" @@ -153,7 +153,10 @@ class VM: ) ) - return command.split(" ") + if command: + command = command.split(' ') + + return command def delete_network_dev(self): try: diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index a43a36c..1803213 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -4,8 +4,8 @@ import argparse import subprocess as sp from os.path import join as join_path -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.settings import settings +from uncloud.common.shared import shared from uncloud.imagescanner import logger @@ -30,7 +30,7 @@ def qemu_img_type(path): return qemu_img_info["format"] -def main(debug=False): +def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( settings["etcd"]["image_prefix"], value_in_json=True diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index e2199b8..ccda60e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -5,8 +5,8 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.settings import settings +from uncloud.common.shared import shared app = Flask(__name__) api = Api(app) @@ -84,40 +84,11 @@ class Root(Resource): data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys return data.value["metadata"], 200 - @staticmethod - def post(): - return {"message": "Previous Implementation is deprecated."} - # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True) - # print(data) - # if data: - # for k in request.json: - # if k not in data.value: - # data.value[k] = request.json[k] - # if k.endswith("-list"): - # data.value[k] = [request.json[k]] - # else: - # if k.endswith("-list"): - # data.value[k].append(request.json[k]) - # else: - # data.value[k] = request.json[k] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data.value, value_in_json=True) - # else: - # data = {} - # for k in request.json: - # data[k] = request.json[k] - # if k.endswith("-list"): - # data[k] = [request.json[k]] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data, value_in_json=True) - api.add_resource(Root, "/") -def main(port=None, debug=False): +def main(arguments): + port = arguments['port'] + debug = arguments['debug'] app.run(debug=debug, host="::", port=port) - - -if __name__ == "__main__": - main() diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index 7edf623..108d126 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -6,8 +6,8 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared +from uncloud.common.settings import settings def accumulated_specs(vms_specs): diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 1ef6226..c25700b 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -6,59 +6,47 @@ import argparse +from uncloud.common.settings import settings from uncloud.common.request import RequestEntry, RequestType -from uncloud.shared import shared -from uncloud.settings import settings -from .helper import (dead_host_mitigation, dead_host_detection, assign_host, NoSuitableHostFound) -from . import logger +from uncloud.common.shared import shared +from uncloud.scheduler import logger +from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection, + assign_host, NoSuitableHostFound) arg_parser = argparse.ArgumentParser('scheduler', add_help=False) -def main(debug=False): - for request_iterator in [ - shared.etcd_client.get_prefix( - settings["etcd"]["request_prefix"], value_in_json=True - ), - shared.etcd_client.watch_prefix( - settings["etcd"]["request_prefix"], - timeout=5, - value_in_json=True, - ), - ]: - for request_event in request_iterator: - request_entry = RequestEntry(request_event) - # Never Run time critical mechanism inside timeout - # mechanism because timeout mechanism only comes - # when no other event is happening. It means under - # heavy load there would not be a timeout event. - if request_entry.type == "TIMEOUT": +def main(arguments): + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for request_iterator in [ + shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + ]: + for request_event in request_iterator: + dead_host_mitigation(dead_host_detection()) + request_entry = RequestEntry(request_event) - # Detect hosts that are dead and set their status - # to "DEAD", and their VMs' status to "KILLED" - dead_hosts = dead_host_detection() - if dead_hosts: - logger.debug("Dead hosts: %s", dead_hosts) - dead_host_mitigation(dead_hosts) + if request_entry.type == RequestType.ScheduleVM: + logger.debug('%s, %s', request_entry.key, request_entry.value) - elif request_entry.type == RequestType.ScheduleVM: - logger.debug("%s, %s", request_entry.key, request_entry.value) + vm_entry = shared.vm_pool.get(request_entry.uuid) + if vm_entry is None: + logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid)) + continue - vm_entry = shared.vm_pool.get(request_entry.uuid) - if vm_entry is None: - logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) - continue + shared.etcd_client.client.delete(request_entry.key) # consume Request - shared.etcd_client.client.delete(request_entry.key) # consume Request + try: + assign_host(vm_entry) + except NoSuitableHostFound: + vm_entry.add_log('Can\'t schedule VM. No Resource Left.') + shared.vm_pool.put(vm_entry) - try: - assign_host(vm_entry) - except NoSuitableHostFound: - vm_entry.add_log("Can't schedule VM. No Resource Left.") - shared.vm_pool.put(vm_entry) - - logger.info("No Resource Left. Emailing admin....") - - -if __name__ == "__main__": - main() + logger.info('No Resource Left. Emailing admin....') diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 6cdd938..4c893f6 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -190,18 +190,10 @@ class VMM: err.stderr.decode("utf-8"), ) else: - with suppress(sp.CalledProcessError): - sp.check_output( - [ - "sudo", - "-p", - "Enter password to correct permission for uncloud-vmm's directory", - "chmod", - "-R", - "o=rwx,g=rwx", - self.vmm_backend, - ] - ) + sp.check_output( + ["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory", + "chmod", "-R", "o=rwx,g=rwx", self.vmm_backend] + ) # TODO: Find some good way to check whether the virtual machine is up and # running without relying on non-guarenteed ways.