From 388127bd11e45eb9eda1be1279030ca4d2901ae6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 5 Jan 2020 18:32:14 +0100 Subject: [PATCH 01/75] [hack] add scripts to start VM --- uncloud/hack/uncloud-hack-init-host | 26 ++++++++++++++++++++++++++ uncloud/hack/uncloud-run-vm | 21 +++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 uncloud/hack/uncloud-hack-init-host create mode 100644 uncloud/hack/uncloud-run-vm diff --git a/uncloud/hack/uncloud-hack-init-host b/uncloud/hack/uncloud-hack-init-host new file mode 100644 index 0000000..787ff80 --- /dev/null +++ b/uncloud/hack/uncloud-hack-init-host @@ -0,0 +1,26 @@ +id=100 +rawdev=eth0 + +# create vxlan +ip -6 link add vxlan${id} type vxlan \ + id ${id} \ + dstport 4789 \ + group ff05::${id} \ + dev ${rawdev} \ + ttl 5 + +ip link set vxlan${id} up + +# create bridge +ip link set vxlan${id} up +ip link set br${id} up + +# Add vxlan into bridge +ip link set vxlan${id} master br${id} + + +# useradd -m uncloud +# [18:05] tablett.place10:~# id uncloud +# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu) +# apk add qemu-system-x86_64 +# also needs group netdev diff --git a/uncloud/hack/uncloud-run-vm b/uncloud/hack/uncloud-run-vm new file mode 100644 index 0000000..1af2037 --- /dev/null +++ b/uncloud/hack/uncloud-run-vm @@ -0,0 +1,21 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo $0 vmid + exit 1 +fi + +id=$1; shift + +memory=512 +macaddress=02:00:b9:cb:70:${id} +netname=net${id}-1 + +qemu-system-x86_64 \ + -name uncloud-${id} \ + -accel kvm \ + -m ${memory} \ + -smp 2,sockets=2,cores=1,threads=1 \ + -device virtio-net-pci,netdev=net0,mac=$macaddress \ + -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \ + -vnc [::]:5900 From e5dd5e45c651f0e407ce8ace972195b0d1044877 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 10 Jan 2020 00:03:10 +0500 Subject: [PATCH 02/75] initial work --- uncloud/api/common_fields.py | 62 --- uncloud/api/create_image_store.py | 5 +- uncloud/api/helper.py | 110 ++---- uncloud/api/main.py | 480 +++++++++--------------- uncloud/api/schemas.py | 604 +++++++++++------------------- uncloud/common/counters.py | 12 +- uncloud/common/etcd_wrapper.py | 39 +- uncloud/common/network.py | 23 +- 8 files changed, 483 insertions(+), 852 deletions(-) delete mode 100755 uncloud/api/common_fields.py diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py deleted file mode 100755 index d1fcb64..0000000 --- a/uncloud/api/common_fields.py +++ /dev/null @@ -1,62 +0,0 @@ -import os - -from uncloud.common.shared import shared -from uncloud.common.settings import settings - - -class Optional: - pass - - -class Field: - def __init__(self, _name, _type, _value=None): - self.name = _name - self.value = _value - self.type = _type - self.__errors = [] - - def validation(self): - return True - - def is_valid(self): - if self.value == KeyError: - self.add_error( - "'{}' field is a required field".format(self.name) - ) - else: - if isinstance(self.value, Optional): - pass - elif not isinstance(self.value, self.type): - self.add_error( - "Incorrect Type for '{}' field".format(self.name) - ) - else: - self.validation() - - if self.__errors: - return False - return True - - def get_errors(self): - return self.__errors - - def add_error(self, error): - self.__errors.append(error) - - -class VmUUIDField(Field): - def __init__(self, data): - self.uuid = data.get("uuid", KeyError) - - super().__init__("uuid", str, self.uuid) - - self.validation = self.vm_uuid_validation - - def vm_uuid_validation(self): - r = shared.etcd_client.get( - os.path.join(settings["etcd"]["vm_prefix"], self.uuid) - ) - if not r: - self.add_error( - "VM with uuid {} does not exists".format(self.uuid) - ) diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 1040e97..802567f 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -14,7 +14,4 @@ data = { 'attributes': {'list': [], 'key': [], 'pool': 'images'}, } -shared.etcd_client.put( - os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), - json.dumps(data), -) +shared.etcd_client.put(os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data)) diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index 0805280..c0205c8 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -1,6 +1,4 @@ import binascii -import ipaddress -import random import logging import requests @@ -15,25 +13,19 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": settings["otp"]["auth_name"], - "auth_token": TOTP(settings["otp"]["auth_seed"]).now(), - "auth_realm": settings["otp"]["auth_realm"], - "name": name, - "realm": realm, - "token": token, + 'auth_name': settings['otp']['auth_name'], + 'auth_token': TOTP(settings['otp']['auth_seed']).now(), + 'auth_realm': settings['otp']['auth_realm'], + 'name': name, + 'realm': realm, + 'token': token, } - except binascii.Error as err: - logger.error( - "Cannot compute OTP for seed: {}".format( - settings["otp"]["auth_seed"] - ) - ) + except binascii.Error: + logger.error('Cannot compute OTP for seed: {}'.format(settings['otp']['auth_seed'])) return 400 - - response = requests.post( - settings["otp"]["verification_controller_url"], json=data - ) - return response.status_code + else: + response = requests.post(settings['otp']['verification_controller_url'], json=data) + return response.status_code def resolve_vm_name(name, owner): @@ -43,29 +35,24 @@ def resolve_vm_name(name, owner): Output: uuid of vm if found otherwise None """ result = next( - filter( - lambda vm: vm.value["owner"] == owner - and vm.value["name"] == name, - shared.vm_pool.vms, - ), - None, + filter(lambda vm: vm.value['owner'] == owner and vm.value['name'] == name, shared.vm_pool.vms), + None ) if result: - return result.key.split("/")[-1] + return result.key.split('/')[-1] return None -def resolve_image_name(name, etcd_client): +def resolve_image_name(name): """Return image uuid given its name and its store * If the provided name is not in correct format i.e {store_name}:{image_name} return ValueError * If no such image found then return KeyError - """ - seperator = ":" + seperator = ':' # Ensure, user/program passed valid name that is of type string try: @@ -73,78 +60,35 @@ def resolve_image_name(name, etcd_client): """ Examples, where it would work and where it would raise exception - "images:alpine" --> ["images", "alpine"] + 'images:alpine' --> ['images', 'alpine'] - "images" --> ["images"] it would raise Exception as non enough value to unpack + 'images' --> ['images'] it would raise Exception as non enough value to unpack - "images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception + 'images:alpine:meow' --> ['images', 'alpine', 'meow'] it would raise Exception as too many values to unpack """ store_name, image_name = store_name_and_image_name except Exception: - raise ValueError( - "Image name not in correct format i.e {store_name}:{image_name}" - ) + raise ValueError('Image name not in correct format i.e {store_name}:{image_name}') - images = etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True - ) + images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True) # Try to find image with name == image_name and store_name == store_name try: image = next( filter( - lambda im: im.value["name"] == image_name - and im.value["store_name"] == store_name, + lambda im: im.value['name'] == image_name + and im.value['store_name'] == store_name, images, ) ) except StopIteration: - raise KeyError("No image with name {} found.".format(name)) + raise KeyError('No image with name {} found.'.format(name)) else: - image_uuid = image.key.split("/")[-1] + image_uuid = image.key.split('/')[-1] return image_uuid -def random_bytes(num=6): - return [random.randrange(256) for _ in range(num)] - - -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): - mac = random_bytes() - if oui: - if type(oui) == str: - oui = [int(chunk) for chunk in oui.split(separator)] - mac = oui + random_bytes(num=6 - len(oui)) - else: - if multicast: - mac[0] |= 1 # set bit 0 - else: - mac[0] &= ~1 # clear bit 0 - if uaa: - mac[0] &= ~(1 << 1) # clear bit 1 - else: - mac[0] |= 1 << 1 # set bit 1 - return separator.join(byte_fmt % b for b in mac) - - -def mac2ipv6(mac, prefix): - # only accept MACs separated by a colon - parts = mac.split(":") - - # modify parts to match IPv6 value - parts.insert(3, "ff") - parts.insert(4, "fe") - parts[0] = "%x" % (int(parts[0], 16) ^ 2) - - # format output - ipv6_parts = [str(0)] * 4 - for i in range(0, len(parts), 2): - ipv6_parts.append("".join(parts[i : i + 2])) - - lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) - prefix = ipaddress.IPv6Address(prefix) - return str(prefix + int(lower_part)) +def make_return_message(err, status_code=200): + return {'message': str(err)}, status_code diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 2d8d035..15f3e2a 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -14,10 +14,13 @@ from uncloud.common.shared import shared from uncloud.common import counters from uncloud.common.vm import VMStatus +from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.settings import settings -from . import schemas -from .helper import generate_mac, mac2ipv6 +from uncloud.api import schemas +from uncloud.api.schemas import ValidationException +from uncloud.common.network import generate_mac, mac2ipv6 +from uncloud.api.helper import make_return_message from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -33,6 +36,7 @@ arg_parser.add_argument('--port', '-p') @app.errorhandler(Exception) def handle_exception(e): app.logger.error(e) + # pass through HTTP errors if isinstance(e, HTTPException): return e @@ -42,13 +46,15 @@ def handle_exception(e): class CreateVM(Resource): - """API Request to Handle Creation of VM""" - @staticmethod def post(): data = request.json - validator = schemas.CreateVMSchema(data) - if validator.is_valid(): + try: + validator = schemas.CreateVMSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: vm_uuid = uuid4().hex vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) specs = { @@ -57,24 +63,22 @@ class CreateVM(Resource): 'os-ssd': validator.specs['os-ssd'], 'hdd': validator.specs['hdd'], } - macs = [generate_mac() for _ in range(len(data['network']))] + macs = [generate_mac() for _ in range(len(validator.network.value))] tap_ids = [ - counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['tap_counter'] - ) - for _ in range(len(data['network'])) + counters.increment_etcd_counter(settings['etcd']['tap_counter']) + for _ in range(len(validator.network.value)) ] vm_entry = { - 'name': data['vm_name'], - 'owner': data['name'], - 'owner_realm': data['realm'], + 'name': validator.vm_name.value, + 'owner': validator.name.value, + 'owner_realm': validator.realm.value, 'specs': specs, 'hostname': '', 'status': VMStatus.stopped, 'image_uuid': validator.image_uuid, 'log': [], 'vnc_socket': '', - 'network': list(zip(data['network'], macs, tap_ids)), + 'network': list(zip(validator.network.value, macs, tap_ids)), 'metadata': {'ssh-keys': []}, 'in_migration': False, } @@ -87,86 +91,76 @@ class CreateVM(Resource): request_prefix=settings['etcd']['request_prefix'], ) shared.request_pool.put(r) - - return {'message': 'VM Creation Queued'}, 200 - return validator.get_errors(), 400 + return make_return_message('VM Creation Queued') class VmStatus(Resource): @staticmethod def post(): data = request.json - validator = schemas.VMStatusSchema(data) - if validator.is_valid(): - vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) - ) + try: + validator = schemas.VMStatusSchema(data) + validator.is_valid() + except (ValidationException, Exception) as err: + return make_return_message(err, 400) + else: + vm = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], data['uuid'])) vm_value = vm.value.copy() vm_value['ip'] = [] for network_mac_and_tap in vm.network: network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( - join_path( - settings['etcd']['network_prefix'], - data['name'], - network_name, - ), + join_path(settings['etcd']['network_prefix'], data['name'], network_name), value_in_json=True, ) - ipv6_addr = ( - network.value.get('ipv6').split('::')[0] + '::' - ) + ipv6_addr = (network.value.get('ipv6').split('::')[0] + '::') vm_value['ip'].append(mac2ipv6(mac, ipv6_addr)) vm.value = vm_value - return vm.value - else: - return validator.get_errors(), 400 + return vm.value, 200 class CreateImage(Resource): @staticmethod def post(): data = request.json - validator = schemas.CreateImageSchema(data) - if validator.is_valid(): - file_entry = shared.etcd_client.get( - join_path(settings['etcd']['file_prefix'], data['uuid']) - ) - file_entry_value = json.loads(file_entry.value) + try: + validator = schemas.CreateImageSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + try: + file_entry = shared.etcd_client.get( + join_path(settings['etcd']['file_prefix'], data['uuid']), value_in_json=True + ) + except KeyError: + # TODO: Add some message + pass + else: + image_entry_json = { + 'status': 'TO_BE_CREATED', + 'owner': file_entry.value['owner'], + 'filename': file_entry.value['filename'], + 'name': validator.name, + 'store_name': validator.image_store, + 'visibility': 'public', + } + shared.etcd_client.put( + join_path(settings['etcd']['image_prefix'], data['uuid']), + json.dumps(image_entry_json), + ) - image_entry_json = { - 'status': 'TO_BE_CREATED', - 'owner': file_entry_value['owner'], - 'filename': file_entry_value['filename'], - 'name': data['name'], - 'store_name': data['image_store'], - 'visibility': 'public', - } - shared.etcd_client.put( - join_path( - settings['etcd']['image_prefix'], data['uuid'] - ), - json.dumps(image_entry_json), - ) - - return {'message': 'Image queued for creation.'} - return validator.get_errors(), 400 + return {'message': 'Image queued for creation.'}, 200 class ListPublicImages(Resource): @staticmethod def get(): - images = shared.etcd_client.get_prefix( - settings['etcd']['image_prefix'], value_in_json=True - ) + images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True) r = {'images': []} for image in images: - image_key = '{}:{}'.format( - image.value['store_name'], image.value['name'] - ) - r['images'].append( - {'name': image_key, 'status': image.value['status']} - ) + image_key = '{}:{}'.format(image.value['store_name'], image.value['name']) + r['images'].append({'name': image_key, 'status': image.value['status']}) return r, 200 @@ -174,35 +168,30 @@ class VMAction(Resource): @staticmethod def post(): data = request.json - validator = schemas.VmActionSchema(data) - - if validator.is_valid(): - vm_entry = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) - ) - action = data['action'] + try: + validator = schemas.VmActionSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + vm_entry = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], data['uuid'])) + action = validator.action.value if action == 'start': action = 'schedule' if action == 'delete' and vm_entry.hostname == '': - if shared.storage_handler.is_vm_image_exists( - vm_entry.uuid - ): - r_status = shared.storage_handler.delete_vm_image( - vm_entry.uuid - ) + if shared.storage_handler.is_vm_image_exists(vm_entry.uuid): + r_status = shared.storage_handler.delete_vm_image(vm_entry.uuid) if r_status: shared.etcd_client.client.delete(vm_entry.key) - return {'message': 'VM successfully deleted'} + return make_return_message('VM successfully deleted') else: - logger.error( - 'Some Error Occurred while deleting VM' - ) - return {'message': 'VM deletion unsuccessfull'} + logger.error('Some Error Occurred while deleting VM') + return make_return_message('VM deletion unsuccessfull') else: shared.etcd_client.client.delete(vm_entry.key) - return {'message': 'VM successfully deleted'} + return make_return_message('VM successfully deleted') r = RequestEntry.from_scratch( type='{}VM'.format(action.title()), @@ -211,22 +200,20 @@ class VMAction(Resource): request_prefix=settings['etcd']['request_prefix'], ) shared.request_pool.put(r) - return ( - {'message': 'VM {} Queued'.format(action.title())}, - 200, - ) - else: - return validator.get_errors(), 400 + return make_return_message('VM {} Queued'.format(action.title())) class VMMigration(Resource): @staticmethod def post(): data = request.json - validator = schemas.VmMigrationSchema(data) - - if validator.is_valid(): - vm = shared.vm_pool.get(data['uuid']) + try: + validator = schemas.VmMigrationSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err), 400 + else: + vm = shared.vm_pool.get(validator.uuid.value) r = RequestEntry.from_scratch( type=RequestType.InitVMMigration, uuid=vm.uuid, @@ -238,28 +225,22 @@ class VMMigration(Resource): ) shared.request_pool.put(r) - return ( - {'message': 'VM Migration Initialization Queued'}, - 200, - ) - else: - return validator.get_errors(), 400 + return make_return_message('VM Migration Initialization Queued') class ListUserVM(Resource): @staticmethod def post(): data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - vms = shared.etcd_client.get_prefix( - settings['etcd']['vm_prefix'], value_in_json=True - ) + try: + validator = schemas.OTPSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + vms = shared.etcd_client.get_prefix(settings['etcd']['vm_prefix'], value_in_json=True) return_vms = [] - user_vms = filter( - lambda v: v.value['owner'] == data['name'], vms - ) + user_vms = filter(lambda v: v.value['owner'] == validator.name.value, vms) for vm in user_vms: return_vms.append( { @@ -271,24 +252,20 @@ class ListUserVM(Resource): 'vnc_socket': vm.value.get('vnc_socket', None), } ) - if return_vms: - return {'message': return_vms}, 200 - return {'message': 'No VM found'}, 404 - - else: - return validator.get_errors(), 400 + return make_return_message(return_vms) class ListUserFiles(Resource): @staticmethod def post(): data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - files = shared.etcd_client.get_prefix( - settings['etcd']['file_prefix'], value_in_json=True - ) + try: + validator = schemas.OTPSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + files = shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) return_files = [] user_files = [f for f in files if f.value['owner'] == data['name']] for file in user_files: @@ -300,33 +277,28 @@ class ListUserFiles(Resource): file.pop('owner', None) return_files.append(file) - return {'message': return_files}, 200 - else: - return validator.get_errors(), 400 + return make_return_message(return_files) class CreateHost(Resource): @staticmethod def post(): data = request.json - validator = schemas.CreateHostSchema(data) - if validator.is_valid(): - host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex - ) + try: + validator = schemas.CreateHostSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + host_key = join_path(settings['etcd']['host_prefix'], uuid4().hex) host_entry = { - 'specs': data['specs'], - 'hostname': data['hostname'], - 'status': 'DEAD', + 'specs': validator.specs.value, + 'hostname': validator.hostname.value, + 'status': HostStatus.dead, 'last_heartbeat': '', } - shared.etcd_client.put( - host_key, host_entry, value_in_json=True - ) - - return {'message': 'Host Created'}, 200 - - return validator.get_errors(), 400 + shared.etcd_client.put(host_key, host_entry, value_in_json=True) + return make_return_message('Host Created.') class ListHost(Resource): @@ -348,196 +320,138 @@ class GetSSHKeys(Resource): @staticmethod def post(): data = request.json - validator = schemas.GetSSHSchema(data) - if validator.is_valid(): + try: + validator = schemas.GetSSHSchema(data) + except ValidationException as err: + return make_return_message(err, 400) + else: + etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm.value, + validator.name.value, 'key') if not validator.key_name.value: - - # {user_prefix}/{realm}/{name}/key/ - etcd_key = join_path( - settings['etcd']['user_prefix'], - data['realm'], - data['name'], - 'key', - ) - etcd_entry = shared.etcd_client.get_prefix( - etcd_key, value_in_json=True - ) - + etcd_entry = shared.etcd_client.get_prefix(etcd_key, value_in_json=True) keys = { key.key.split('/')[-1]: key.value for key in etcd_entry } return {'keys': keys} else: - - # {user_prefix}/{realm}/{name}/key/{key_name} - etcd_key = join_path( - settings['etcd']['user_prefix'], - data['realm'], - data['name'], - 'key', - data['key_name'], - ) - etcd_entry = shared.etcd_client.get( - etcd_key, value_in_json=True - ) - - if etcd_entry: - return { - 'keys': { - etcd_entry.key.split('/')[ - -1 - ]: etcd_entry.value - } - } + etcd_key = join_path(validator.key_name.value) + try: + etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) + except KeyError: + return make_return_message('No such key found.', 400) else: - return {'keys': {}} - else: - return validator.get_errors(), 400 + return { + 'keys': {etcd_entry.key.split('/')[-1]: etcd_entry.value} + } class AddSSHKey(Resource): @staticmethod def post(): data = request.json - validator = schemas.AddSSHSchema(data) - if validator.is_valid(): - + try: + validator = schemas.AddSSHSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], - data['realm'], - data['name'], - 'key', - data['key_name'], + settings['etcd']['user_prefix'], validator.realm.value, + validator.name.value, 'key', validator.key_name.value ) - etcd_entry = shared.etcd_client.get( - etcd_key, value_in_json=True - ) - if etcd_entry: - return { - 'message': 'Key with name "{}" already exists'.format( - data['key_name'] - ) - } - else: + try: + shared.etcd_client.get(etcd_key, value_in_json=True) + except KeyError: # Key Not Found. It implies user' haven't added any key yet. - shared.etcd_client.put( - etcd_key, data['key'], value_in_json=True - ) - return {'message': 'Key added successfully'} - else: - return validator.get_errors(), 400 + shared.etcd_client.put(etcd_key, validator.key.value, value_in_json=True) + return make_return_message('Key added successfully') + else: + return make_return_message('Key "{}" already exists'.format(validator.key_name.value)) class RemoveSSHKey(Resource): @staticmethod def post(): data = request.json - validator = schemas.RemoveSSHSchema(data) - if validator.is_valid(): - + try: + validator = schemas.RemoveSSHSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: # {user_prefix}/{realm}/{name}/key/{key_name} - etcd_key = join_path( - settings['etcd']['user_prefix'], - data['realm'], - data['name'], - 'key', - data['key_name'], - ) - etcd_entry = shared.etcd_client.get( - etcd_key, value_in_json=True - ) + etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm.value, + validator.name.value, 'key', validator.key_name.value) + try: + etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) + except KeyError: + return make_return_message('No Key "{}" exists.'.format(validator.key_name.value)) if etcd_entry: shared.etcd_client.client.delete(etcd_key) return {'message': 'Key successfully removed.'} - else: - return { - 'message': 'No Key with name "{}" Exists at all.'.format( - data['key_name'] - ) - } - else: - return validator.get_errors(), 400 class CreateNetwork(Resource): @staticmethod def post(): data = request.json - validator = schemas.CreateNetwork(data) - - if validator.is_valid(): - + try: + validator = schemas.CreateNetwork(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: network_entry = { - 'id': counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['vxlan_counter'] - ), - 'type': data['type'], + 'id': counters.increment_etcd_counter(settings['etcd']['vxlan_counter']), + 'type': validator.type.value, } if validator.user.value: try: - nb = pynetbox.api( - url=settings['netbox']['url'], - token=settings['netbox']['token'], - ) - nb_prefix = nb.ipam.prefixes.get( - prefix=settings['network']['prefix'] - ) + nb = pynetbox.api(url=settings['netbox']['url'], token=settings['netbox']['token']) + nb_prefix = nb.ipam.prefixes.get(prefix=settings['network']['prefix']) prefix = nb_prefix.available_prefixes.create( data={ - 'prefix_length': int( - settings['network']['prefix_length'] - ), + 'prefix_length': int(settings['network']['prefix_length']), 'description': '{}\'s network "{}"'.format( - data['name'], data['network_name'] + validator.name.value, + validator.network_name.value ), 'is_pool': True, } ) except Exception as err: app.logger.error(err) - return { - 'message': 'Error occured while creating network.' - } + return make_return_message('Error occured while creating network.', 400) else: network_entry['ipv6'] = prefix['prefix'] else: network_entry['ipv6'] = 'fd00::/64' - network_key = join_path( - settings['etcd']['network_prefix'], - data['name'], - data['network_name'], - ) - shared.etcd_client.put( - network_key, network_entry, value_in_json=True - ) - return {'message': 'Network successfully added.'} - else: - return validator.get_errors(), 400 + network_key = join_path(settings['etcd']['network_prefix'], validator.name.value, + validator.network_name.value) + shared.etcd_client.put(network_key, network_entry, value_in_json=True) + return make_return_message('Network successfully added.') class ListUserNetwork(Resource): @staticmethod def post(): data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - prefix = join_path( - settings['etcd']['network_prefix'], data['name'] - ) - networks = shared.etcd_client.get_prefix( - prefix, value_in_json=True - ) + try: + validator = schemas.OTPSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + prefix = join_path(settings['etcd']['network_prefix'], data['name']) + networks = shared.etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] for net in networks: net.value['name'] = net.key.split('/')[-1] user_networks.append(net.value) return {'networks': user_networks}, 200 - else: - return validator.get_errors(), 400 api.add_resource(CreateVM, '/vm/create') @@ -565,37 +479,7 @@ api.add_resource(CreateNetwork, '/network/create') def main(debug=False, port=None): try: - image_stores = list( - shared.etcd_client.get_prefix( - settings['etcd']['image_store_prefix'], value_in_json=True - ) - ) - except KeyError: - image_stores = False - - # Do not inject default values that might be very wrong - # fail when required, not before - # - # if not image_stores: - # data = { - # 'is_public': True, - # 'type': 'ceph', - # 'name': 'images', - # 'description': 'first ever public image-store', - # 'attributes': {'list': [], 'key': [], 'pool': 'images'}, - # } - - # shared.etcd_client.put( - # join_path( - # settings['etcd']['image_store_prefix'], uuid4().hex - # ), - # json.dumps(data), - # ) - - try: - app.run(host='::', - port=port, - debug=debug) + app.run(host='::', port=port, debug=debug) except OSError as e: raise UncloudException('Failed to start Flask: {}'.format(e)) diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index e4de9a8..ffa33f6 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -1,19 +1,3 @@ -""" -This module contain classes thats validates and intercept/modify -data coming from uncloud-cli (user) - -It was primarily developed as an alternative to argument parser -of Flask_Restful which is going to be deprecated. I also tried -marshmallow for that purpose but it was an overkill (because it -do validation + serialization + deserialization) and little -inflexible for our purpose. -""" - -# TODO: Fix error message when user's mentioned VM (referred by name) -# does not exists. -# -# Currently, it says uuid is a required field. - import json import os @@ -23,19 +7,54 @@ from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus from uncloud.common.shared import shared from uncloud.common.settings import settings -from . import helper, logger -from .common_fields import Field, VmUUIDField -from .helper import check_otp, resolve_vm_name +from uncloud.api import helper +from uncloud.api.helper import check_otp, resolve_vm_name + + +class ValidationException(Exception): + """Validation Error""" + + +class Field: + def __init__(self, _name, _type, _value=None, validators=None): + if validators is None: + validators = [] + + assert isinstance(validators, list) + + self.name = _name + self.value = _value + self.type = _type + self.validators = validators + + def is_valid(self): + if not isinstance(self.value, self.type): + raise ValidationException("Incorrect Type for '{}' field".format(self.name)) + else: + for validator in self.validators: + validator() + + def __repr__(self): + return self.name + + +class VmUUIDField(Field): + def __init__(self, data): + self.uuid = data.get('uuid', KeyError) + + super().__init__('uuid', str, self.uuid, validators=[self.vm_uuid_validation]) + + def vm_uuid_validation(self): + try: + shared.etcd_client.get(os.path.join(settings['etcd']['vm_prefix'], self.uuid)) + except KeyError: + raise ValidationException('VM with uuid {} does not exists'.format(self.uuid)) class BaseSchema: - def __init__(self, data, fields=None): - _ = data # suppress linter warning - self.__errors = [] - if fields is None: - self.fields = [] - else: - self.fields = fields + def __init__(self, data): + print(data) + self.fields = [getattr(self, field) for field in dir(self) if isinstance(getattr(self, field), Field)] def validation(self): # custom validation is optional @@ -44,517 +63,338 @@ class BaseSchema: def is_valid(self): for field in self.fields: field.is_valid() - self.add_field_errors(field) for parent in self.__class__.__bases__: - try: - parent.validation(self) - except AttributeError: - pass - if not self.__errors: - self.validation() + parent.validation(self) - if self.__errors: - return False - return True + self.validation() - def get_errors(self): - return {"message": self.__errors} - def add_field_errors(self, field: Field): - self.__errors += field.get_errors() - - def add_error(self, error): - self.__errors.append(error) +def get(dictionary: dict, key: str, return_default=False, default=None): + if dictionary is None: + raise ValidationException('No data provided at all.') + try: + value = dictionary[key] + except KeyError: + if return_default: + return default + raise ValidationException("Missing data for '{}' field.".format(key)) + else: + return value class OTPSchema(BaseSchema): - def __init__(self, data: dict, fields=None): - self.name = Field("name", str, data.get("name", KeyError)) - self.realm = Field("realm", str, data.get("realm", KeyError)) - self.token = Field("token", str, data.get("token", KeyError)) - - _fields = [self.name, self.realm, self.token] - if fields: - _fields += fields - super().__init__(data=data, fields=_fields) + def __init__(self, data: dict): + self.name = Field('name', str, get(data, 'name')) + self.realm = Field('realm', str, get(data, 'realm')) + self.token = Field('token', str, get(data, 'token')) + super().__init__(data=data) def validation(self): - if ( - check_otp( - self.name.value, self.realm.value, self.token.value - ) - != 200 - ): - self.add_error("Wrong Credentials") - - -########################## Image Operations ############################################### + if check_otp(self.name.value, self.realm.value, self.token.value) != 200: + raise ValidationException('Wrong Credentials') class CreateImageSchema(BaseSchema): def __init__(self, data): - # Fields - self.uuid = Field("uuid", str, data.get("uuid", KeyError)) - self.name = Field("name", str, data.get("name", KeyError)) - self.image_store = Field( - "image_store", str, data.get("image_store", KeyError) - ) + self.uuid = Field('uuid', str, get(data, 'uuid'), validators=[self.file_uuid_validation]) + self.name = Field('name', str, get(data, 'name')) + self.image_store = Field('image_store', str, get(data, 'image_store'), + validators=[self.image_store_name_validation]) - # Validations - self.uuid.validation = self.file_uuid_validation - self.image_store.validation = self.image_store_name_validation - - # All Fields - fields = [self.uuid, self.name, self.image_store] - super().__init__(data, fields) + super().__init__(data) def file_uuid_validation(self): - file_entry = shared.etcd_client.get( - os.path.join( - settings["etcd"]["file_prefix"], self.uuid.value - ) - ) - if file_entry is None: - self.add_error( - "Image File with uuid '{}' Not Found".format( - self.uuid.value - ) - ) + try: + shared.etcd_client.get(os.path.join(settings['etcd']['file_prefix'], self.uuid.value)) + except KeyError: + raise ValidationException("Image File with uuid '{}' Not Found".format(self.uuid.value)) def image_store_name_validation(self): - image_stores = list( - shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"] - ) - ) - - image_store = next( - filter( - lambda s: json.loads(s.value)["name"] - == self.image_store.value, - image_stores, - ), - None, - ) - if not image_store: - self.add_error( - "Store '{}' does not exists".format( - self.image_store.value - ) - ) - - -# Host Operations + image_stores = list(shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix'])) + try: + next(filter(lambda s: json.loads(s.value)['name'] == self.image_store.value, image_stores)) + except StopIteration: + raise ValidationException("Store '{}' does not exists".format(self.image_store.value)) class CreateHostSchema(OTPSchema): def __init__(self, data): - # Fields - self.specs = Field("specs", dict, data.get("specs", KeyError)) - self.hostname = Field( - "hostname", str, data.get("hostname", KeyError) - ) + self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation]) + self.hostname = Field('hostname', str, get(data, 'hostname')) - # Validation - self.specs.validation = self.specs_validation - - fields = [self.hostname, self.specs] - - super().__init__(data=data, fields=fields) + super().__init__(data=data) def specs_validation(self): - ALLOWED_BASE = 10 + allowed_base = 10 - _cpu = self.specs.value.get("cpu", KeyError) - _ram = self.specs.value.get("ram", KeyError) - _os_ssd = self.specs.value.get("os-ssd", KeyError) - _hdd = self.specs.value.get("hdd", KeyError) + _cpu = self.specs.value.get('cpu', KeyError) + _ram = self.specs.value.get('ram', KeyError) + _os_ssd = self.specs.value.get('os-ssd', KeyError) + _hdd = self.specs.value.get('hdd', KeyError) - if KeyError in [_cpu, _ram, _os_ssd, _hdd]: - self.add_error( - "You must specify CPU, RAM and OS-SSD in your specs" - ) - return None + if KeyError in [_cpu, _ram, _os_ssd]: + raise ValidationException('You must specify CPU, RAM and OS-SSD in your specs') try: parsed_ram = bitmath.parse_string_unsafe(_ram) parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) - if parsed_ram.base != ALLOWED_BASE: - self.add_error( - "Your specified RAM is not in correct units" - ) - if parsed_os_ssd.base != ALLOWED_BASE: - self.add_error( - "Your specified OS-SSD is not in correct units" - ) + if parsed_ram.base != allowed_base: + raise ValidationException('Your specified RAM is not in correct units') + + if parsed_os_ssd.base != allowed_base: + raise ValidationException('Your specified OS-SSD is not in correct units') if _cpu < 1: - self.add_error("CPU must be atleast 1") + raise ValidationException('CPU must be atleast 1') if parsed_ram < bitmath.GB(1): - self.add_error("RAM must be atleast 1 GB") + raise ValidationException('RAM must be atleast 1 GB') if parsed_os_ssd < bitmath.GB(10): - self.add_error("OS-SSD must be atleast 10 GB") + raise ValidationException('OS-SSD must be atleast 10 GB') parsed_hdd = [] for hdd in _hdd: _parsed_hdd = bitmath.parse_string_unsafe(hdd) - if _parsed_hdd.base != ALLOWED_BASE: - self.add_error( - "Your specified HDD is not in correct units" - ) - break + if _parsed_hdd.base != allowed_base: + raise ValidationException('Your specified HDD is not in correct units') else: parsed_hdd.append(str(_parsed_hdd)) except ValueError: - # TODO: Find some good error message - self.add_error("Specs are not correct.") + raise ValidationException('Specs are not correct.') else: - if self.get_errors(): - self.specs = { - "cpu": _cpu, - "ram": str(parsed_ram), - "os-ssd": str(parsed_os_ssd), - "hdd": parsed_hdd, - } + self.specs = { + 'cpu': _cpu, + 'ram': str(parsed_ram), + 'os-ssd': str(parsed_os_ssd), + 'hdd': parsed_hdd, + } def validation(self): - if self.realm.value != "ungleich-admin": - self.add_error( - "Invalid Credentials/Insufficient Permission" - ) - - -# VM Operations + if self.realm.value != 'ungleich-admin': + raise ValidationException('Invalid Credentials/Insufficient Permission') class CreateVMSchema(OTPSchema): def __init__(self, data): - # Fields - self.specs = Field("specs", dict, data.get("specs", KeyError)) - self.vm_name = Field( - "vm_name", str, data.get("vm_name", KeyError) - ) - self.image = Field("image", str, data.get("image", KeyError)) - self.network = Field( - "network", list, data.get("network", KeyError) - ) + self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation]) + self.vm_name = Field('vm_name', str, get(data, 'vm_name'), validators=[self.vm_name_validation]) + self.image = Field('image', str, get(data, 'image'), validators=[self.image_validation]) + self.network = Field('network', list, get(data, 'network', return_default=True, default=[]), + validators=[self.network_validation]) + self.image_uuid = None - # Validation - self.image.validation = self.image_validation - self.vm_name.validation = self.vm_name_validation - self.specs.validation = self.specs_validation - self.network.validation = self.network_validation - - fields = [self.vm_name, self.image, self.specs, self.network] - - super().__init__(data=data, fields=fields) + super().__init__(data=data) def image_validation(self): try: - image_uuid = helper.resolve_image_name( - self.image.value, shared.etcd_client - ) + image_uuid = helper.resolve_image_name(self.image.value) except Exception as e: - logger.exception( - "Cannot resolve image name = %s", self.image.value - ) - self.add_error(str(e)) + raise ValidationException('No image of name \'{}\' found'.format(self.image.value)) else: self.image_uuid = image_uuid def vm_name_validation(self): - if resolve_vm_name( - name=self.vm_name.value, owner=self.name.value - ): - self.add_error( - 'VM with same name "{}" already exists'.format( - self.vm_name.value - ) - ) + if resolve_vm_name(name=self.vm_name.value, owner=self.name.value): + raise ValidationException("VM with same name '{}' already exists".format(self.vm_name.value)) def network_validation(self): _network = self.network.value if _network: for net in _network: - network = shared.etcd_client.get( - os.path.join( - settings["etcd"]["network_prefix"], - self.name.value, - net, - ), - value_in_json=True, - ) - if not network: - self.add_error( - "Network with name {} does not exists".format( - net - ) + try: + shared.etcd_client.get( + os.path.join(settings['etcd']['network_prefix'], self.name.value, net), + value_in_json=True ) + except KeyError: + raise ValidationException('Network with name {} does not exists'.format(net)) def specs_validation(self): ALLOWED_BASE = 10 - _cpu = self.specs.value.get("cpu", KeyError) - _ram = self.specs.value.get("ram", KeyError) - _os_ssd = self.specs.value.get("os-ssd", KeyError) - _hdd = self.specs.value.get("hdd", KeyError) - - if KeyError in [_cpu, _ram, _os_ssd, _hdd]: - self.add_error( - "You must specify CPU, RAM and OS-SSD in your specs" - ) - return None try: - parsed_ram = bitmath.parse_string_unsafe(_ram) - parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) - - if parsed_ram.base != ALLOWED_BASE: - self.add_error( - "Your specified RAM is not in correct units" - ) - if parsed_os_ssd.base != ALLOWED_BASE: - self.add_error( - "Your specified OS-SSD is not in correct units" - ) - - if int(_cpu) < 1: - self.add_error("CPU must be atleast 1") - - if parsed_ram < bitmath.GB(1): - self.add_error("RAM must be atleast 1 GB") - - if parsed_os_ssd < bitmath.GB(1): - self.add_error("OS-SSD must be atleast 1 GB") - - parsed_hdd = [] - for hdd in _hdd: - _parsed_hdd = bitmath.parse_string_unsafe(hdd) - if _parsed_hdd.base != ALLOWED_BASE: - self.add_error( - "Your specified HDD is not in correct units" - ) - break - else: - parsed_hdd.append(str(_parsed_hdd)) - - except ValueError: - # TODO: Find some good error message - self.add_error("Specs are not correct.") + _cpu = get(self.specs.value, 'cpu') + _ram = get(self.specs.value, 'ram') + _os_ssd = get(self.specs.value, 'os-ssd') + _hdd = get(self.specs.value, 'hdd', return_default=True, default=[]) + except (KeyError, Exception): + raise ValidationException('You must specify CPU, RAM and OS-SSD in your specs') else: - if self.get_errors(): + try: + parsed_ram = bitmath.parse_string_unsafe(_ram) + parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) + + if parsed_ram.base != ALLOWED_BASE: + raise ValidationException('Your specified RAM is not in correct units') + + if parsed_os_ssd.base != ALLOWED_BASE: + raise ValidationException('Your specified OS-SSD is not in correct units') + + if int(_cpu) < 1: + raise ValidationException('CPU must be atleast 1') + + if parsed_ram < bitmath.GB(1): + raise ValidationException('RAM must be atleast 1 GB') + + if parsed_os_ssd < bitmath.GB(1): + raise ValidationException('OS-SSD must be atleast 1 GB') + + parsed_hdd = [] + for hdd in _hdd: + _parsed_hdd = bitmath.parse_string_unsafe(hdd) + if _parsed_hdd.base != ALLOWED_BASE: + raise ValidationException('Your specified HDD is not in correct units') + else: + parsed_hdd.append(str(_parsed_hdd)) + + except ValueError: + raise ValidationException('Specs are not correct.') + else: self.specs = { - "cpu": _cpu, - "ram": str(parsed_ram), - "os-ssd": str(parsed_os_ssd), - "hdd": parsed_hdd, + 'cpu': _cpu, + 'ram': str(parsed_ram), + 'os-ssd': str(parsed_os_ssd), + 'hdd': parsed_hdd, } class VMStatusSchema(OTPSchema): def __init__(self, data): - data["uuid"] = ( + data['uuid'] = ( resolve_vm_name( - name=data.get("vm_name", None), + name=get(data, 'vm_name', return_default=True), owner=( - data.get("in_support_of", None) - or data.get("name", None) - ), + get(data, 'in_support_of', return_default=True) or + get(data, 'name', return_default=True) + ) ) or KeyError ) self.uuid = VmUUIDField(data) - fields = [self.uuid] - - super().__init__(data, fields) + super().__init__(data) def validation(self): vm = shared.vm_pool.get(self.uuid.value) - if not ( - vm.value["owner"] == self.name.value - or self.realm.value == "ungleich-admin" - ): - self.add_error("Invalid User") + if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'): + raise ValidationException('Invalid User') class VmActionSchema(OTPSchema): def __init__(self, data): - data["uuid"] = ( + data['uuid'] = ( resolve_vm_name( - name=data.get("vm_name", None), + name=get(data, 'vm_name', return_default=True), owner=( - data.get("in_support_of", None) - or data.get("name", None) - ), + get(data, 'in_support_of', return_default=True) or + get(data, 'name', return_default=True) + ) ) or KeyError ) self.uuid = VmUUIDField(data) - self.action = Field("action", str, data.get("action", KeyError)) + self.action = Field('action', str, get(data, 'action'), validators=[self.action_validation]) - self.action.validation = self.action_validation - - _fields = [self.uuid, self.action] - - super().__init__(data=data, fields=_fields) + super().__init__(data=data) def action_validation(self): - allowed_actions = ["start", "stop", "delete"] + allowed_actions = ['start', 'stop', 'delete'] if self.action.value not in allowed_actions: - self.add_error( - "Invalid Action. Allowed Actions are {}".format( - allowed_actions - ) - ) + raise ValidationException('Invalid Action. Allowed Actions are {}'.format(allowed_actions)) def validation(self): vm = shared.vm_pool.get(self.uuid.value) - if not ( - vm.value["owner"] == self.name.value - or self.realm.value == "ungleich-admin" - ): - self.add_error("Invalid User") + if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'): + raise ValidationException('Invalid User.') - if ( - self.action.value == "start" - and vm.status == VMStatus.running - and vm.hostname != "" - ): - self.add_error("VM Already Running") + if self.action.value == 'start' and vm.status == VMStatus.running and vm.hostname != '': + raise ValidationException('VM Already Running') - if self.action.value == "stop": + if self.action.value == 'stop': if vm.status == VMStatus.stopped: - self.add_error("VM Already Stopped") + raise ValidationException('VM Already Stopped') elif vm.status != VMStatus.running: - self.add_error("Cannot stop non-running VM") + raise ValidationException('Cannot stop non-running VM') class VmMigrationSchema(OTPSchema): def __init__(self, data): - data["uuid"] = ( + data['uuid'] = ( resolve_vm_name( - name=data.get("vm_name", None), + name=get(data, 'vm_name', return_default=True), owner=( - data.get("in_support_of", None) - or data.get("name", None) - ), - ) - or KeyError + get(data, 'in_support_of', return_default=True) or + get(data, 'name', return_default=True) + ) + ) or KeyError ) self.uuid = VmUUIDField(data) - self.destination = Field( - "destination", str, data.get("destination", KeyError) - ) + self.destination = Field('destination', str, get(data, 'destination'), + validators=[self.destination_validation]) - self.destination.validation = self.destination_validation - - fields = [self.destination] - super().__init__(data=data, fields=fields) + super().__init__(data=data) def destination_validation(self): hostname = self.destination.value - host = next( - filter( - lambda h: h.hostname == hostname, shared.host_pool.hosts - ), - None, - ) + host = next(filter(lambda h: h.hostname == hostname, shared.host_pool.hosts), None,) if not host: - self.add_error( - "No Such Host ({}) exists".format( - self.destination.value - ) - ) + raise ValidationException('No Such Host ({}) exists'.format(self.destination.value)) elif host.status != HostStatus.alive: - self.add_error("Destination Host is dead") + raise ValidationException('Destination Host is dead') else: self.destination.value = host.key def validation(self): vm = shared.vm_pool.get(self.uuid.value) - if not ( - vm.value["owner"] == self.name.value - or self.realm.value == "ungleich-admin" - ): - self.add_error("Invalid User") + if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'): + raise ValidationException('Invalid User') if vm.status != VMStatus.running: - self.add_error("Can't migrate non-running VM") + raise ValidationException("Can't migrate non-running VM") - if vm.hostname == os.path.join( - settings["etcd"]["host_prefix"], self.destination.value - ): - self.add_error( - "Destination host couldn't be same as Source Host" - ) + if vm.hostname == os.path.join(settings['etcd']['host_prefix'], self.destination.value): + raise ValidationException("Destination host couldn't be same as Source Host") class AddSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field( - "key_name", str, data.get("key_name", KeyError) - ) - self.key = Field("key", str, data.get("key_name", KeyError)) - - fields = [self.key_name, self.key] - super().__init__(data=data, fields=fields) + self.key_name = Field('key_name', str, get(data, 'key_name')) + self.key = Field('key', str, get(data, 'key')) + super().__init__(data=data) class RemoveSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field( - "key_name", str, data.get("key_name", KeyError) - ) - - fields = [self.key_name] - super().__init__(data=data, fields=fields) + self.key_name = Field('key_name', str, get(data, 'key_name')) + super().__init__(data=data) class GetSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field( - "key_name", str, data.get("key_name", None) - ) - - fields = [self.key_name] - super().__init__(data=data, fields=fields) + self.key_name = Field('key_name', str, get(data, 'key_name', return_default=True)) + super().__init__(data=data) class CreateNetwork(OTPSchema): def __init__(self, data): - self.network_name = Field("network_name", str, data.get("network_name", KeyError)) - self.type = Field("type", str, data.get("type", KeyError)) - self.user = Field("user", bool, bool(data.get("user", False))) - - self.network_name.validation = self.network_name_validation - self.type.validation = self.network_type_validation - - fields = [self.network_name, self.type, self.user] - super().__init__(data, fields=fields) + self.network_name = Field('network_name', str, get(data, 'name'), + validators=[self.network_name_validation]) + self.type = Field('type', str, get(data, 'type'), validators=[self.network_type_validation]) + self.user = Field('user', bool, bool(get(data, 'user', return_default=True, default=False))) + super().__init__(data) def network_name_validation(self): - print(self.name.value, self.network_name.value) - key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) - print(key) + key = os.path.join(settings['etcd']['network_prefix'], self.name.value, self.network_name.value) network = shared.etcd_client.get(key, value_in_json=True) if network: - self.add_error( - "Network with name {} already exists".format( - self.network_name.value - ) - ) + raise ValidationException('Network with name {} already exists'.format(self.network_name.value)) def network_type_validation(self): - supported_network_types = ["vxlan"] + supported_network_types = ['vxlan'] if self.type.value not in supported_network_types: - self.add_error( - "Unsupported Network Type. Supported network types are {}".format( - supported_network_types - ) - ) + raise ValidationException('Unsupported Network Type. Supported network types are {}'.format(supported_network_types)) diff --git a/uncloud/common/counters.py b/uncloud/common/counters.py index 2d4a8e9..dba8a08 100644 --- a/uncloud/common/counters.py +++ b/uncloud/common/counters.py @@ -1,8 +1,8 @@ -from .etcd_wrapper import Etcd3Wrapper +from uncloud.common.shared import shared -def increment_etcd_counter(etcd_client: Etcd3Wrapper, key): - kv = etcd_client.get(key) +def increment_etcd_counter(key): + kv = shared.etcd_client.get(key) if kv: counter = int(kv.value) @@ -10,12 +10,12 @@ def increment_etcd_counter(etcd_client: Etcd3Wrapper, key): else: counter = 1 - etcd_client.put(key, str(counter)) + shared.etcd_client.put(key, str(counter)) return counter -def get_etcd_counter(etcd_client: Etcd3Wrapper, key): - kv = etcd_client.get(key) +def get_etcd_counter(key): + kv = shared.etcd_client.get(key) if kv: return int(kv.value) return None diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index fe768ac..119d0e6 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -5,6 +5,7 @@ from functools import wraps from uncloud import UncloudException from uncloud.common import logger +from typing import Iterator class EtcdEntry: @@ -42,14 +43,30 @@ class Etcd3Wrapper: self.client = etcd3.client(*args, **kwargs) @readable_errors - def get(self, *args, value_in_json=False, **kwargs): + def get(self, *args, value_in_json=False, **kwargs) -> EtcdEntry: + """Get a key/value pair from etcd + + :return: + EtcdEntry: if a key/value pair is found in etcd + :raises: + KeyError: If key is not found in etcd + Exception: Different type of exception can be raised depending on + situation + """ _value, _key = self.client.get(*args, **kwargs) if _key is None or _value is None: - return None + raise KeyError return EtcdEntry(_key, _value, value_in_json=value_in_json) @readable_errors def put(self, *args, value_in_json=False, **kwargs): + """Put key/value pair in etcd + + :return: a response containing a header and the prev_kv + :raises: + Exception: Different type of exception can be raised depending on + situation + """ _key, _value = args if value_in_json: _value = json.dumps(_value) @@ -60,20 +77,14 @@ class Etcd3Wrapper: return self.client.put(_key, _value, **kwargs) @readable_errors - def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): - try: - event_iterator = self.client.get_prefix(*args, **kwargs) - for e in event_iterator: - yield EtcdEntry(*e[::-1], value_in_json=value_in_json) - except Exception as err: - if raise_exception: - raise Exception('Exception in etcd_wrapper.get_prefix') from err - else: - logger.exception('Error in etcd_wrapper') - return iter([]) + def get_prefix(self, *args, value_in_json=False, **kwargs) -> \ + Iterator[EtcdEntry]: + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) @readable_errors - def watch_prefix(self, key, raise_exception=True, value_in_json=False): + def watch_prefix(self, key, raise_exception=True, value_in_json=False) -> Iterator[EtcdEntry]: try: event_iterator, cancel = self.client.watch_prefix(key) for e in event_iterator: diff --git a/uncloud/common/network.py b/uncloud/common/network.py index 32f6951..e74359d 100644 --- a/uncloud/common/network.py +++ b/uncloud/common/network.py @@ -1,6 +1,7 @@ import subprocess as sp import random import logging +import ipaddress logger = logging.getLogger(__name__) @@ -9,9 +10,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -68,3 +67,21 @@ def delete_network_interface(iface): except Exception: logger.exception("Interface %s Deletion failed", iface) + +def mac2ipv6(mac, prefix): + # only accept MACs separated by a colon + parts = mac.split(':') + + # modify parts to match IPv6 value + parts.insert(3, 'ff') + parts.insert(4, 'fe') + parts[0] = '%x' % (int(parts[0], 16) ^ 2) + + # format output + ipv6_parts = [str(0)] * 4 + for i in range(0, len(parts), 2): + ipv6_parts.append(''.join(parts[i: i + 2])) + + lower_part = ipaddress.IPv6Address(':'.join(ipv6_parts)) + prefix = ipaddress.IPv6Address(prefix) + return str(prefix + int(lower_part)) From f1bb1ee3cae6fe006060ab2a1f9917d98ecc33f2 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 10 Jan 2020 00:33:35 +0500 Subject: [PATCH 03/75] 2nd commit --- uncloud/api/main.py | 98 +++++++++++++++++++++--------------------- uncloud/api/schemas.py | 29 +++++++------ 2 files changed, 64 insertions(+), 63 deletions(-) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 15f3e2a..032864c 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -33,16 +33,16 @@ arg_parser = argparse.ArgumentParser('api', add_help=False) arg_parser.add_argument('--port', '-p') -@app.errorhandler(Exception) -def handle_exception(e): - app.logger.error(e) - - # pass through HTTP errors - if isinstance(e, HTTPException): - return e - - # now you're handling non-HTTP exceptions only - return {'message': 'Server Error'}, 500 +# @app.errorhandler(Exception) +# def handle_exception(e): +# app.logger.error(e) +# +# # pass through HTTP errors +# if isinstance(e, HTTPException): +# return e +# +# # now you're handling non-HTTP exceptions only +# return {'message': 'Server Error'}, 500 class CreateVM(Resource): @@ -63,22 +63,22 @@ class CreateVM(Resource): 'os-ssd': validator.specs['os-ssd'], 'hdd': validator.specs['hdd'], } - macs = [generate_mac() for _ in range(len(validator.network.value))] + macs = [generate_mac() for _ in range(len(validator.network))] tap_ids = [ counters.increment_etcd_counter(settings['etcd']['tap_counter']) - for _ in range(len(validator.network.value)) + for _ in range(len(validator.network)) ] vm_entry = { - 'name': validator.vm_name.value, - 'owner': validator.name.value, - 'owner_realm': validator.realm.value, + 'name': validator.vm_name, + 'owner': validator.name, + 'owner_realm': validator.realm, 'specs': specs, 'hostname': '', 'status': VMStatus.stopped, 'image_uuid': validator.image_uuid, 'log': [], 'vnc_socket': '', - 'network': list(zip(validator.network.value, macs, tap_ids)), + 'network': list(zip(validator.network, macs, tap_ids)), 'metadata': {'ssh-keys': []}, 'in_migration': False, } @@ -94,7 +94,7 @@ class CreateVM(Resource): return make_return_message('VM Creation Queued') -class VmStatus(Resource): +class GetVMStatus(Resource): @staticmethod def post(): data = request.json @@ -104,13 +104,13 @@ class VmStatus(Resource): except (ValidationException, Exception) as err: return make_return_message(err, 400) else: - vm = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], data['uuid'])) + vm = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], validator.uuid)) vm_value = vm.value.copy() vm_value['ip'] = [] for network_mac_and_tap in vm.network: network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( - join_path(settings['etcd']['network_prefix'], data['name'], network_name), + join_path(settings['etcd']['network_prefix'], validator.name, network_name), value_in_json=True, ) ipv6_addr = (network.value.get('ipv6').split('::')[0] + '::') @@ -131,7 +131,7 @@ class CreateImage(Resource): else: try: file_entry = shared.etcd_client.get( - join_path(settings['etcd']['file_prefix'], data['uuid']), value_in_json=True + join_path(settings['etcd']['file_prefix'], validator.uuid), value_in_json=True ) except KeyError: # TODO: Add some message @@ -146,7 +146,7 @@ class CreateImage(Resource): 'visibility': 'public', } shared.etcd_client.put( - join_path(settings['etcd']['image_prefix'], data['uuid']), + join_path(settings['etcd']['image_prefix'], validator.uuid), json.dumps(image_entry_json), ) @@ -174,8 +174,8 @@ class VMAction(Resource): except ValidationException as err: return make_return_message(err, 400) else: - vm_entry = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], data['uuid'])) - action = validator.action.value + vm_entry = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], validator.uuid)) + action = validator.action if action == 'start': action = 'schedule' @@ -195,7 +195,7 @@ class VMAction(Resource): r = RequestEntry.from_scratch( type='{}VM'.format(action.title()), - uuid=data['uuid'], + uuid=validator.uuid, hostname=vm_entry.hostname, request_prefix=settings['etcd']['request_prefix'], ) @@ -213,13 +213,13 @@ class VMMigration(Resource): except ValidationException as err: return make_return_message(err), 400 else: - vm = shared.vm_pool.get(validator.uuid.value) + vm = shared.vm_pool.get(validator.uuid) r = RequestEntry.from_scratch( type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( settings['etcd']['host_prefix'], - validator.destination.value, + validator.destination, ), request_prefix=settings['etcd']['request_prefix'], ) @@ -240,7 +240,7 @@ class ListUserVM(Resource): else: vms = shared.etcd_client.get_prefix(settings['etcd']['vm_prefix'], value_in_json=True) return_vms = [] - user_vms = filter(lambda v: v.value['owner'] == validator.name.value, vms) + user_vms = filter(lambda v: v.value['owner'] == validator.name, vms) for vm in user_vms: return_vms.append( { @@ -267,7 +267,7 @@ class ListUserFiles(Resource): else: files = shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) return_files = [] - user_files = [f for f in files if f.value['owner'] == data['name']] + user_files = [f for f in files if f.value['owner'] == validator.name] for file in user_files: file_uuid = file.key.split('/')[-1] file = file.value @@ -292,8 +292,8 @@ class CreateHost(Resource): else: host_key = join_path(settings['etcd']['host_prefix'], uuid4().hex) host_entry = { - 'specs': validator.specs.value, - 'hostname': validator.hostname.value, + 'specs': validator.specs, + 'hostname': validator.hostname, 'status': HostStatus.dead, 'last_heartbeat': '', } @@ -325,9 +325,9 @@ class GetSSHKeys(Resource): except ValidationException as err: return make_return_message(err, 400) else: - etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm.value, - validator.name.value, 'key') - if not validator.key_name.value: + etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm, + validator.name, 'key') + if not validator.key_name: etcd_entry = shared.etcd_client.get_prefix(etcd_key, value_in_json=True) keys = { key.key.split('/')[-1]: key.value @@ -335,7 +335,7 @@ class GetSSHKeys(Resource): } return {'keys': keys} else: - etcd_key = join_path(validator.key_name.value) + etcd_key = join_path(validator.key_name) try: etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) except KeyError: @@ -358,17 +358,17 @@ class AddSSHKey(Resource): else: # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], validator.realm.value, - validator.name.value, 'key', validator.key_name.value + settings['etcd']['user_prefix'], validator.realm, + validator.name, 'key', validator.key_name ) try: shared.etcd_client.get(etcd_key, value_in_json=True) except KeyError: # Key Not Found. It implies user' haven't added any key yet. - shared.etcd_client.put(etcd_key, validator.key.value, value_in_json=True) + shared.etcd_client.put(etcd_key, validator.key, value_in_json=True) return make_return_message('Key added successfully') else: - return make_return_message('Key "{}" already exists'.format(validator.key_name.value)) + return make_return_message('Key "{}" already exists'.format(validator.key_name)) class RemoveSSHKey(Resource): @@ -382,12 +382,12 @@ class RemoveSSHKey(Resource): return make_return_message(err, 400) else: # {user_prefix}/{realm}/{name}/key/{key_name} - etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm.value, - validator.name.value, 'key', validator.key_name.value) + etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm, + validator.name, 'key', validator.key_name) try: etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True) except KeyError: - return make_return_message('No Key "{}" exists.'.format(validator.key_name.value)) + return make_return_message('No Key "{}" exists.'.format(validator.key_name)) if etcd_entry: shared.etcd_client.client.delete(etcd_key) return {'message': 'Key successfully removed.'} @@ -405,9 +405,9 @@ class CreateNetwork(Resource): else: network_entry = { 'id': counters.increment_etcd_counter(settings['etcd']['vxlan_counter']), - 'type': validator.type.value, + 'type': validator.type, } - if validator.user.value: + if validator.user: try: nb = pynetbox.api(url=settings['netbox']['url'], token=settings['netbox']['token']) nb_prefix = nb.ipam.prefixes.get(prefix=settings['network']['prefix']) @@ -415,8 +415,8 @@ class CreateNetwork(Resource): data={ 'prefix_length': int(settings['network']['prefix_length']), 'description': '{}\'s network "{}"'.format( - validator.name.value, - validator.network_name.value + validator.name, + validator.network_name ), 'is_pool': True, } @@ -429,8 +429,8 @@ class CreateNetwork(Resource): else: network_entry['ipv6'] = 'fd00::/64' - network_key = join_path(settings['etcd']['network_prefix'], validator.name.value, - validator.network_name.value) + network_key = join_path(settings['etcd']['network_prefix'], validator.name, + validator.network_name) shared.etcd_client.put(network_key, network_entry, value_in_json=True) return make_return_message('Network successfully added.') @@ -445,7 +445,7 @@ class ListUserNetwork(Resource): except ValidationException as err: return make_return_message(err, 400) else: - prefix = join_path(settings['etcd']['network_prefix'], data['name']) + prefix = join_path(settings['etcd']['network_prefix'], validator.name) networks = shared.etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] for net in networks: @@ -455,7 +455,7 @@ class ListUserNetwork(Resource): api.add_resource(CreateVM, '/vm/create') -api.add_resource(VmStatus, '/vm/status') +api.add_resource(GetVMStatus, '/vm/status') api.add_resource(VMAction, '/vm/action') api.add_resource(VMMigration, '/vm/migrate') diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index ffa33f6..7d8969b 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -30,9 +30,9 @@ class Field: def is_valid(self): if not isinstance(self.value, self.type): raise ValidationException("Incorrect Type for '{}' field".format(self.name)) - else: - for validator in self.validators: - validator() + + for validator in self.validators: + validator() def __repr__(self): return self.name @@ -52,8 +52,7 @@ class VmUUIDField(Field): class BaseSchema: - def __init__(self, data): - print(data) + def __init__(self): self.fields = [getattr(self, field) for field in dir(self) if isinstance(getattr(self, field), Field)] def validation(self): @@ -69,6 +68,9 @@ class BaseSchema: self.validation() + for field in self.fields: + setattr(self, field.name, field.value) + def get(dictionary: dict, key: str, return_default=False, default=None): if dictionary is None: @@ -88,7 +90,7 @@ class OTPSchema(BaseSchema): self.name = Field('name', str, get(data, 'name')) self.realm = Field('realm', str, get(data, 'realm')) self.token = Field('token', str, get(data, 'token')) - super().__init__(data=data) + super().__init__() def validation(self): if check_otp(self.name.value, self.realm.value, self.token.value) != 200: @@ -101,8 +103,7 @@ class CreateImageSchema(BaseSchema): self.name = Field('name', str, get(data, 'name')) self.image_store = Field('image_store', str, get(data, 'image_store'), validators=[self.image_store_name_validation]) - - super().__init__(data) + super().__init__() def file_uuid_validation(self): try: @@ -123,7 +124,7 @@ class CreateHostSchema(OTPSchema): self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation]) self.hostname = Field('hostname', str, get(data, 'hostname')) - super().__init__(data=data) + super().__init__(data) def specs_validation(self): allowed_base = 10 @@ -191,7 +192,7 @@ class CreateVMSchema(OTPSchema): def image_validation(self): try: image_uuid = helper.resolve_image_name(self.image.value) - except Exception as e: + except Exception: raise ValidationException('No image of name \'{}\' found'.format(self.image.value)) else: self.image_uuid = image_uuid @@ -214,7 +215,7 @@ class CreateVMSchema(OTPSchema): raise ValidationException('Network with name {} does not exists'.format(net)) def specs_validation(self): - ALLOWED_BASE = 10 + allowed_base = 10 try: _cpu = get(self.specs.value, 'cpu') @@ -228,10 +229,10 @@ class CreateVMSchema(OTPSchema): parsed_ram = bitmath.parse_string_unsafe(_ram) parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) - if parsed_ram.base != ALLOWED_BASE: + if parsed_ram.base != allowed_base: raise ValidationException('Your specified RAM is not in correct units') - if parsed_os_ssd.base != ALLOWED_BASE: + if parsed_os_ssd.base != allowed_base: raise ValidationException('Your specified OS-SSD is not in correct units') if int(_cpu) < 1: @@ -246,7 +247,7 @@ class CreateVMSchema(OTPSchema): parsed_hdd = [] for hdd in _hdd: _parsed_hdd = bitmath.parse_string_unsafe(hdd) - if _parsed_hdd.base != ALLOWED_BASE: + if _parsed_hdd.base != allowed_base: raise ValidationException('Your specified HDD is not in correct units') else: parsed_hdd.append(str(_parsed_hdd)) From feb334cf0451655d8dfc210beef4be11f817781d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 10:07:01 +0100 Subject: [PATCH 04/75] Exit code == 1 in case we died with an exception --- scripts/uncloud | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/uncloud b/scripts/uncloud index 1ca9c68..533fc4b 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -40,7 +40,6 @@ if __name__ == '__main__': if not args.command: arg_parser.print_help() else: - # if we start etcd in seperate process with default settings # i.e inheriting few things from parent process etcd3 module # errors out, so the following command configure multiprocessing @@ -54,5 +53,7 @@ if __name__ == '__main__': main(**arguments) except UncloudException as err: logger.error(err) + sys.exit(1) except Exception as err: logger.exception(err) + sys.exit(1) From 92f985c857ac872a2a027cec032a65f536d231e1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 10:10:37 +0100 Subject: [PATCH 05/75] Handle etcd connection error --- uncloud/common/etcd_wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index fe768ac..211bd3c 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -65,6 +65,8 @@ class Etcd3Wrapper: event_iterator = self.client.get_prefix(*args, **kwargs) for e in event_iterator: yield EtcdEntry(*e[::-1], value_in_json=value_in_json) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException("Cannot connect to etcd: {}".format(e)) except Exception as err: if raise_exception: raise Exception('Exception in etcd_wrapper.get_prefix') from err From 71fd0ca7d9c141045ea42c34cda4a38730fda789 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:00:00 +0100 Subject: [PATCH 06/75] Remove double try/except blocks (with wraps) --- uncloud/common/etcd_wrapper.py | 38 ++++++++-------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py index 211bd3c..38471ab 100644 --- a/uncloud/common/etcd_wrapper.py +++ b/uncloud/common/etcd_wrapper.py @@ -61,35 +61,15 @@ class Etcd3Wrapper: @readable_errors def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): - try: - event_iterator = self.client.get_prefix(*args, **kwargs) - for e in event_iterator: - yield EtcdEntry(*e[::-1], value_in_json=value_in_json) - except etcd3.exceptions.ConnectionFailedError as e: - raise UncloudException("Cannot connect to etcd: {}".format(e)) - except Exception as err: - if raise_exception: - raise Exception('Exception in etcd_wrapper.get_prefix') from err - else: - logger.exception('Error in etcd_wrapper') - return iter([]) + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) @readable_errors def watch_prefix(self, key, raise_exception=True, value_in_json=False): - try: - event_iterator, cancel = self.client.watch_prefix(key) - for e in event_iterator: - if hasattr(e, '_event'): - e = e._event - if e.type == e.PUT: - yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) - except Exception as err: - if raise_exception: - raise Exception('Exception in etcd_wrapper.get_prefix') from err - else: - logger.exception('Error in etcd_wrapper.watch_prefix') - try: - cancel() - except Exception: - pass - return iter([]) + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = e._event + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) From b7596e071a127c5a3bd1af17bff5efbd400d3dd7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:30:23 +0100 Subject: [PATCH 07/75] begin phasing in arguments instead of **arguments --- scripts/uncloud | 35 +++++++++++++++++++++++++++++++---- uncloud/api/main.py | 5 ++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 533fc4b..4f83e9e 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -8,6 +8,17 @@ import multiprocessing as mp from uncloud import UncloudException from contextlib import suppress +# the components that use etcd +ETCD_COMPONENTS= ['api', + 'scheduler', + 'host', + 'filescanner', + 'imagescanner', + 'metadata', + 'configure' ] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('cli') def exception_hook(exc_type, exc_value, exc_traceback): logging.getLogger(__name__).error( @@ -30,11 +41,25 @@ if __name__ == '__main__': parent_parser.add_argument('--debug', '-d', action='store_true', default=False, help='More verbose logging') - for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', - 'metadata', 'configure', 'cli']: + etcd_parser = argparse.ArgumentParser(add_help=False) + etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-port') + etcd_parser.add_argument('--etcd-ca-cert', + help="CA that signed the etcd certificate") + etcd_parser.add_argument('--etcd-cert-cert', + help="Path to client certificate") + etcd_parser.add_argument('--etcd-cert-key', + help="Path to client certificate key") + + for component in ALL_COMPONENTS: mod = importlib.import_module('uncloud.{}.main'.format(component)) parser = getattr(mod, 'arg_parser') - subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) + + if component in ETCD_COMPONENTS: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) + else: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) + args = arg_parser.parse_args() if not args.command: @@ -46,11 +71,13 @@ if __name__ == '__main__': # module to not inherit anything from parent. # mp.set_start_method('spawn') arguments = vars(args) + print(arguments) + print(etcd_parser) try: name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') - main(**arguments) + main(arguments) except UncloudException as err: logger.error(err) sys.exit(1) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 2d8d035..de75f07 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -563,7 +563,10 @@ api.add_resource(ListHost, '/host/list') api.add_resource(CreateNetwork, '/network/create') -def main(debug=False, port=None): +def main(arguments): + debug = arguments['debug'] + port = arguments['port'] + try: image_stores = list( shared.etcd_client.get_prefix( From d9dd6b48dcce5e70f94c0a3c29548258ed6c05ca Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:35:04 +0100 Subject: [PATCH 08/75] No try: needed for pop/importlib/getattr --- scripts/uncloud | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 4f83e9e..6da98e0 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -73,10 +73,14 @@ if __name__ == '__main__': arguments = vars(args) print(arguments) print(etcd_parser) + + name = arguments.pop('command') + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') + + if component in ETCD_COMPONENTS: + import etcd3 try: - name = arguments.pop('command') - mod = importlib.import_module('uncloud.{}.main'.format(name)) - main = getattr(mod, 'main') main(arguments) except UncloudException as err: logger.error(err) From 82a69701ceb554d3838089ef4f92326ebf603d16 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:43:53 +0100 Subject: [PATCH 09/75] catch etcd in scripts/ --- scripts/uncloud | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 6da98e0..4f9b38b 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -71,20 +71,22 @@ if __name__ == '__main__': # module to not inherit anything from parent. # mp.set_start_method('spawn') arguments = vars(args) - print(arguments) - print(etcd_parser) - name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') - if component in ETCD_COMPONENTS: + # If the component requires etcd3, we import it and catch the + # etcd3.exceptions.ConnectionFailedError + if name in ETCD_COMPONENTS: import etcd3 + try: main(arguments) except UncloudException as err: logger.error(err) sys.exit(1) + except etcd3.exceptions.ConnectionFailedError as err: + logger.error("Cannot connect to etcd") except Exception as err: logger.exception(err) sys.exit(1) From 31ec024be621be09b564979cab2f2702e6141329 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 10 Jan 2020 15:45:48 +0500 Subject: [PATCH 10/75] passing arguments dict to componenets instead of **kwargs --- scripts/uncloud | 30 +++++++----------------------- uncloud/api/main.py | 10 ++-------- uncloud/filescanner/main.py | 3 ++- uncloud/host/main.py | 3 ++- uncloud/imagescanner/main.py | 2 +- uncloud/metadata/main.py | 8 +++----- uncloud/scheduler/main.py | 6 +----- 7 files changed, 18 insertions(+), 44 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 6da98e0..1975f84 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -3,23 +3,16 @@ import logging import sys import importlib import argparse -import multiprocessing as mp from uncloud import UncloudException -from contextlib import suppress # the components that use etcd -ETCD_COMPONENTS= ['api', - 'scheduler', - 'host', - 'filescanner', - 'imagescanner', - 'metadata', - 'configure' ] +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() ALL_COMPONENTS.append('cli') + def exception_hook(exc_type, exc_value, exc_traceback): logging.getLogger(__name__).error( 'Uncaught exception', @@ -44,12 +37,9 @@ if __name__ == '__main__': etcd_parser = argparse.ArgumentParser(add_help=False) etcd_parser.add_argument('--etcd-host') etcd_parser.add_argument('--etcd-port') - etcd_parser.add_argument('--etcd-ca-cert', - help="CA that signed the etcd certificate") - etcd_parser.add_argument('--etcd-cert-cert', - help="Path to client certificate") - etcd_parser.add_argument('--etcd-cert-key', - help="Path to client certificate key") + etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') + etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') + etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') for component in ALL_COMPONENTS: mod = importlib.import_module('uncloud.{}.main'.format(component)) @@ -60,19 +50,13 @@ if __name__ == '__main__': else: subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - args = arg_parser.parse_args() if not args.command: arg_parser.print_help() else: - # if we start etcd in seperate process with default settings - # i.e inheriting few things from parent process etcd3 module - # errors out, so the following command configure multiprocessing - # module to not inherit anything from parent. - # mp.set_start_method('spawn') arguments = vars(args) - print(arguments) - print(etcd_parser) + # print(arguments) + # print(etcd_parser) name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) diff --git a/uncloud/api/main.py b/uncloud/api/main.py index de75f07..34e1dd1 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -565,7 +565,7 @@ api.add_resource(CreateNetwork, '/network/create') def main(arguments): debug = arguments['debug'] - port = arguments['port'] + port = arguments['port'] try: image_stores = list( @@ -596,12 +596,6 @@ def main(arguments): # ) try: - app.run(host='::', - port=port, - debug=debug) + app.run(host='::', port=port, debug=debug) except OSError as e: raise UncloudException('Failed to start Flask: {}'.format(e)) - - -if __name__ == '__main__': - main() diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index 314481f..c5660dd 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -68,7 +68,8 @@ def track_file(file, base_dir, host): shared.etcd_client.put(entry_key, entry_value, value_in_json=True) -def main(hostname, debug=False): +def main(arguments): + hostname = arguments['hostname'] base_dir = settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index 695e3d1..ccffd77 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -44,7 +44,8 @@ def maintenance(host): shared.vm_pool.put(vm) -def main(hostname, debug=False): +def main(arguments): + hostname = arguments['hostname'] host_pool = shared.host_pool host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index 91f100e..1803213 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -30,7 +30,7 @@ def qemu_img_type(path): return qemu_img_info["format"] -def main(debug=False): +def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( settings["etcd"]["image_prefix"], value_in_json=True diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index 73d59cd..ccda60e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -88,9 +88,7 @@ class Root(Resource): api.add_resource(Root, "/") -def main(port=None, debug=False): +def main(arguments): + port = arguments['port'] + debug = arguments['debug'] app.run(debug=debug, host="::", port=port) - - -if __name__ == "__main__": - main() diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index 20a52cb..c25700b 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -16,7 +16,7 @@ from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection, arg_parser = argparse.ArgumentParser('scheduler', add_help=False) -def main(debug=False): +def main(arguments): # The below while True is neccessary for gracefully handling leadership transfer and temporary # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) @@ -50,7 +50,3 @@ def main(debug=False): shared.vm_pool.put(vm_entry) logger.info('No Resource Left. Emailing admin....') - - -if __name__ == '__main__': - main() From ec66a756a011325a96719a2aada9c7b1b1d6c2fc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 11:56:47 +0100 Subject: [PATCH 11/75] ++confdir --- scripts/uncloud | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/uncloud b/scripts/uncloud index 4f9b38b..a6747be 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -38,8 +38,12 @@ if __name__ == '__main__': subparsers = arg_parser.add_subparsers(dest='command') parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument('--debug', '-d', action='store_true', default=False, + parent_parser.add_argument('--debug', '-d', + action='store_true', + default=False, help='More verbose logging') + parent_parser.add_argument('--conf-dir', '-c', + help='Configuration directory') etcd_parser = argparse.ArgumentParser(add_help=False) etcd_parser.add_argument('--etcd-host') From e91fd9e24af37bcf7703dfd39a8496bfb28ac1f7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 12:00:02 +0100 Subject: [PATCH 12/75] disable cli until bug #25 is fixed --- scripts/uncloud | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uncloud b/scripts/uncloud index bfb3174..3690ad9 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -10,7 +10,7 @@ from uncloud import UncloudException ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('cli') +#ALL_COMPONENTS.append('cli') def exception_hook(exc_type, exc_value, exc_traceback): From 00d876aea1a92c8aaf2be2a2e95bd42d3985335a Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 10 Jan 2020 16:39:40 +0500 Subject: [PATCH 13/75] Do not break if client section/or OTP creds missing from conf file --- scripts/uncloud | 2 -- uncloud/cli/helper.py | 17 +++++++++++++---- uncloud/cli/main.py | 10 +++++----- uncloud/common/settings.py | 6 ++---- 4 files changed, 20 insertions(+), 15 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 3690ad9..8aac240 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -73,8 +73,6 @@ if __name__ == '__main__': except UncloudException as err: logger.error(err) sys.exit(1) - except etcd3.exceptions.ConnectionFailedError as err: - logger.error("Cannot connect to etcd") except Exception as err: logger.exception(err) sys.exit(1) diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py index 0495fac..3c63073 100644 --- a/uncloud/cli/helper.py +++ b/uncloud/cli/helper.py @@ -10,10 +10,19 @@ from uncloud.common.settings import settings def get_otp_parser(): otp_parser = argparse.ArgumentParser('otp') - otp_parser.add_argument('--name', default=settings['client']['name']) - otp_parser.add_argument('--realm', default=settings['client']['realm']) - otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'], - dest='token', metavar='SEED') + try: + name = settings['client']['name'] + realm = settings['client']['realm'] + seed = settings['client']['seed'] + except Exception: + otp_parser.add_argument('--name', required=True) + otp_parser.add_argument('--realm', required=True) + otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED') + else: + otp_parser.add_argument('--name', default=name) + otp_parser.add_argument('--realm', default=realm) + otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED') + return otp_parser diff --git a/uncloud/cli/main.py b/uncloud/cli/main.py index 7f5e367..9a42497 100644 --- a/uncloud/cli/main.py +++ b/uncloud/cli/main.py @@ -12,12 +12,12 @@ for component in ['user', 'host', 'image', 'network', 'vm']: subparser.add_parser(name=parser.prog, parents=[parser]) -def main(**kwargs): - if not kwargs['subcommand']: +def main(arguments): + if not arguments['subcommand']: arg_parser.print_help() else: - name = kwargs.pop('subcommand') - kwargs.pop('debug') + name = arguments.pop('subcommand') + arguments.pop('debug') mod = importlib.import_module('uncloud.cli.{}'.format(name)) _main = getattr(mod, 'main') - _main(**kwargs) + _main(**arguments) diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 47ad5a7..0d524a7 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -99,12 +99,10 @@ class Settings(object): def read_config_file_values(self, config_file): try: # Trying to read configuration file - with open(config_file, 'r') as config_file_handle: + with open(config_file) as config_file_handle: self.config_parser.read_file(config_file_handle) except FileNotFoundError: - sys.exit( - 'Configuration file {} not found!'.format(config_file) - ) + sys.exit('Configuration file {} not found!'.format(config_file)) except Exception as err: logger.exception(err) sys.exit('Error occurred while reading configuration file') From cf4930ee84c0b588fd322a34a40e232d9d4c583c Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 10 Jan 2020 16:42:07 +0500 Subject: [PATCH 14/75] cli enabled again --- scripts/uncloud | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uncloud b/scripts/uncloud index 8aac240..a6e61aa 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -10,7 +10,7 @@ from uncloud import UncloudException ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() -#ALL_COMPONENTS.append('cli') +ALL_COMPONENTS.append('cli') def exception_hook(exc_type, exc_value, exc_traceback): From ebcb1680d70114364dfdcab9cc6e8699ffc52f50 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 23:27:21 +0100 Subject: [PATCH 15/75] add hack scripts --- uncloud/hack/hackcloud/foo | 2 ++ uncloud/hack/hackcloud/ifup.sh | 8 ++++++ uncloud/hack/hackcloud/net.sh | 21 +++++++++++++++ uncloud/hack/hackcloud/vm.sh | 48 ++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 uncloud/hack/hackcloud/foo create mode 100755 uncloud/hack/hackcloud/ifup.sh create mode 100755 uncloud/hack/hackcloud/net.sh create mode 100755 uncloud/hack/hackcloud/vm.sh diff --git a/uncloud/hack/hackcloud/foo b/uncloud/hack/hackcloud/foo new file mode 100644 index 0000000..1033abf --- /dev/null +++ b/uncloud/hack/hackcloud/foo @@ -0,0 +1,2 @@ +tap0 +tap0 diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh new file mode 100755 index 0000000..95bfe5a --- /dev/null +++ b/uncloud/hack/hackcloud/ifup.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +echo $@ >> foo + +dev=$1; shift + +# bridge is setup from outside +ip link set dev "$dev" master ${bridge} diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh new file mode 100755 index 0000000..e56822f --- /dev/null +++ b/uncloud/hack/hackcloud/net.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +netid=100 +dev=wlp2s0 +dev=wlp0s20f3 +dev=wlan0 + +vxlandev=vxlan${netid} +bridgedev=br${netid} + +ip -6 link add ${vxlandev} type vxlan \ + id ${netid} \ + dstport 4789 \ + group ff05::${netid} \ + dev ${dev} \ + ttl 5 + +ip link set ${vxlandev} up + +ip link add ${bridgedev} type bridge +ip link set ${bridgedev} up diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh new file mode 100755 index 0000000..2a8b794 --- /dev/null +++ b/uncloud/hack/hackcloud/vm.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +vmid=$1; shift + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +accel=tcg + +memory=1024 +cores=2 +uuid=732e08c7-84f8-4d43-9571-263db4f80080 + +export bridge=br100 + +$qemu -name uc${vmid} \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e + + + +exit 0 + +-S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-17-one-24992/master-key.aes +-machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off + +-m 2048 +-realtime mlock=off +-smp 1,sockets=1,cores=1,threads=1 +-uuid 732e08c7-84f8-4d43-9571-263db4f80080 -no-user-config \ + -nodefaults +-chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-17-one-24992/monitor.sock,server,nowait +-mon chardev=charmonitor,id=monitor,mode=control +-rtc base=utc -no-shutdown +-boot strict=on +-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 +-drive file=rbd:ssd/one-292-24992-0:id=libvirt:auth_supported=cephx\;none:mon_host=ceph1\:6789\;ceph2\:6789\;ceph3\:6789,format=raw,if=none,id=drive-virtio-disk0,cache=none +-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 +-drive file=/var/lib/one//datastores/104/24992/disk.1,format=raw,if=none,id=drive-ide0-0-0,readonly=on +-device ide-cd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0 +-netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38 +-device virtio-net-pci,netdev=hostnet0,id=net0,mac=02:00:f0:a9:c4:4e,bus=pci.0,addr=0x3 +-vnc [::]:4414 -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on From b9c9a5e0eca41009f5461ff601b11a3bf2151ea7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2020 23:55:21 +0100 Subject: [PATCH 16/75] add working network --- uncloud/hack/hackcloud/.gitignore | 3 +++ uncloud/hack/hackcloud/foo | 2 -- uncloud/hack/hackcloud/ifup.sh | 1 + uncloud/hack/hackcloud/net.sh | 3 +++ uncloud/hack/hackcloud/radvd.conf | 13 +++++++++++++ uncloud/hack/hackcloud/radvd.sh | 3 +++ 6 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 uncloud/hack/hackcloud/.gitignore delete mode 100644 uncloud/hack/hackcloud/foo create mode 100644 uncloud/hack/hackcloud/radvd.conf create mode 100644 uncloud/hack/hackcloud/radvd.sh diff --git a/uncloud/hack/hackcloud/.gitignore b/uncloud/hack/hackcloud/.gitignore new file mode 100644 index 0000000..0ad647b --- /dev/null +++ b/uncloud/hack/hackcloud/.gitignore @@ -0,0 +1,3 @@ +*.iso +radvdpid +foo diff --git a/uncloud/hack/hackcloud/foo b/uncloud/hack/hackcloud/foo deleted file mode 100644 index 1033abf..0000000 --- a/uncloud/hack/hackcloud/foo +++ /dev/null @@ -1,2 +0,0 @@ -tap0 -tap0 diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh index 95bfe5a..99e8690 100755 --- a/uncloud/hack/hackcloud/ifup.sh +++ b/uncloud/hack/hackcloud/ifup.sh @@ -6,3 +6,4 @@ dev=$1; shift # bridge is setup from outside ip link set dev "$dev" master ${bridge} +ip link set dev "$dev" up diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh index e56822f..7d4b88f 100755 --- a/uncloud/hack/hackcloud/net.sh +++ b/uncloud/hack/hackcloud/net.sh @@ -5,6 +5,7 @@ dev=wlp2s0 dev=wlp0s20f3 dev=wlan0 +ip=2a0a:e5c1:111:888::42/64 vxlandev=vxlan${netid} bridgedev=br${netid} @@ -19,3 +20,5 @@ ip link set ${vxlandev} up ip link add ${bridgedev} type bridge ip link set ${bridgedev} up + +ip addr add ${ip} dev ${bridgedev} diff --git a/uncloud/hack/hackcloud/radvd.conf b/uncloud/hack/hackcloud/radvd.conf new file mode 100644 index 0000000..3d8ce4d --- /dev/null +++ b/uncloud/hack/hackcloud/radvd.conf @@ -0,0 +1,13 @@ +interface br100 +{ + AdvSendAdvert on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 5; + AdvDefaultLifetime 3600; + + prefix 2a0a:e5c1:111:888::/64 { + }; + + RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; }; + DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ; +}; diff --git a/uncloud/hack/hackcloud/radvd.sh b/uncloud/hack/hackcloud/radvd.sh new file mode 100644 index 0000000..9d0e7d1 --- /dev/null +++ b/uncloud/hack/hackcloud/radvd.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +radvd -C ./radvd.conf -n -p ./radvdpid From 3825c7c210fefa9112a0efdeea3f8c2e20ad7bec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 00:23:55 +0100 Subject: [PATCH 17/75] Add vxlan into the bridge --- uncloud/hack/hackcloud/net.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh index 7d4b88f..0a96fe1 100755 --- a/uncloud/hack/hackcloud/net.sh +++ b/uncloud/hack/hackcloud/net.sh @@ -18,7 +18,10 @@ ip -6 link add ${vxlandev} type vxlan \ ip link set ${vxlandev} up + ip link add ${bridgedev} type bridge ip link set ${bridgedev} up +ip link set ${vxlandev} master ${bridgedev} up + ip addr add ${ip} dev ${bridgedev} From 23d805f04fc8349d5e503414174819f7d26a1f78 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 00:24:17 +0100 Subject: [PATCH 18/75] ++stuff --- uncloud/hack/hackcloud/net.sh | 4 ++-- uncloud/hack/uncloud-run-vm | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh index 7d4b88f..4cb6498 100755 --- a/uncloud/hack/hackcloud/net.sh +++ b/uncloud/hack/hackcloud/net.sh @@ -3,9 +3,9 @@ netid=100 dev=wlp2s0 dev=wlp0s20f3 -dev=wlan0 +#dev=wlan0 -ip=2a0a:e5c1:111:888::42/64 +ip=2a0a:e5c1:111:888::48/64 vxlandev=vxlan${netid} bridgedev=br${netid} diff --git a/uncloud/hack/uncloud-run-vm b/uncloud/hack/uncloud-run-vm index 1af2037..33e5860 100644 --- a/uncloud/hack/uncloud-run-vm +++ b/uncloud/hack/uncloud-run-vm @@ -18,4 +18,8 @@ qemu-system-x86_64 \ -smp 2,sockets=2,cores=1,threads=1 \ -device virtio-net-pci,netdev=net0,mac=$macaddress \ -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \ - -vnc [::]:5900 + -vnc [::]:0 + +# To be changed: +# -vnc to unix path +# or -spice From c1cabb7220e208ed2ba976c695f786a23edac185 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 02:42:04 +0100 Subject: [PATCH 19/75] add working nft --- uncloud/hack/hackcloud/nftrules | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 uncloud/hack/hackcloud/nftrules diff --git a/uncloud/hack/hackcloud/nftrules b/uncloud/hack/hackcloud/nftrules new file mode 100644 index 0000000..661d91f --- /dev/null +++ b/uncloud/hack/hackcloud/nftrules @@ -0,0 +1,32 @@ +flush ruleset + +table bridge filter { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; + ibrname br100 jump netpublic + } + chain netpublic { + + iifname tap1 jump vm1 + + icmpv6 type {nd-router-solicit, nd-router-advert, + nd-neighbor-solicit, nd-neighbor-advert, nd-redirect } log + + } + chain vm1 { + ether saddr != 02:00:f0:a9:c4:4e drop + } +} + +table ip6 filter { + chain forward { + type filter hook forward priority 0; + + # policy drop; + + ct state established,related accept; + + } + +} From 029ef36d62d0fd4767cbfc9da7a721f3973f1bd2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 15:54:19 +0100 Subject: [PATCH 20/75] net +debug --- uncloud/hack/hackcloud/net.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncloud/hack/hackcloud/net.sh b/uncloud/hack/hackcloud/net.sh index 5c7a6f5..4e2bfa1 100755 --- a/uncloud/hack/hackcloud/net.sh +++ b/uncloud/hack/hackcloud/net.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -x + netid=100 dev=wlp2s0 dev=wlp0s20f3 From 3b68a589d47f40b8d6a0e4020e2a4e0254741424 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 16:17:35 +0100 Subject: [PATCH 21/75] cleanup vm.sh --- uncloud/hack/hackcloud/vm.sh | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index 2a8b794..2cc0480 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -21,28 +21,3 @@ $qemu -name uc${vmid} \ -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ -netdev tap,id=netmain,script=./ifup.sh \ -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e - - - -exit 0 - --S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-17-one-24992/master-key.aes --machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off - --m 2048 --realtime mlock=off --smp 1,sockets=1,cores=1,threads=1 --uuid 732e08c7-84f8-4d43-9571-263db4f80080 -no-user-config \ - -nodefaults --chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-17-one-24992/monitor.sock,server,nowait --mon chardev=charmonitor,id=monitor,mode=control --rtc base=utc -no-shutdown --boot strict=on --device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 --drive file=rbd:ssd/one-292-24992-0:id=libvirt:auth_supported=cephx\;none:mon_host=ceph1\:6789\;ceph2\:6789\;ceph3\:6789,format=raw,if=none,id=drive-virtio-disk0,cache=none --device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1 --drive file=/var/lib/one//datastores/104/24992/disk.1,format=raw,if=none,id=drive-ide0-0-0,readonly=on --device ide-cd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0 --netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38 --device virtio-net-pci,netdev=hostnet0,id=net0,mac=02:00:f0:a9:c4:4e,bus=pci.0,addr=0x3 --vnc [::]:4414 -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on From 708e3ebb971745a3ea8e6ad81179365829bc7398 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 16:20:29 +0100 Subject: [PATCH 22/75] cleanup ifup.sh --- uncloud/hack/hackcloud/ifup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/uncloud/hack/hackcloud/ifup.sh b/uncloud/hack/hackcloud/ifup.sh index 99e8690..e0a3ca0 100755 --- a/uncloud/hack/hackcloud/ifup.sh +++ b/uncloud/hack/hackcloud/ifup.sh @@ -1,7 +1,5 @@ #!/bin/sh -echo $@ >> foo - dev=$1; shift # bridge is setup from outside From 8544df8bad808910f31eaa85bf0808f056f09512 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 16:36:41 +0100 Subject: [PATCH 23/75] don't use tcg --- uncloud/hack/hackcloud/vm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index 2cc0480..dfef8cc 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -5,7 +5,7 @@ vmid=$1; shift qemu=/usr/bin/qemu-system-x86_64 accel=kvm -accel=tcg +#accel=tcg memory=1024 cores=2 From c6b7152464f8a68fc2e9e7383c5044fef3a84005 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2020 21:21:30 +0100 Subject: [PATCH 24/75] update nftrules example --- uncloud/hack/hackcloud/nftrules | 72 ++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/uncloud/hack/hackcloud/nftrules b/uncloud/hack/hackcloud/nftrules index 661d91f..035b3a8 100644 --- a/uncloud/hack/hackcloud/nftrules +++ b/uncloud/hack/hackcloud/nftrules @@ -1,32 +1,64 @@ flush ruleset table bridge filter { - chain prerouting { - type filter hook prerouting priority 0; - policy accept; - ibrname br100 jump netpublic - } - chain netpublic { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; - iifname tap1 jump vm1 + ibrname br100 jump netpublic + } - icmpv6 type {nd-router-solicit, nd-router-advert, - nd-neighbor-solicit, nd-neighbor-advert, nd-redirect } log + chain netpublic { + iifname vxlan100 jump from_uncloud - } - chain vm1 { - ether saddr != 02:00:f0:a9:c4:4e drop - } + # Default blocks: router advertisements, dhcpv6, dhcpv4 + icmpv6 type nd-router-advert drop + ip6 version 6 udp sport 547 drop + ip version 4 udp sport 67 drop + + # Individual blocks + iifname tap1 jump vm1 + } + + chain vm1 { + ether saddr != 02:00:f0:a9:c4:4e drop + ip6 saddr != 2a0a:e5c1:111:888:0:f0ff:fea9:c44e drop + } + + chain from_uncloud { + accept + } } -table ip6 filter { - chain forward { - type filter hook forward priority 0; +# table ip6 filter { +# chain forward { +# type filter hook forward priority 0; - # policy drop; +# # policy drop; - ct state established,related accept; +# ct state established,related accept; - } +# } -} +# } + +# table ip filter { +# chain input { +# type filter hook input priority filter; policy drop; +# iif "lo" accept +# icmp type { echo-reply, destination-unreachable, source-quench, redirect, echo-request, router-advertisement, router-solicitation, time-exceeded, parameter-problem, timestamp-request, timestamp-reply, info-request, info-reply, address-mask-request, address-mask-reply } accept +# ct state established,related accept +# tcp dport { 22 } accept +# log prefix "firewall-ipv4: " +# udp sport 67 drop +# } + +# chain forward { +# type filter hook forward priority filter; policy drop; +# log prefix "firewall-ipv4: " +# } + +# chain output { +# type filter hook output priority filter; policy accept; +# } +# } From 6d51e2a8c4cfe5bb6695e390fd55414657beb8bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 00:32:17 +0100 Subject: [PATCH 25/75] [metadata] change default port to 1234 --- uncloud/metadata/main.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index ccda60e..c47364e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -13,8 +13,10 @@ api = Api(app) app.logger.handlers.clear() +DEFAULT_PORT=1234 + arg_parser = argparse.ArgumentParser('metadata', add_help=False) -arg_parser.add_argument('--port', '-p', default=80, help='By default bind to port 80') +arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT)) @app.errorhandler(Exception) From aaf29adcbb6c1aea64952099d14c1aaceb644e43 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 00:41:31 +0100 Subject: [PATCH 26/75] + mac prefix --- uncloud/hack/hackcloud/mac-prefix | 1 + 1 file changed, 1 insertion(+) create mode 100644 uncloud/hack/hackcloud/mac-prefix diff --git a/uncloud/hack/hackcloud/mac-prefix b/uncloud/hack/hackcloud/mac-prefix new file mode 100644 index 0000000..5084a2f --- /dev/null +++ b/uncloud/hack/hackcloud/mac-prefix @@ -0,0 +1 @@ +02:00 From b017df4879a4daf8e1f4542fd9f7fcf0aab7fc40 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 13:20:38 +0100 Subject: [PATCH 27/75] ignore iso, update nft rules --- .gitignore | 2 ++ uncloud/hack/hackcloud/nftrules | 57 +++++++-------------------------- 2 files changed, 14 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 5c55899..6f0d9df 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ uncloud/version.py build/ venv/ dist/ + +*.iso diff --git a/uncloud/hack/hackcloud/nftrules b/uncloud/hack/hackcloud/nftrules index 035b3a8..636c63d 100644 --- a/uncloud/hack/hackcloud/nftrules +++ b/uncloud/hack/hackcloud/nftrules @@ -5,60 +5,27 @@ table bridge filter { type filter hook prerouting priority 0; policy accept; - ibrname br100 jump netpublic + ibrname br100 jump br100 } - chain netpublic { - iifname vxlan100 jump from_uncloud + chain br100 { + # Allow all incoming traffic from outside + iifname vxlan100 accept # Default blocks: router advertisements, dhcpv6, dhcpv4 icmpv6 type nd-router-advert drop ip6 version 6 udp sport 547 drop ip version 4 udp sport 67 drop - # Individual blocks - iifname tap1 jump vm1 + jump br100_vmlist + drop } + chain br100_vmlist { + # VM1 + iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept - chain vm1 { - ether saddr != 02:00:f0:a9:c4:4e drop - ip6 saddr != 2a0a:e5c1:111:888:0:f0ff:fea9:c44e drop - } - - chain from_uncloud { - accept + # VM2 + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept } } - -# table ip6 filter { -# chain forward { -# type filter hook forward priority 0; - -# # policy drop; - -# ct state established,related accept; - -# } - -# } - -# table ip filter { -# chain input { -# type filter hook input priority filter; policy drop; -# iif "lo" accept -# icmp type { echo-reply, destination-unreachable, source-quench, redirect, echo-request, router-advertisement, router-solicitation, time-exceeded, parameter-problem, timestamp-request, timestamp-reply, info-request, info-reply, address-mask-request, address-mask-reply } accept -# ct state established,related accept -# tcp dport { 22 } accept -# log prefix "firewall-ipv4: " -# udp sport 67 drop -# } - -# chain forward { -# type filter hook forward priority filter; policy drop; -# log prefix "firewall-ipv4: " -# } - -# chain output { -# type filter hook output priority filter; policy accept; -# } -# } From 64ab011299fa230399cba5c401962974a4b6c069 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 13:41:54 +0100 Subject: [PATCH 28/75] import mac.py from cinv --- uncloud/hack/hackcloud/mac-gen.py | 171 ++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 uncloud/hack/hackcloud/mac-gen.py diff --git a/uncloud/hack/hackcloud/mac-gen.py b/uncloud/hack/hackcloud/mac-gen.py new file mode 100644 index 0000000..9f23854 --- /dev/null +++ b/uncloud/hack/hackcloud/mac-gen.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2012 Nico Schottelius (nico-cinv at schottelius.org) +# +# This file is part of cinv. +# +# cinv is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cinv is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cinv. If not, see . +# +# + +import argparse +import logging +import os.path +import os +import re + +import cinv +from cinv import fsproperty + +log = logging.getLogger(__name__) + +class Error(cinv.Error): + pass + + +class Mac(object): + + def __init__(self): + self.base_dir = self.get_base_dir() + + _prefix = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_dir, "prefix")) + free = fsproperty.FileListProperty(lambda obj: os.path.join(obj.base_dir, "free")) + last = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_dir, "last")) + + def _init_base_dir(self): + try: + os.makedirs(self.base_dir, exist_ok=True) + except OSError as e: + raise Error(e) + + @staticmethod + def validate_mac(mac): + if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): + raise Error("Not a valid mac address: %s" % mac) + + def free_append(self, mac): + if mac in self.free: + raise Error("Mac already in free database: %s" % mac) + + self._init_base_dir() + self.free.append(mac) + + @staticmethod + def get_base_dir(): + return cinv.get_base_dir("db/mac") + + @classmethod + def exists(cls): + return os.path.exists(cls.get_base_dir()) + + def get_next(self): + self._init_base_dir() + + if self.free: + return self.free.pop() + + if not self.prefix: + raise Error("Cannot generate address without prefix - use prefix-set") + + if self.last: + suffix = re.search(r'([0-9A-F]{2}[-:]){2}[0-9A-F]{2}$', self.last, re.I) + last_number_hex = "0x%s" % suffix.group().replace(":", "") + last_number = int(last_number_hex, 16) + + if last_number == int('0xffffff', 16): + raise Error("Exhausted all possible mac addresses - try to free some") + + next_number = last_number + 1 + else: + next_number = 0 + + next_number_hex = "%0.6x" % next_number + next_suffix = "%s:%s:%s" % (next_number_hex[0:2], next_number_hex[2:4], next_number_hex[4:6]) + + next_mac = "%s:%s" % (self.prefix, next_suffix) + + self.last = next_mac + + return next_mac + + + @property + def prefix(self): + return self._prefix + + @prefix.setter + def prefix(self, prefix): + if not re.match(r'([0-9A-F]{2}[-:]){2}[0-9A-F]{2}$', prefix, re.I): + raise Error("Wrong mac address format - use 00:11:22") + + self._init_base_dir() + self._prefix = prefix + + @classmethod + def commandline_generate(cls, args): + mac = Mac() + print(mac.get_next()) + + @classmethod + def commandline_free_add(cls, args): + mac = Mac() + mac.validate_mac(args.address) + mac.free_append(args.address) + + @classmethod + def commandline_free_list(cls, args): + mac = Mac() + for mac in mac.free: + print(mac) + + @classmethod + def commandline_prefix_set(cls, args): + mac = Mac() + mac.prefix = args.prefix + + @classmethod + def commandline_prefix_get(cls, args): + mac = cls() + print(mac.prefix) + + @classmethod + def commandline_add(cls, args): + host = cls(fqdn=args.fqdn) + host.host_type = args.type + + @classmethod + def commandline_args(cls, parent_parser, parents): + """Add us to the parent parser and add all parents to our parsers""" + + parser = {} + parser['sub'] = parent_parser.add_subparsers(title="Mac Commands") + + parser['free-add'] = parser['sub'].add_parser('free-add', parents=parents) + parser['free-add'].add_argument('address', help='Address to add to free database') + parser['free-add'].set_defaults(func=cls.commandline_free_add) + + parser['free-list'] = parser['sub'].add_parser('free-list', parents=parents, + help="List free mac addresses") + parser['free-list'].set_defaults(func=cls.commandline_free_list) + + parser['generate'] = parser['sub'].add_parser('generate', parents=parents) + parser['generate'].set_defaults(func=cls.commandline_generate) + + parser['prefix-get'] = parser['sub'].add_parser('prefix-get', parents=parents) + parser['prefix-get'].set_defaults(func=cls.commandline_prefix_get) + + parser['prefix-set'] = parser['sub'].add_parser('prefix-set', parents=parents) + parser['prefix-set'].add_argument('prefix', help="3 Byte address prefix (f.i. '00:16:3e')") + parser['prefix-set'].set_defaults(func=cls.commandline_prefix_set) From 53c6a14d608c5c7d564a14d4ca31eac9c413c930 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:03:04 +0100 Subject: [PATCH 29/75] mac: begin to downstrip --- uncloud/hack/hackcloud/mac-gen.py | 56 ++++++++++++------------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/uncloud/hack/hackcloud/mac-gen.py b/uncloud/hack/hackcloud/mac-gen.py index 9f23854..f2a5db0 100644 --- a/uncloud/hack/hackcloud/mac-gen.py +++ b/uncloud/hack/hackcloud/mac-gen.py @@ -26,29 +26,32 @@ import os.path import os import re -import cinv -from cinv import fsproperty - log = logging.getLogger(__name__) -class Error(cinv.Error): +class Error(Exception): pass class Mac(object): - def __init__(self): - self.base_dir = self.get_base_dir() + self.base_dir = "." - _prefix = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_dir, "prefix")) - free = fsproperty.FileListProperty(lambda obj: os.path.join(obj.base_dir, "free")) - last = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_dir, "last")) + self._prefix = "02:00" + + self.free = self.read_file("mac-free") + self.last = self.read_file("mac-last") + + def read_file(self, filename): + fname = os.path.join(self.base_dir, filename) + ret = [] - def _init_base_dir(self): try: - os.makedirs(self.base_dir, exist_ok=True) - except OSError as e: - raise Error(e) + with open(fname, "r") as fd: + ret = fd.readlines() + except Exception as e: + pass + + return ret @staticmethod def validate_mac(mac): @@ -146,26 +149,11 @@ class Mac(object): host.host_type = args.type @classmethod - def commandline_args(cls, parent_parser, parents): - """Add us to the parent parser and add all parents to our parsers""" + def commandline(cls): + pass - parser = {} - parser['sub'] = parent_parser.add_subparsers(title="Mac Commands") - parser['free-add'] = parser['sub'].add_parser('free-add', parents=parents) - parser['free-add'].add_argument('address', help='Address to add to free database') - parser['free-add'].set_defaults(func=cls.commandline_free_add) - - parser['free-list'] = parser['sub'].add_parser('free-list', parents=parents, - help="List free mac addresses") - parser['free-list'].set_defaults(func=cls.commandline_free_list) - - parser['generate'] = parser['sub'].add_parser('generate', parents=parents) - parser['generate'].set_defaults(func=cls.commandline_generate) - - parser['prefix-get'] = parser['sub'].add_parser('prefix-get', parents=parents) - parser['prefix-get'].set_defaults(func=cls.commandline_prefix_get) - - parser['prefix-set'] = parser['sub'].add_parser('prefix-set', parents=parents) - parser['prefix-set'].add_argument('prefix', help="3 Byte address prefix (f.i. '00:16:3e')") - parser['prefix-set'].set_defaults(func=cls.commandline_prefix_set) +if __name__ == '__main__': + m = Mac() + m.commandline() + print(m.free) From 94dad7c9b6ae8bd5e5bf4b035b582ce7d4a44a01 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:35:59 +0100 Subject: [PATCH 30/75] Add script to generate mac addresses --- uncloud/hack/hackcloud/mac-gen.py | 96 +++++++++---------------------- uncloud/hack/hackcloud/mac-last | 1 + 2 files changed, 28 insertions(+), 69 deletions(-) mode change 100644 => 100755 uncloud/hack/hackcloud/mac-gen.py create mode 100644 uncloud/hack/hackcloud/mac-last diff --git a/uncloud/hack/hackcloud/mac-gen.py b/uncloud/hack/hackcloud/mac-gen.py old mode 100644 new mode 100755 index f2a5db0..e2b4bc5 --- a/uncloud/hack/hackcloud/mac-gen.py +++ b/uncloud/hack/hackcloud/mac-gen.py @@ -36,7 +36,8 @@ class Mac(object): def __init__(self): self.base_dir = "." - self._prefix = "02:00" + self.prefix = 0x002000000000 + #self.prefix = "{:012x}".format(self._prefix) self.free = self.read_file("mac-free") self.last = self.read_file("mac-last") @@ -53,6 +54,11 @@ class Mac(object): return ret + def append_to_file(self, text, filename): + fname = os.path.join(self.base_dir, filename) + with open(fname, "a+") as fd: + fd.write("{}\n".format(text)) + @staticmethod def validate_mac(mac): if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): @@ -62,30 +68,24 @@ class Mac(object): if mac in self.free: raise Error("Mac already in free database: %s" % mac) - self._init_base_dir() - self.free.append(mac) + self.append_to_file(mac, "mac-free") + self.free = self.read_file("mac-free") + @staticmethod - def get_base_dir(): - return cinv.get_base_dir("db/mac") + def int_to_mac(number): + b = number.to_bytes(6, byteorder="big") + return ':'.join(format(s, '02x') for s in b) - @classmethod - def exists(cls): - return os.path.exists(cls.get_base_dir()) + def getnext(self): +# if self.free: +# return self.free.pop() - def get_next(self): - self._init_base_dir() - - if self.free: - return self.free.pop() - - if not self.prefix: - raise Error("Cannot generate address without prefix - use prefix-set") +# if not self.prefix: +# raise Error("Cannot generate address without prefix - use prefix-set") if self.last: - suffix = re.search(r'([0-9A-F]{2}[-:]){2}[0-9A-F]{2}$', self.last, re.I) - last_number_hex = "0x%s" % suffix.group().replace(":", "") - last_number = int(last_number_hex, 16) + last_number = int(self.last[0], 16) if last_number == int('0xffffff', 16): raise Error("Exhausted all possible mac addresses - try to free some") @@ -94,60 +94,16 @@ class Mac(object): else: next_number = 0 - next_number_hex = "%0.6x" % next_number - next_suffix = "%s:%s:%s" % (next_number_hex[0:2], next_number_hex[2:4], next_number_hex[4:6]) + next_number_string = "{:012x}".format(next_number) - next_mac = "%s:%s" % (self.prefix, next_suffix) + next_mac_number = self.prefix + next_number + next_mac = self.int_to_mac(next_mac_number) - self.last = next_mac + with open(os.path.join(self.base_dir, "mac-last"), "w+") as fd: + fd.write("{}\n".format(next_number_string)) return next_mac - - @property - def prefix(self): - return self._prefix - - @prefix.setter - def prefix(self, prefix): - if not re.match(r'([0-9A-F]{2}[-:]){2}[0-9A-F]{2}$', prefix, re.I): - raise Error("Wrong mac address format - use 00:11:22") - - self._init_base_dir() - self._prefix = prefix - - @classmethod - def commandline_generate(cls, args): - mac = Mac() - print(mac.get_next()) - - @classmethod - def commandline_free_add(cls, args): - mac = Mac() - mac.validate_mac(args.address) - mac.free_append(args.address) - - @classmethod - def commandline_free_list(cls, args): - mac = Mac() - for mac in mac.free: - print(mac) - - @classmethod - def commandline_prefix_set(cls, args): - mac = Mac() - mac.prefix = args.prefix - - @classmethod - def commandline_prefix_get(cls, args): - mac = cls() - print(mac.prefix) - - @classmethod - def commandline_add(cls, args): - host = cls(fqdn=args.fqdn) - host.host_type = args.type - @classmethod def commandline(cls): pass @@ -156,4 +112,6 @@ class Mac(object): if __name__ == '__main__': m = Mac() m.commandline() - print(m.free) + # print(m.free) + #print(m.last) + print(m.getnext()) diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last new file mode 100644 index 0000000..df32b47 --- /dev/null +++ b/uncloud/hack/hackcloud/mac-last @@ -0,0 +1 @@ +000000000006 From 3188787c2a9f4ad9bb7c4b0f66818d21ea5d8579 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:38:01 +0100 Subject: [PATCH 31/75] ++mac change --- uncloud/hack/hackcloud/mac-last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last index df32b47..90a4264 100644 --- a/uncloud/hack/hackcloud/mac-last +++ b/uncloud/hack/hackcloud/mac-last @@ -1 +1 @@ -000000000006 +000000000245 From 02526baaf979783e3f1ed661f8f16a256b3b9f5a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:43:06 +0100 Subject: [PATCH 32/75] add ifdown support --- uncloud/hack/hackcloud/ifdown.sh | 3 +++ uncloud/hack/hackcloud/vm.sh | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 uncloud/hack/hackcloud/ifdown.sh diff --git a/uncloud/hack/hackcloud/ifdown.sh b/uncloud/hack/hackcloud/ifdown.sh new file mode 100644 index 0000000..70fe1db --- /dev/null +++ b/uncloud/hack/hackcloud/ifdown.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo $@! diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index dfef8cc..56956ea 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -1,7 +1,5 @@ #!/bin/sh -vmid=$1; shift - qemu=/usr/bin/qemu-system-x86_64 accel=kvm @@ -9,15 +7,16 @@ accel=kvm memory=1024 cores=2 -uuid=732e08c7-84f8-4d43-9571-263db4f80080 +uuid=$(uuidgen) +mac=$(./mac-gen.py) export bridge=br100 -$qemu -name uc${vmid} \ +$qemu -name "uncloud-!${uuid}" \ -machine pc,accel=${accel} \ -m ${memory} \ -smp ${cores} \ -uuid ${uuid} \ -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ - -netdev tap,id=netmain,script=./ifup.sh \ - -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e + -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac} From e6d22a73c5efbe9c91d316cf1c7ee576bd239e92 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 14:44:53 +0100 Subject: [PATCH 33/75] ++ cleanup --- uncloud/hack/hackcloud/ifdown.sh | 2 +- uncloud/hack/hackcloud/mac-last | 2 +- uncloud/hack/hackcloud/vm.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) mode change 100644 => 100755 uncloud/hack/hackcloud/ifdown.sh diff --git a/uncloud/hack/hackcloud/ifdown.sh b/uncloud/hack/hackcloud/ifdown.sh old mode 100644 new mode 100755 index 70fe1db..5753099 --- a/uncloud/hack/hackcloud/ifdown.sh +++ b/uncloud/hack/hackcloud/ifdown.sh @@ -1,3 +1,3 @@ #!/bin/sh -echo $@! +echo $@ diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last index 90a4264..59f6410 100644 --- a/uncloud/hack/hackcloud/mac-last +++ b/uncloud/hack/hackcloud/mac-last @@ -1 +1 @@ -000000000245 +000000000251 diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index 56956ea..a0e111b 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -12,7 +12,8 @@ mac=$(./mac-gen.py) export bridge=br100 -$qemu -name "uncloud-!${uuid}" \ +set -x +$qemu -name "uncloud-${uuid}" \ -machine pc,accel=${accel} \ -m ${memory} \ -smp ${cores} \ From c3b42aabc626f4cca1617dfcba9b52fbf587502a Mon Sep 17 00:00:00 2001 From: Ahmed Bilal <49-ahmedbilal@users.noreply.code.ungleich.ch> Date: Mon, 13 Jan 2020 05:57:41 +0100 Subject: [PATCH 34/75] Added --conf-dir, --etcd-{host,port,ca_cert,cert_cert,cert_key} parameters to cli and settings is now accessbile through uncloud.shared.shared.settings --- scripts/uncloud | 59 ++++++++++++++++++----------- uncloud/api/common_fields.py | 7 +--- uncloud/api/create_image_store.py | 3 +- uncloud/api/helper.py | 18 ++++----- uncloud/api/main.py | 61 +++++++++++++++--------------- uncloud/api/schemas.py | 13 +++---- uncloud/cli/helper.py | 29 ++++++-------- uncloud/common/cli.py | 26 +++++++++++++ uncloud/common/settings.py | 19 ++++++---- uncloud/common/shared.py | 22 +++++------ uncloud/common/storage_handlers.py | 15 ++++---- uncloud/configure/main.py | 11 +++--- uncloud/filescanner/main.py | 7 ++-- uncloud/host/main.py | 11 +++--- uncloud/host/virtualmachine.py | 11 +++--- uncloud/imagescanner/main.py | 7 ++-- uncloud/metadata/main.py | 3 +- uncloud/scheduler/helper.py | 3 +- uncloud/scheduler/main.py | 5 +-- 19 files changed, 176 insertions(+), 154 deletions(-) create mode 100644 uncloud/common/cli.py diff --git a/scripts/uncloud b/scripts/uncloud index a6e61aa..1a6483b 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -3,14 +3,13 @@ import logging import sys import importlib import argparse +import os +from etcd3.exceptions import ConnectionFailedError + +from uncloud.common import settings from uncloud import UncloudException - -# the components that use etcd -ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] - -ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('cli') +from uncloud.common.cli import resolve_otp_credentials def exception_hook(exc_type, exc_value, exc_traceback): @@ -22,6 +21,13 @@ def exception_hook(exc_type, exc_value, exc_traceback): sys.excepthook = exception_hook +# the components that use etcd +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('cli') + + if __name__ == '__main__': # Setting up root logger logger = logging.getLogger() @@ -31,15 +37,13 @@ if __name__ == '__main__': subparsers = arg_parser.add_subparsers(dest='command') parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument('--debug', '-d', - action='store_true', - default=False, + parent_parser.add_argument('--debug', '-d', action='store_true', default=False, help='More verbose logging') - parent_parser.add_argument('--conf-dir', '-c', - help='Configuration directory') + parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory', + default=os.path.expanduser('~/uncloud')) etcd_parser = argparse.ArgumentParser(add_help=False) - etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-host', dest='etcd_url') etcd_parser.add_argument('--etcd-port') etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') @@ -54,25 +58,36 @@ if __name__ == '__main__': else: subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - args = arg_parser.parse_args() - if not args.command: + arguments = vars(arg_parser.parse_args()) + etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value] + etcd_arguments = { + 'etcd': { + key.replace('etcd_', ''): arguments[key] + for key in etcd_arguments + } + } + if not arguments['command']: arg_parser.print_help() else: - arguments = vars(args) + # Initializing Settings and resolving otp_credentials + # It is neccessary to resolve_otp_credentials after argument parsing is done because + # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and + # providing the default values for --name, --realm and --seed arguments from the values + # we read from file. But, now we are asking user about where the config file lives. So, + # to providing default value is not possible before parsing arguments. So, we are doing + # it after.. + settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) + resolve_otp_credentials(arguments) + name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') - # If the component requires etcd3, we import it and catch the - # etcd3.exceptions.ConnectionFailedError - if name in ETCD_COMPONENTS: - import etcd3 - try: main(arguments) except UncloudException as err: logger.error(err) - sys.exit(1) + except ConnectionFailedError: + logger.error('Cannot connect to etcd') except Exception as err: logger.exception(err) - sys.exit(1) diff --git a/uncloud/api/common_fields.py b/uncloud/api/common_fields.py index d1fcb64..ba9fb37 100755 --- a/uncloud/api/common_fields.py +++ b/uncloud/api/common_fields.py @@ -1,7 +1,6 @@ import os from uncloud.common.shared import shared -from uncloud.common.settings import settings class Optional: @@ -54,9 +53,7 @@ class VmUUIDField(Field): def vm_uuid_validation(self): r = shared.etcd_client.get( - os.path.join(settings["etcd"]["vm_prefix"], self.uuid) + os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid) ) if not r: - self.add_error( - "VM with uuid {} does not exists".format(self.uuid) - ) + self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py index 1040e97..90e0f92 100755 --- a/uncloud/api/create_image_store.py +++ b/uncloud/api/create_image_store.py @@ -4,7 +4,6 @@ import os from uuid import uuid4 from uncloud.common.shared import shared -from uncloud.common.settings import settings data = { 'is_public': True, @@ -15,6 +14,6 @@ data = { } shared.etcd_client.put( - os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), + os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data), ) diff --git a/uncloud/api/helper.py b/uncloud/api/helper.py index 0805280..8ceb3a6 100755 --- a/uncloud/api/helper.py +++ b/uncloud/api/helper.py @@ -7,7 +7,6 @@ import requests from pyotp import TOTP from uncloud.common.shared import shared -from uncloud.common.settings import settings logger = logging.getLogger(__name__) @@ -15,9 +14,9 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": settings["otp"]["auth_name"], - "auth_token": TOTP(settings["otp"]["auth_seed"]).now(), - "auth_realm": settings["otp"]["auth_realm"], + "auth_name": shared.settings["otp"]["auth_name"], + "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), + "auth_realm": shared.settings["otp"]["auth_realm"], "name": name, "realm": realm, "token": token, @@ -25,13 +24,13 @@ def check_otp(name, realm, token): except binascii.Error as err: logger.error( "Cannot compute OTP for seed: {}".format( - settings["otp"]["auth_seed"] + shared.settings["otp"]["auth_seed"] ) ) return 400 response = requests.post( - settings["otp"]["verification_controller_url"], json=data + shared.settings["otp"]["verification_controller_url"], json=data ) return response.status_code @@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client): ) images = etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) # Try to find image with name == image_name and store_name == store_name @@ -111,9 +110,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix): lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) prefix = ipaddress.IPv6Address(prefix) return str(prefix + int(lower_part)) + diff --git a/uncloud/api/main.py b/uncloud/api/main.py index 34e1dd1..73e8e21 100644 --- a/uncloud/api/main.py +++ b/uncloud/api/main.py @@ -15,9 +15,8 @@ from uncloud.common.shared import shared from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType -from uncloud.common.settings import settings -from . import schemas -from .helper import generate_mac, mac2ipv6 +from uncloud.api import schemas +from uncloud.api.helper import generate_mac, mac2ipv6 from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -50,7 +49,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) + vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) specs = { 'cpu': validator.specs['cpu'], 'ram': validator.specs['ram'], @@ -60,7 +59,7 @@ class CreateVM(Resource): macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['tap_counter'] + shared.etcd_client, shared.settings['etcd']['tap_counter'] ) for _ in range(len(data['network'])) ] @@ -84,7 +83,7 @@ class CreateVM(Resource): r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) @@ -99,7 +98,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) vm_value = vm.value.copy() vm_value['ip'] = [] @@ -107,7 +106,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( join_path( - settings['etcd']['network_prefix'], + shared.settings['etcd']['network_prefix'], data['name'], network_name, ), @@ -130,7 +129,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = shared.etcd_client.get( - join_path(settings['etcd']['file_prefix'], data['uuid']) + join_path(shared.settings['etcd']['file_prefix'], data['uuid']) ) file_entry_value = json.loads(file_entry.value) @@ -144,7 +143,7 @@ class CreateImage(Resource): } shared.etcd_client.put( join_path( - settings['etcd']['image_prefix'], data['uuid'] + shared.settings['etcd']['image_prefix'], data['uuid'] ), json.dumps(image_entry_json), ) @@ -157,7 +156,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = shared.etcd_client.get_prefix( - settings['etcd']['image_prefix'], value_in_json=True + shared.settings['etcd']['image_prefix'], value_in_json=True ) r = {'images': []} for image in images: @@ -178,7 +177,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) action = data['action'] @@ -208,7 +207,7 @@ class VMAction(Resource): type='{}VM'.format(action.title()), uuid=data['uuid'], hostname=vm_entry.hostname, - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( @@ -231,10 +230,10 @@ class VMMigration(Resource): type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( - settings['etcd']['host_prefix'], + shared.settings['etcd']['host_prefix'], validator.destination.value, ), - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) @@ -254,7 +253,7 @@ class ListUserVM(Resource): if validator.is_valid(): vms = shared.etcd_client.get_prefix( - settings['etcd']['vm_prefix'], value_in_json=True + shared.settings['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter( @@ -287,7 +286,7 @@ class ListUserFiles(Resource): if validator.is_valid(): files = shared.etcd_client.get_prefix( - settings['etcd']['file_prefix'], value_in_json=True + shared.settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] user_files = [f for f in files if f.value['owner'] == data['name']] @@ -312,7 +311,7 @@ class CreateHost(Resource): validator = schemas.CreateHostSchema(data) if validator.is_valid(): host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': data['specs'], @@ -354,7 +353,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -372,7 +371,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -405,7 +404,7 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -439,7 +438,7 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -471,23 +470,23 @@ class CreateNetwork(Resource): network_entry = { 'id': counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['vxlan_counter'] + shared.etcd_client, shared.settings['etcd']['vxlan_counter'] ), 'type': data['type'], } if validator.user.value: try: nb = pynetbox.api( - url=settings['netbox']['url'], - token=settings['netbox']['token'], + url=shared.settings['netbox']['url'], + token=shared.settings['netbox']['token'], ) nb_prefix = nb.ipam.prefixes.get( - prefix=settings['network']['prefix'] + prefix=shared.settings['network']['prefix'] ) prefix = nb_prefix.available_prefixes.create( data={ 'prefix_length': int( - settings['network']['prefix_length'] + shared.settings['network']['prefix_length'] ), 'description': '{}\'s network "{}"'.format( data['name'], data['network_name'] @@ -506,7 +505,7 @@ class CreateNetwork(Resource): network_entry['ipv6'] = 'fd00::/64' network_key = join_path( - settings['etcd']['network_prefix'], + shared.settings['etcd']['network_prefix'], data['name'], data['network_name'], ) @@ -526,7 +525,7 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - settings['etcd']['network_prefix'], data['name'] + shared.settings['etcd']['network_prefix'], data['name'] ) networks = shared.etcd_client.get_prefix( prefix, value_in_json=True @@ -570,7 +569,7 @@ def main(arguments): try: image_stores = list( shared.etcd_client.get_prefix( - settings['etcd']['image_store_prefix'], value_in_json=True + shared.settings['etcd']['image_store_prefix'], value_in_json=True ) ) except KeyError: @@ -590,7 +589,7 @@ def main(arguments): # shared.etcd_client.put( # join_path( - # settings['etcd']['image_store_prefix'], uuid4().hex + # shared.settings['etcd']['image_store_prefix'], uuid4().hex # ), # json.dumps(data), # ) diff --git a/uncloud/api/schemas.py b/uncloud/api/schemas.py index e4de9a8..87f20c9 100755 --- a/uncloud/api/schemas.py +++ b/uncloud/api/schemas.py @@ -22,7 +22,6 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from uncloud.common.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema): def file_uuid_validation(self): file_entry = shared.etcd_client.get( os.path.join( - settings["etcd"]["file_prefix"], self.uuid.value + shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value ) ) if file_entry is None: @@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema): def image_store_name_validation(self): image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"] + shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] ) ) @@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema): for net in _network: network = shared.etcd_client.get( os.path.join( - settings["etcd"]["network_prefix"], + shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, net, ), @@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema): self.add_error("Can't migrate non-running VM") if vm.hostname == os.path.join( - settings["etcd"]["host_prefix"], self.destination.value + shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value ): self.add_error( "Destination host couldn't be same as Source Host" @@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - print(self.name.value, self.network_name.value) - key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) - print(key) + key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) network = shared.etcd_client.get(key, value_in_json=True) if network: self.add_error( diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py index 3c63073..51a4355 100644 --- a/uncloud/cli/helper.py +++ b/uncloud/cli/helper.py @@ -5,23 +5,14 @@ import binascii from pyotp import TOTP from os.path import join as join_path -from uncloud.common.settings import settings +from uncloud.common.shared import shared def get_otp_parser(): otp_parser = argparse.ArgumentParser('otp') - try: - name = settings['client']['name'] - realm = settings['client']['realm'] - seed = settings['client']['seed'] - except Exception: - otp_parser.add_argument('--name', required=True) - otp_parser.add_argument('--realm', required=True) - otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED') - else: - otp_parser.add_argument('--name', default=name) - otp_parser.add_argument('--realm', default=realm) - otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED') + otp_parser.add_argument('--name') + otp_parser.add_argument('--realm') + otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') return otp_parser @@ -34,11 +25,15 @@ def load_dump_pretty(content): def make_request(*args, data=None, request_method=requests.post): - r = request_method(join_path(settings['client']['api_server'], *args), json=data) try: - print(load_dump_pretty(r.content)) - except Exception: - print('Error occurred while getting output from api server.') + r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) + except requests.exceptions.RequestException: + print('Error occurred while connecting to API server.') + else: + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') def get_token(seed): diff --git a/uncloud/common/cli.py b/uncloud/common/cli.py new file mode 100644 index 0000000..3d3c248 --- /dev/null +++ b/uncloud/common/cli.py @@ -0,0 +1,26 @@ +from uncloud.common.shared import shared +from pyotp import TOTP + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except Exception: + raise Exception('Invalid seed') + else: + return token + + +def resolve_otp_credentials(kwargs): + d = { + 'name': shared.settings['client']['name'], + 'realm': shared.settings['client']['realm'], + 'token': get_token(shared.settings['client']['seed']) + } + + for k, v in d.items(): + if k in kwargs and kwargs[k] is None: + kwargs.update({k: v}) + + return d diff --git a/uncloud/common/settings.py b/uncloud/common/settings.py index 0d524a7..8503f42 100644 --- a/uncloud/common/settings.py +++ b/uncloud/common/settings.py @@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper from os.path import join as join_path logger = logging.getLogger(__name__) +settings = None class CustomConfigParser(configparser.RawConfigParser): @@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): - def __init__(self): + def __init__(self, conf_dir, seed_value=None): conf_name = 'uncloud.conf' - conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/')) self.config_file = join_path(conf_dir, conf_name) # this is used to cache config from etcd for 1 minutes. Without this we @@ -38,15 +38,19 @@ class Settings(object): self.config_parser.add_section('etcd') self.config_parser.set('etcd', 'base_prefix', '/') - try: + if os.access(self.config_file, os.R_OK): self.config_parser.read(self.config_file) - except Exception as err: - logger.error('%s', err) - + else: + raise FileNotFoundError('Config file %s not found!', self.config_file) self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') self.read_internal_values() + if seed_value is None: + seed_value = dict() + + self.config_parser.read_dict(seed_value) + def get_etcd_client(self): args = tuple() try: @@ -128,4 +132,5 @@ class Settings(object): return self.config_parser[key] -settings = Settings() +def get_settings(): + return settings diff --git a/uncloud/common/shared.py b/uncloud/common/shared.py index 918dd0c..aea7cbc 100644 --- a/uncloud/common/shared.py +++ b/uncloud/common/shared.py @@ -1,34 +1,34 @@ -from uncloud.common.settings import settings +from uncloud.common.settings import get_settings from uncloud.common.vm import VmPool from uncloud.common.host import HostPool from uncloud.common.request import RequestPool -from uncloud.common.storage_handlers import get_storage_handler +import uncloud.common.storage_handlers as storage_handlers class Shared: + @property + def settings(self): + return get_settings() + @property def etcd_client(self): - return settings.get_etcd_client() + return self.settings.get_etcd_client() @property def host_pool(self): - return HostPool( - self.etcd_client, settings["etcd"]["host_prefix"] - ) + return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) @property def vm_pool(self): - return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) + return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) @property def request_pool(self): - return RequestPool( - self.etcd_client, settings["etcd"]["request_prefix"] - ) + return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) @property def storage_handler(self): - return get_storage_handler() + return storage_handlers.get_storage_handler() shared = Shared() diff --git a/uncloud/common/storage_handlers.py b/uncloud/common/storage_handlers.py index 6f9b29e..58c2dc2 100644 --- a/uncloud/common/storage_handlers.py +++ b/uncloud/common/storage_handlers.py @@ -6,8 +6,7 @@ import stat from abc import ABC from . import logger from os.path import join as join_path - -from uncloud.common.settings import settings as config +import uncloud.common.shared as shared class ImageStorageHandler(ABC): @@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): def get_storage_handler(): - __storage_backend = config["storage"]["storage_backend"] + __storage_backend = shared.shared.settings["storage"]["storage_backend"] if __storage_backend == "filesystem": return FileSystemBasedImageStorageHandler( - vm_base=config["storage"]["vm_dir"], - image_base=config["storage"]["image_dir"], + vm_base=shared.shared.settings["storage"]["vm_dir"], + image_base=shared.shared.settings["storage"]["image_dir"], ) elif __storage_backend == "ceph": return CEPHBasedImageStorageHandler( - vm_base=config["storage"]["ceph_vm_pool"], - image_base=config["storage"]["ceph_image_pool"], + vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], + image_base=shared.shared.settings["storage"]["ceph_image_pool"], ) else: - raise Exception("Unknown Image Storage Handler") + raise Exception("Unknown Image Storage Handler") \ No newline at end of file diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py index e190460..87f5752 100644 --- a/uncloud/configure/main.py +++ b/uncloud/configure/main.py @@ -1,7 +1,6 @@ import os import argparse -from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('configure', add_help=False) @@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True) def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) + uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) + shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) -def main(**kwargs): - subcommand = kwargs.pop('subcommand') +def main(arguments): + subcommand = arguments['subcommand'] if not subcommand: arg_parser.print_help() else: - update_config(subcommand, kwargs) + update_config(subcommand, arguments) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index c5660dd..046f915 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -9,7 +9,6 @@ import bitmath from uuid import uuid4 from . import logger -from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('filescanner', add_help=False) @@ -53,7 +52,7 @@ def track_file(file, base_dir, host): file_path = file_path.relative_to(owner) creation_date = time.ctime(os.stat(file_str).st_ctime) - entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) + entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) entry_value = { 'filename': str(file_path), 'owner': owner, @@ -70,7 +69,7 @@ def track_file(file, base_dir, host): def main(arguments): hostname = arguments['hostname'] - base_dir = settings['storage']['file_dir'] + base_dir = shared.settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] @@ -78,7 +77,7 @@ def main(arguments): # Files that are already tracked tracked_files = [ pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) - for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) + for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) if f.value['host'] == hostname ] untracked_files = set(files) - set(tracked_files) diff --git a/uncloud/host/main.py b/uncloud/host/main.py index ccffd77..f680991 100755 --- a/uncloud/host/main.py +++ b/uncloud/host/main.py @@ -6,7 +6,6 @@ from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared -from uncloud.common.settings import settings from uncloud.common.vm import VMStatus from uncloud.vmm import VMM from os.path import join as join_path @@ -36,7 +35,7 @@ def maintenance(host): if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], vm_uuid) + join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) ) vm.status = VMStatus.running vm.vnc_socket = vmm.get_vnc(vm_uuid) @@ -52,7 +51,7 @@ def main(arguments): # Does not yet exist, create it if not host: host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': '', @@ -80,9 +79,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for events_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False) ]: for request_event in events_iterator: @@ -95,7 +94,7 @@ def main(arguments): shared.request_pool.client.client.delete(request_event.key) vm_entry = shared.etcd_client.get( - join_path(settings['etcd']['vm_prefix'], request_event.uuid) + join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) ) logger.debug('VM hostname: {}'.format(vm_entry.value)) diff --git a/uncloud/host/virtualmachine.py b/uncloud/host/virtualmachine.py index 2f6a5e3..a592efc 100755 --- a/uncloud/host/virtualmachine.py +++ b/uncloud/host/virtualmachine.py @@ -17,7 +17,6 @@ from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger from uncloud.common.shared import shared -from uncloud.common.settings import settings from uncloud.vmm import VMM from marshmallow import ValidationError @@ -91,7 +90,7 @@ class VM: self.vmm.socket_dir, self.uuid ), destination_host_key=destination_host_key, # Where source host transfer VM - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) else: @@ -119,7 +118,7 @@ class VM: network_name, mac, tap = network_mac_and_tap _key = os.path.join( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], self.vm["owner"], network_name, ) @@ -133,13 +132,13 @@ class VM: if network["type"] == "vxlan": tap = create_vxlan_br_tap( _id=network["id"], - _dev=settings["network"]["vxlan_phy_dev"], + _dev=shared.settings["network"]["vxlan_phy_dev"], tap_id=tap, ip=network["ipv6"], ) all_networks = shared.etcd_client.get_prefix( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], value_in_json=True, ) @@ -229,7 +228,7 @@ class VM: def resolve_network(network_name, network_owner): network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], network_owner, network_name, ), diff --git a/uncloud/imagescanner/main.py b/uncloud/imagescanner/main.py index 1803213..ee9da2e 100755 --- a/uncloud/imagescanner/main.py +++ b/uncloud/imagescanner/main.py @@ -4,7 +4,6 @@ import argparse import subprocess as sp from os.path import join as join_path -from uncloud.common.settings import settings from uncloud.common.shared import shared from uncloud.imagescanner import logger @@ -33,7 +32,7 @@ def qemu_img_type(path): def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) images_to_be_created = list( filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) @@ -46,13 +45,13 @@ def main(arguments): image_filename = image.value["filename"] image_store_name = image.value["store_name"] image_full_path = join_path( - settings["storage"]["file_dir"], + shared.settings["storage"]["file_dir"], image_owner, image_filename, ) image_stores = shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], + shared.settings["etcd"]["image_store_prefix"], value_in_json=True, ) user_image_store = next( diff --git a/uncloud/metadata/main.py b/uncloud/metadata/main.py index c47364e..374260e 100644 --- a/uncloud/metadata/main.py +++ b/uncloud/metadata/main.py @@ -5,7 +5,6 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from uncloud.common.settings import settings from uncloud.common.shared import shared app = Flask(__name__) @@ -74,7 +73,7 @@ class Root(Resource): ) else: etcd_key = os.path.join( - settings["etcd"]["user_prefix"], + shared.settings["etcd"]["user_prefix"], data.value["owner_realm"], data.value["owner"], "key", diff --git a/uncloud/scheduler/helper.py b/uncloud/scheduler/helper.py index 108d126..79db322 100755 --- a/uncloud/scheduler/helper.py +++ b/uncloud/scheduler/helper.py @@ -7,7 +7,6 @@ from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from uncloud.common.settings import settings def accumulated_specs(vms_specs): @@ -130,7 +129,7 @@ def assign_host(vm): type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py index c25700b..38c07bf 100755 --- a/uncloud/scheduler/main.py +++ b/uncloud/scheduler/main.py @@ -6,7 +6,6 @@ import argparse -from uncloud.common.settings import settings from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared from uncloud.scheduler import logger @@ -24,9 +23,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for request_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), ]: for request_event in request_iterator: From 091131d3509ecae41aedacc3788e7c166f623b99 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 13 Jan 2020 11:52:40 +0100 Subject: [PATCH 35/75] dummy --- uncloud/hack/hackcloud/mac-last | 2 +- uncloud/vmm/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/uncloud/hack/hackcloud/mac-last b/uncloud/hack/hackcloud/mac-last index 59f6410..8c5f254 100644 --- a/uncloud/hack/hackcloud/mac-last +++ b/uncloud/hack/hackcloud/mac-last @@ -1 +1 @@ -000000000251 +000000000252 diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 4c893f6..719bdbe 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -100,9 +100,9 @@ class TransferVM(Process): class VMM: # Virtual Machine Manager def __init__( - self, - qemu_path="/usr/bin/qemu-system-x86_64", - vmm_backend=os.path.expanduser("~/uncloud/vmm/"), + self, + qemu_path="/usr/bin/qemu-system-x86_64", + vmm_backend=os.path.expanduser("~/uncloud/vmm/"), ): self.qemu_path = qemu_path self.vmm_backend = vmm_backend From 10c8dc85ba58398203046c4c303689c8d3e45bd5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 13 Jan 2020 12:14:30 +0100 Subject: [PATCH 36/75] Begin hacky database handling --- uncloud/hack/hackcloud/db.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 uncloud/hack/hackcloud/db.py diff --git a/uncloud/hack/hackcloud/db.py b/uncloud/hack/hackcloud/db.py new file mode 100644 index 0000000..3d885e9 --- /dev/null +++ b/uncloud/hack/hackcloud/db.py @@ -0,0 +1,17 @@ +import etcd3 + + +if __name__ == '__main__': + endpoints = [ "https://etcd1.ungleich.ch:2379", + "!https://etcd2.ungleich.ch:2379", + "https://etcd3.ungleich.ch:2379" ] + + clients = [] + + for endpoint in endpoints: + client = etcd3.client(host=endpoint, + ca_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem", + cert_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem", + cert_key="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem") + + clients.append(client) From 9f02b31b1b2035cda0fb663781add044990d942b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 13 Jan 2020 12:54:02 +0100 Subject: [PATCH 37/75] Add hacky etcd client --- uncloud/hack/hackcloud/etcd-client.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 uncloud/hack/hackcloud/etcd-client.sh diff --git a/uncloud/hack/hackcloud/etcd-client.sh b/uncloud/hack/hackcloud/etcd-client.sh new file mode 100644 index 0000000..ab102a5 --- /dev/null +++ b/uncloud/hack/hackcloud/etcd-client.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \ + --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \ + --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \ + --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@" From b96e56b453bff53898bb105560881c88ffd63218 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 11:05:42 +0100 Subject: [PATCH 38/75] Begin to integrate hack into the main script --- scripts/uncloud | 3 +- uncloud/hack/__init__.py | 0 uncloud/hack/hackcloud/__init__.py | 1 + uncloud/hack/hackcloud/db.py | 55 ++++++++++++++++++++++++------ uncloud/hack/hackcloud/vm.py | 53 ++++++++++++++++++++++++++++ uncloud/hack/hackcloud/vm.sh | 6 ++++ uncloud/hack/main.py | 10 ++++++ 7 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 uncloud/hack/__init__.py create mode 100644 uncloud/hack/hackcloud/__init__.py create mode 100755 uncloud/hack/hackcloud/vm.py create mode 100644 uncloud/hack/main.py diff --git a/scripts/uncloud b/scripts/uncloud index 1a6483b..70cb535 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -22,7 +22,8 @@ def exception_hook(exc_type, exc_value, exc_traceback): sys.excepthook = exception_hook # the components that use etcd -ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', + 'imagescanner', 'metadata', 'configure', 'hack'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() ALL_COMPONENTS.append('cli') diff --git a/uncloud/hack/__init__.py b/uncloud/hack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/hack/hackcloud/__init__.py b/uncloud/hack/hackcloud/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/uncloud/hack/hackcloud/__init__.py @@ -0,0 +1 @@ + diff --git a/uncloud/hack/hackcloud/db.py b/uncloud/hack/hackcloud/db.py index 3d885e9..0e6bd0a 100644 --- a/uncloud/hack/hackcloud/db.py +++ b/uncloud/hack/hackcloud/db.py @@ -1,17 +1,52 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + import etcd3 +class DB(object): + def __init__(self, urls): + self.urls = urls + self.prefix = "/nicohack/" + + def connect(self): + self.clients = [] + for endpoint in self.urls: + client = etcd3.client(host=endpoint, + ca_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem", + cert_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem", + cert_key="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem") + + clients.append(client) + + def get_value(self, key): + pass + + def set_value(self, key, val): + pass + if __name__ == '__main__': endpoints = [ "https://etcd1.ungleich.ch:2379", - "!https://etcd2.ungleich.ch:2379", + "https://etcd2.ungleich.ch:2379", "https://etcd3.ungleich.ch:2379" ] - clients = [] - - for endpoint in endpoints: - client = etcd3.client(host=endpoint, - ca_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem", - cert_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem", - cert_key="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem") - - clients.append(client) + db = DB(url=endpoints) diff --git a/uncloud/hack/hackcloud/vm.py b/uncloud/hack/hackcloud/vm.py new file mode 100755 index 0000000..9dd80bf --- /dev/null +++ b/uncloud/hack/hackcloud/vm.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +import subprocess +import uuid + +from . import db + +qemu="/usr/bin/qemu-system-x86_64" +accel="kvm" +memory=1024 +cores=2 +uuid=uuid.uuid4() + +#mac=$(./mac-gen.py) +mac="" + +owner="nico" + +bridge="br100" + +if __name__ == '__main__': + p = ["qemu", + "-name", "uncloud-{}".format(uuid), + "-machine", "pc,accel={}".format(accel), + "-m", "{}".format(memory), + "-smp", "{}".format(cores), + "-uuid", "{}".format(uuid), + "-drive", "file=alpine-virt-3.11.2-x86_64.iso,media=cdrom", + "-netdev", "tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh", + "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(mac) + ] + print(" ".join(p)) + subprocess.run(p) diff --git a/uncloud/hack/hackcloud/vm.sh b/uncloud/hack/hackcloud/vm.sh index a0e111b..dd9be84 100755 --- a/uncloud/hack/hackcloud/vm.sh +++ b/uncloud/hack/hackcloud/vm.sh @@ -1,5 +1,10 @@ #!/bin/sh +# if [ $# -ne 1 ]; then +# echo "$0: owner" +# exit 1 +# fi + qemu=/usr/bin/qemu-system-x86_64 accel=kvm @@ -9,6 +14,7 @@ memory=1024 cores=2 uuid=$(uuidgen) mac=$(./mac-gen.py) +owner=nico export bridge=br100 diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py new file mode 100644 index 0000000..ce105e8 --- /dev/null +++ b/uncloud/hack/main.py @@ -0,0 +1,10 @@ +import argparse + +arg_parser = argparse.ArgumentParser('hack', add_help=False) +arg_parser.add_argument('--create-vm') + + +def main(arguments): + print(arguments)! + debug = arguments['debug'] + port = arguments['port'] From 22531a7459e1b1b3da7de9b765daddf2d483f5bd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 11:09:45 +0100 Subject: [PATCH 39/75] Disable cli / otp reading for the moment Imho this should clearly not leak into scripts/uncloud and additionally it is broken at the moment --- scripts/uncloud | 6 +++--- uncloud/hack/main.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index 70cb535..263d99e 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -26,7 +26,7 @@ ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure', 'hack'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('cli') +#ALL_COMPONENTS.append('cli') if __name__ == '__main__': @@ -77,8 +77,8 @@ if __name__ == '__main__': # we read from file. But, now we are asking user about where the config file lives. So, # to providing default value is not possible before parsing arguments. So, we are doing # it after.. - settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) - resolve_otp_credentials(arguments) +# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) +# resolve_otp_credentials(arguments) name = arguments.pop('command') mod = importlib.import_module('uncloud.{}.main'.format(name)) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index ce105e8..2ce19da 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -5,6 +5,6 @@ arg_parser.add_argument('--create-vm') def main(arguments): - print(arguments)! - debug = arguments['debug'] - port = arguments['port'] + print(arguments) + #debug = arguments['debug'] + #port = arguments['port'] From 083ba439183cbedb1baf30a5dfcc0f4da5e65d24 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 11:22:04 +0100 Subject: [PATCH 40/75] Integrate hack + vm create into python code --- uncloud/hack/hackcloud/vm.py | 53 --------------------------------- uncloud/hack/main.py | 9 +++++- uncloud/hack/vm.py | 57 ++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 54 deletions(-) delete mode 100755 uncloud/hack/hackcloud/vm.py create mode 100755 uncloud/hack/vm.py diff --git a/uncloud/hack/hackcloud/vm.py b/uncloud/hack/hackcloud/vm.py deleted file mode 100755 index 9dd80bf..0000000 --- a/uncloud/hack/hackcloud/vm.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) -# -# This file is part of uncloud. -# -# uncloud is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# uncloud is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with uncloud. If not, see . -# -# - -import subprocess -import uuid - -from . import db - -qemu="/usr/bin/qemu-system-x86_64" -accel="kvm" -memory=1024 -cores=2 -uuid=uuid.uuid4() - -#mac=$(./mac-gen.py) -mac="" - -owner="nico" - -bridge="br100" - -if __name__ == '__main__': - p = ["qemu", - "-name", "uncloud-{}".format(uuid), - "-machine", "pc,accel={}".format(accel), - "-m", "{}".format(memory), - "-smp", "{}".format(cores), - "-uuid", "{}".format(uuid), - "-drive", "file=alpine-virt-3.11.2-x86_64.iso,media=cdrom", - "-netdev", "tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh", - "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(mac) - ] - print(" ".join(p)) - subprocess.run(p) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 2ce19da..4baed98 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -1,10 +1,17 @@ import argparse +from uncloud.hack.vm import VM + arg_parser = argparse.ArgumentParser('hack', add_help=False) -arg_parser.add_argument('--create-vm') +arg_parser.add_argument('--create-vm', action='store_true') def main(arguments): print(arguments) + if arguments['create_vm']: + print("Creating VM") + vm = VM() + vm.create() + #debug = arguments['debug'] #port = arguments['port'] diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py new file mode 100755 index 0000000..988ea2b --- /dev/null +++ b/uncloud/hack/vm.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +import subprocess +import uuid +import os + +class VM(object): + def __init__(self): + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" + + self.qemu="/usr/bin/qemu-system-x86_64" + self.accel="kvm" + self.memory=1024 + self.cores=2 + self.uuid=uuid.uuid4() +# self.mac=$(./mac-gen.py) + self.mac="42:00:00:00:00:42" + self.owner="nico" + self.bridge="br100" + self.os_image = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + + def create(self): + p = [ "sudo", + "{}".format(self.qemu), + "-name", "uncloud-{}".format(self.uuid), + "-machine", "pc,accel={}".format(self.accel), + "-m", "{}".format(self.memory), + "-smp", "{}".format(self.cores), + "-uuid", "{}".format(self.uuid), + "-drive", "file={},media=cdrom".format(self.os_image), + "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), + "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) + ] + print(" ".join(p)) + subprocess.run(p) From c0e6d6a0d85dddecccd93f4f20b47b2a3c62f177 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 11:25:06 +0100 Subject: [PATCH 41/75] Begin further integration of code into hack --- uncloud/hack/{hackcloud => }/db.py | 0 uncloud/hack/{hackcloud/mac-gen.py => mac.py} | 0 uncloud/hack/main.py | 1 + 3 files changed, 1 insertion(+) rename uncloud/hack/{hackcloud => }/db.py (100%) rename uncloud/hack/{hackcloud/mac-gen.py => mac.py} (100%) diff --git a/uncloud/hack/hackcloud/db.py b/uncloud/hack/db.py similarity index 100% rename from uncloud/hack/hackcloud/db.py rename to uncloud/hack/db.py diff --git a/uncloud/hack/hackcloud/mac-gen.py b/uncloud/hack/mac.py similarity index 100% rename from uncloud/hack/hackcloud/mac-gen.py rename to uncloud/hack/mac.py diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 4baed98..2e1e9d5 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -3,6 +3,7 @@ import argparse from uncloud.hack.vm import VM arg_parser = argparse.ArgumentParser('hack', add_help=False) + #description="Commands that are unfinished - use at own risk") arg_parser.add_argument('--create-vm', action='store_true') From 1b36c2f96f945e317e5ef2cec2a5b00d6194ab35 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 14:23:26 +0100 Subject: [PATCH 42/75] Write VM to etcd --- scripts/uncloud | 6 +++--- uncloud/hack/config.py | 39 +++++++++++++++++++++++++++++++++++ uncloud/hack/db.py | 29 ++++++++++++++------------ uncloud/hack/main.py | 8 +++---- uncloud/hack/vm.py | 47 ++++++++++++++++++++++++++++-------------- 5 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 uncloud/hack/config.py diff --git a/scripts/uncloud b/scripts/uncloud index 263d99e..ab5b40d 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -44,7 +44,7 @@ if __name__ == '__main__': default=os.path.expanduser('~/uncloud')) etcd_parser = argparse.ArgumentParser(add_help=False) - etcd_parser.add_argument('--etcd-host', dest='etcd_url') + etcd_parser.add_argument('--etcd-host') etcd_parser.add_argument('--etcd-port') etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') @@ -88,7 +88,7 @@ if __name__ == '__main__': main(arguments) except UncloudException as err: logger.error(err) - except ConnectionFailedError: - logger.error('Cannot connect to etcd') +# except ConnectionFailedError as err: +# logger.error('Cannot connect to etcd: {}'.format(err)) except Exception as err: logger.exception(err) diff --git a/uncloud/hack/config.py b/uncloud/hack/config.py new file mode 100644 index 0000000..7e2655d --- /dev/null +++ b/uncloud/hack/config.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +class Config(object): + def __init__(self, arguments): + """ read arguments dicts as a base """ + + self.arguments = arguments + + # Split them so *etcd_args can be used and we can + # iterate over etcd_hosts + self.etcd_hosts = [ arguments['etcd_host'] ] + self.etcd_args = { + 'ca_cert': arguments['etcd_ca_cert'], + 'cert_cert': arguments['etcd_cert_cert'], + 'cert_key': arguments['etcd_cert_key'], +# 'user': None, +# 'password': None + } + self.etcd_prefix = '/nicohack/' diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py index 0e6bd0a..be0342a 100644 --- a/uncloud/hack/db.py +++ b/uncloud/hack/db.py @@ -21,28 +21,31 @@ # import etcd3 +import json class DB(object): - def __init__(self, urls): - self.urls = urls - self.prefix = "/nicohack/" + def __init__(self, config): + self.config = config + self.prefix= '/nicohack/' + self.connect() def connect(self): - self.clients = [] - for endpoint in self.urls: - client = etcd3.client(host=endpoint, - ca_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem", - cert_cert="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem", - cert_key="/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem") - - clients.append(client) + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) def get_value(self, key): pass - def set_value(self, key, val): - pass + def set(self, key, value, store_as_json=False, **kwargs): + if store_as_json: + value = json.dumps(value) + key = "{}/{}".format(self.prefix, key) + + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(key, value, **kwargs) if __name__ == '__main__': endpoints = [ "https://etcd1.ungleich.ch:2379", diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 2e1e9d5..df618c6 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -1,6 +1,7 @@ import argparse from uncloud.hack.vm import VM +from uncloud.hack.config import Config arg_parser = argparse.ArgumentParser('hack', add_help=False) #description="Commands that are unfinished - use at own risk") @@ -9,10 +10,9 @@ arg_parser.add_argument('--create-vm', action='store_true') def main(arguments): print(arguments) + config = Config(arguments) + if arguments['create_vm']: print("Creating VM") - vm = VM() + vm = VM(config) vm.create() - - #debug = arguments['debug'] - #port = arguments['port'] diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 988ea2b..e33e473 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -24,34 +24,49 @@ import subprocess import uuid import os -class VM(object): - def __init__(self): - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" +from uncloud.hack.db import DB +class VM(object): + def __init__(self, config): + self.config = config + self.db = DB(config) + + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" self.qemu="/usr/bin/qemu-system-x86_64" + + self.vm = {} + self.accel="kvm" - self.memory=1024 - self.cores=2 - self.uuid=uuid.uuid4() + # self.mac=$(./mac-gen.py) self.mac="42:00:00:00:00:42" self.owner="nico" self.bridge="br100" - self.os_image = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + self.ifup = os.path.join(self.hackprefix, "ifup.sh") self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") - def create(self): - p = [ "sudo", + self.uuid = uuid.uuid4() + self.vm['uuid'] = str(self.uuid) + self.vm['memory']=1024 + self.vm['cores']=2 + self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + + self.vm['commandline' ] = [ "sudo", "{}".format(self.qemu), - "-name", "uncloud-{}".format(self.uuid), + "-name", "uncloud-{}".format(self.vm['uuid']), "-machine", "pc,accel={}".format(self.accel), - "-m", "{}".format(self.memory), - "-smp", "{}".format(self.cores), - "-uuid", "{}".format(self.uuid), - "-drive", "file={},media=cdrom".format(self.os_image), + "-m", "{}".format(self.vm['memory']), + "-smp", "{}".format(self.vm['cores']), + "-uuid", "{}".format(self.vm['uuid']), + "-drive", "file={},media=cdrom".format(self.vm['os_image']), "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] - print(" ".join(p)) - subprocess.run(p) + + def create(self): + self.db.set("vm/{}".format(str(self.vm['uuid'])), + self.vm, store_as_json=True) + + print(" ".join(self.vm['commandline'])) + subprocess.run(self.vm['commandline']) From 8078ffae5a379f338c1e65f4acbb0832a73454f5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 19:02:15 +0100 Subject: [PATCH 43/75] Add working --last-used-mac {'create_vm': False, 'last_used_mac': True, 'get_new_mac': False, 'debug': False, 'conf_dir': '/home/nico/uncloud', 'etcd_host': 'etcd1.ungleich.ch', 'etcd_port': None, 'etcd_ca_cert': '/home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem', 'etcd_cert_cert': '/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem', 'etcd_cert_key': '/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem'} 00:20:00:00:00:00 (venv) [19:02] diamond:uncloud% ./bin/uncloud-run-reinstall hack --etcd-host etcd1.ungleich.ch --etcd-ca-cert /home/nico/vcs/ungleich-dot-cdist/files/etcd/ca.pem --etcd-cert-cert /home/nico/vcs/ungleich-dot-cdist/files/etcd/nico.pem --etcd-cert-key /home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem --last-used-mac --- uncloud/hack/db.py | 63 ++++++++++++++++++++++++++----- uncloud/hack/mac.py | 90 +++++++++++++++----------------------------- uncloud/hack/main.py | 11 ++++++ uncloud/hack/vm.py | 70 +++++++++++++++++----------------- 4 files changed, 132 insertions(+), 102 deletions(-) diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py index be0342a..ac643bd 100644 --- a/uncloud/hack/db.py +++ b/uncloud/hack/db.py @@ -22,30 +22,75 @@ import etcd3 import json +import logging + +from functools import wraps +from uncloud import UncloudException + +log = logging.getLogger(__name__) + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + class DB(object): - def __init__(self, config): + def __init__(self, config, prefix="/"): self.config = config - self.prefix= '/nicohack/' + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + self.connect() + @readable_errors def connect(self): self._db_clients = [] for endpoint in self.config.etcd_hosts: client = etcd3.client(host=endpoint, **self.config.etcd_args) self._db_clients.append(client) - def get_value(self, key): - pass + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) - def set(self, key, value, store_as_json=False, **kwargs): - if store_as_json: + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: value = json.dumps(value) - key = "{}/{}".format(self.prefix, key) - # FIXME: iterate over clients in case of failure ? - return self._db_clients[0].put(key, value, **kwargs) + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + @readable_errors + def increment(key, **kwargs): + with self._db_clients[0].lock(key) as lock: + value = int(self.get(self.realkey(key), **kwargs)) + self.set(self.realkey(key), str(value + 1), **kwargs) + if __name__ == '__main__': endpoints = [ "https://etcd1.ungleich.ch:2379", diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index e2b4bc5..4ac05f2 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -25,93 +25,65 @@ import logging import os.path import os import re +import json + +from uncloud import UncloudException +from uncloud.hack.db import DB log = logging.getLogger(__name__) -class Error(Exception): - pass - -class Mac(object): - def __init__(self): - self.base_dir = "." +class MAC(object): + def __init__(self, config): + self.config = config + self.db = DB(config, prefix="/mac") self.prefix = 0x002000000000 - #self.prefix = "{:012x}".format(self._prefix) - - self.free = self.read_file("mac-free") - self.last = self.read_file("mac-last") - - def read_file(self, filename): - fname = os.path.join(self.base_dir, filename) - ret = [] - - try: - with open(fname, "r") as fd: - ret = fd.readlines() - except Exception as e: - pass - - return ret - - def append_to_file(self, text, filename): - fname = os.path.join(self.base_dir, filename) - with open(fname, "a+") as fd: - fd.write("{}\n".format(text)) @staticmethod def validate_mac(mac): if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): raise Error("Not a valid mac address: %s" % mac) - def free_append(self, mac): - if mac in self.free: - raise Error("Mac already in free database: %s" % mac) - - self.append_to_file(mac, "mac-free") - self.free = self.read_file("mac-free") + def last_used_index(self): + value = self.db.get("last_used_index") + if not value: + return 0 + return int(value) + def last_used_mac(self): + return self.int_to_mac(self.prefix + self.last_used_index()) @staticmethod def int_to_mac(number): b = number.to_bytes(6, byteorder="big") return ':'.join(format(s, '02x') for s in b) - def getnext(self): + def get_next(self, vmuuid=None, as_int=False): # if self.free: # return self.free.pop() -# if not self.prefix: -# raise Error("Cannot generate address without prefix - use prefix-set") + last_number = self.last_used_index() - if self.last: - last_number = int(self.last[0], 16) - - if last_number == int('0xffffff', 16): - raise Error("Exhausted all possible mac addresses - try to free some") - - next_number = last_number + 1 - else: - next_number = 0 + # FIXME: compare to 48bit minus prefix length + if last_number == int('0xffffff', 16): + raise UncloudException("Exhausted all possible mac addresses - try to free some") + next_number = last_number + 1 next_number_string = "{:012x}".format(next_number) next_mac_number = self.prefix + next_number next_mac = self.int_to_mac(next_mac_number) - with open(os.path.join(self.base_dir, "mac-last"), "w+") as fd: - fd.write("{}\n".format(next_number_string)) + db_entry = {} + db_entry['vm_uuid'] = vmuuid + db_entry['index'] = next_number + db_entry['mac_address'] = next_mac - return next_mac + self.db.set("used/{}".format(next_mac), + db_entry) - @classmethod - def commandline(cls): - pass - - -if __name__ == '__main__': - m = Mac() - m.commandline() - # print(m.free) - #print(m.last) - print(m.getnext()) + if as_int: + return next_mac_number + else: + return next_mac diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index df618c6..ffd0374 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -2,10 +2,13 @@ import argparse from uncloud.hack.vm import VM from uncloud.hack.config import Config +from uncloud.hack.mac import MAC arg_parser = argparse.ArgumentParser('hack', add_help=False) #description="Commands that are unfinished - use at own risk") arg_parser.add_argument('--create-vm', action='store_true') +arg_parser.add_argument('--last-used-mac', action='store_true') +arg_parser.add_argument('--get-new-mac', action='store_true') def main(arguments): @@ -16,3 +19,11 @@ def main(arguments): print("Creating VM") vm = VM(config) vm.create() + + if arguments['last_used_mac']: + m = MAC(config) + print(m.last_used_mac()) + + if arguments['get_new_mac']: + m = MAC(config).get_next() + print(m.last_used()) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index e33e473..eb75902 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -25,48 +25,50 @@ import uuid import os from uncloud.hack.db import DB +from uncloud.hack.mac import MAC class VM(object): - def __init__(self, config): - self.config = config - self.db = DB(config) + def __init__(self, config): + self.config = config + self.db = DB(config, prefix="/vm") - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" - self.qemu="/usr/bin/qemu-system-x86_64" + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" + self.qemu="/usr/bin/qemu-system-x86_64" + self.accel="kvm" - self.vm = {} + self.vm = {} - self.accel="kvm" -# self.mac=$(./mac-gen.py) - self.mac="42:00:00:00:00:42" - self.owner="nico" - self.bridge="br100" + self.owner="nico" + self.bridge="br100" - self.ifup = os.path.join(self.hackprefix, "ifup.sh") - self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") - self.uuid = uuid.uuid4() - self.vm['uuid'] = str(self.uuid) - self.vm['memory']=1024 - self.vm['cores']=2 - self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + def create(self): + self.uuid = uuid.uuid4() + self.vm['uuid'] = str(self.uuid) + self.vm['memory'] = 1024 + self.vm['cores'] = 2 + self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") - self.vm['commandline' ] = [ "sudo", - "{}".format(self.qemu), - "-name", "uncloud-{}".format(self.vm['uuid']), - "-machine", "pc,accel={}".format(self.accel), - "-m", "{}".format(self.vm['memory']), - "-smp", "{}".format(self.vm['cores']), - "-uuid", "{}".format(self.vm['uuid']), - "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), - "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) - ] + self.mac=MAC().next() - def create(self): - self.db.set("vm/{}".format(str(self.vm['uuid'])), - self.vm, store_as_json=True) + self.vm['commandline' ] = [ "sudo", + "{}".format(self.qemu), + "-name", "uncloud-{}".format(self.vm['uuid']), + "-machine", "pc,accel={}".format(self.accel), + "-m", "{}".format(self.vm['memory']), + "-smp", "{}".format(self.vm['cores']), + "-uuid", "{}".format(self.vm['uuid']), + "-drive", "file={},media=cdrom".format(self.vm['os_image']), + "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), + "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) + ] - print(" ".join(self.vm['commandline'])) - subprocess.run(self.vm['commandline']) + self.db.set(str(self.vm['uuid']), + self.vm, + as_json=True) + + print(" ".join(self.vm['commandline'])) + subprocess.run(self.vm['commandline']) From 12e8ccd01c62b8dab8f20cb8ff624c5d1d8aac1c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Jan 2020 19:10:59 +0100 Subject: [PATCH 44/75] Cleanups for mac handling --- uncloud/hack/mac.py | 5 +---- uncloud/hack/main.py | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index 4ac05f2..a408103 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -60,9 +60,6 @@ class MAC(object): return ':'.join(format(s, '02x') for s in b) def get_next(self, vmuuid=None, as_int=False): -# if self.free: -# return self.free.pop() - last_number = self.last_used_index() # FIXME: compare to 48bit minus prefix length @@ -81,7 +78,7 @@ class MAC(object): db_entry['mac_address'] = next_mac self.db.set("used/{}".format(next_mac), - db_entry) + db_entry, as_json=True) if as_int: return next_mac_number diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index ffd0374..2980516 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -25,5 +25,4 @@ def main(arguments): print(m.last_used_mac()) if arguments['get_new_mac']: - m = MAC(config).get_next() - print(m.last_used()) + print(MAC(config).get_next()) From b877ab13b34b058540fc10fabde67501be1b79f8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 10:02:37 +0100 Subject: [PATCH 45/75] add hack code --- uncloud/hack/db.py | 21 +++++++++++++++++---- uncloud/hack/mac.py | 8 ++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py index ac643bd..cb5e490 100644 --- a/uncloud/hack/db.py +++ b/uncloud/hack/db.py @@ -86,10 +86,23 @@ class DB(object): return self._db_clients[0].put(self.realkey(key), value, **kwargs) @readable_errors - def increment(key, **kwargs): - with self._db_clients[0].lock(key) as lock: - value = int(self.get(self.realkey(key), **kwargs)) - self.set(self.realkey(key), str(value + 1), **kwargs) + def increment(self, key, **kwargs): + print(self.realkey(key)) + + + print("prelock") + lock = self._db_clients[0].lock('/nicohack/foo') + print("prelockacq") + lock.acquire() + print("prelockrelease") + lock.release() + + with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock: + print("in lock") + pass + +# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs)) +# self.set(self.realkey(key), str(value + 1), **kwargs) if __name__ == '__main__': diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index a408103..e7f41a2 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -48,7 +48,9 @@ class MAC(object): def last_used_index(self): value = self.db.get("last_used_index") if not value: - return 0 + self.db.set("last_used_index", "0") + value = self.db.get("last_used_index") + return int(value) def last_used_mac(self): @@ -62,7 +64,7 @@ class MAC(object): def get_next(self, vmuuid=None, as_int=False): last_number = self.last_used_index() - # FIXME: compare to 48bit minus prefix length + # FIXME: compare to 48bit minus prefix length to the power of 2 if last_number == int('0xffffff', 16): raise UncloudException("Exhausted all possible mac addresses - try to free some") @@ -77,6 +79,8 @@ class MAC(object): db_entry['index'] = next_number db_entry['mac_address'] = next_mac + # should be one transaction + self.db.increment("last_used_index") self.db.set("used/{}".format(next_mac), db_entry, as_json=True) From 26d5c916256ccf91c99de92ae6a80353d58b4720 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 10:53:22 +0100 Subject: [PATCH 46/75] Update hacking docs --- uncloud/docs/source/hacking.rst | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/uncloud/docs/source/hacking.rst b/uncloud/docs/source/hacking.rst index 2df42a7..d198126 100644 --- a/uncloud/docs/source/hacking.rst +++ b/uncloud/docs/source/hacking.rst @@ -1,17 +1,25 @@ Hacking ======= -How to hack on the code. +Using uncloud in hacking (aka development) mode. -[ to be done by Balazs: -* make nice -* indent with shell script mode +Get the code +------------ +.. code-block:: sh + :linenos: -] + git clone https://code.ungleich.ch/uncloud/uncloud.git -* git clone the repo -* cd to the repo -* Setup your venv: python -m venv venv -* . ./venv/bin/activate # you need the leading dot for sourcing! -* Run ./bin/ucloud-run-reinstall - it should print you an error - message on how to use ucloud + + +Install python requirements +--------------------------- +You need to have python3 installed. + +.. code-block:: sh + :linenos: + + cd uncloud! + python -m venv venv + . ./venv/bin/activate + ./bin/uncloud-run-reinstall From bd03f95e9925589375d30e60b0dc4b1960dae6ff Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 11:32:23 +0100 Subject: [PATCH 47/75] [docs] move one level higher --- {uncloud/docs => docs}/Makefile | 2 +- {uncloud/docs => docs}/README.md | 0 {uncloud/docs => docs}/__init__.py | 0 {uncloud/docs => docs}/source/__init__.py | 0 {uncloud/docs => docs}/source/admin-guide | 0 {uncloud/docs => docs}/source/conf.py | 0 {uncloud/docs => docs}/source/diagram-code/ucloud | 0 {uncloud/docs => docs}/source/hacking.rst | 0 {uncloud/docs => docs}/source/images/ucloud.svg | 0 {uncloud/docs => docs}/source/index.rst | 0 {uncloud/docs => docs}/source/introduction.rst | 0 {uncloud/docs => docs}/source/misc/todo.rst | 0 {uncloud/docs => docs}/source/setup-install.rst | 0 {uncloud/docs => docs}/source/theory/summary.rst | 0 {uncloud/docs => docs}/source/troubleshooting.rst | 0 {uncloud/docs => docs}/source/user-guide.rst | 0 .../source/user-guide/how-to-create-an-os-image-for-ucloud.rst | 0 17 files changed, 1 insertion(+), 1 deletion(-) rename {uncloud/docs => docs}/Makefile (93%) rename {uncloud/docs => docs}/README.md (100%) rename {uncloud/docs => docs}/__init__.py (100%) rename {uncloud/docs => docs}/source/__init__.py (100%) rename {uncloud/docs => docs}/source/admin-guide (100%) rename {uncloud/docs => docs}/source/conf.py (100%) rename {uncloud/docs => docs}/source/diagram-code/ucloud (100%) rename {uncloud/docs => docs}/source/hacking.rst (100%) rename {uncloud/docs => docs}/source/images/ucloud.svg (100%) rename {uncloud/docs => docs}/source/index.rst (100%) rename {uncloud/docs => docs}/source/introduction.rst (100%) rename {uncloud/docs => docs}/source/misc/todo.rst (100%) rename {uncloud/docs => docs}/source/setup-install.rst (100%) rename {uncloud/docs => docs}/source/theory/summary.rst (100%) rename {uncloud/docs => docs}/source/troubleshooting.rst (100%) rename {uncloud/docs => docs}/source/user-guide.rst (100%) rename {uncloud/docs => docs}/source/user-guide/how-to-create-an-os-image-for-ucloud.rst (100%) diff --git a/uncloud/docs/Makefile b/docs/Makefile similarity index 93% rename from uncloud/docs/Makefile rename to docs/Makefile index 5e7ea85..246b56c 100644 --- a/uncloud/docs/Makefile +++ b/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source/ BUILDDIR = build/ -DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/ +DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/ .PHONY: all build clean diff --git a/uncloud/docs/README.md b/docs/README.md similarity index 100% rename from uncloud/docs/README.md rename to docs/README.md diff --git a/uncloud/docs/__init__.py b/docs/__init__.py similarity index 100% rename from uncloud/docs/__init__.py rename to docs/__init__.py diff --git a/uncloud/docs/source/__init__.py b/docs/source/__init__.py similarity index 100% rename from uncloud/docs/source/__init__.py rename to docs/source/__init__.py diff --git a/uncloud/docs/source/admin-guide b/docs/source/admin-guide similarity index 100% rename from uncloud/docs/source/admin-guide rename to docs/source/admin-guide diff --git a/uncloud/docs/source/conf.py b/docs/source/conf.py similarity index 100% rename from uncloud/docs/source/conf.py rename to docs/source/conf.py diff --git a/uncloud/docs/source/diagram-code/ucloud b/docs/source/diagram-code/ucloud similarity index 100% rename from uncloud/docs/source/diagram-code/ucloud rename to docs/source/diagram-code/ucloud diff --git a/uncloud/docs/source/hacking.rst b/docs/source/hacking.rst similarity index 100% rename from uncloud/docs/source/hacking.rst rename to docs/source/hacking.rst diff --git a/uncloud/docs/source/images/ucloud.svg b/docs/source/images/ucloud.svg similarity index 100% rename from uncloud/docs/source/images/ucloud.svg rename to docs/source/images/ucloud.svg diff --git a/uncloud/docs/source/index.rst b/docs/source/index.rst similarity index 100% rename from uncloud/docs/source/index.rst rename to docs/source/index.rst diff --git a/uncloud/docs/source/introduction.rst b/docs/source/introduction.rst similarity index 100% rename from uncloud/docs/source/introduction.rst rename to docs/source/introduction.rst diff --git a/uncloud/docs/source/misc/todo.rst b/docs/source/misc/todo.rst similarity index 100% rename from uncloud/docs/source/misc/todo.rst rename to docs/source/misc/todo.rst diff --git a/uncloud/docs/source/setup-install.rst b/docs/source/setup-install.rst similarity index 100% rename from uncloud/docs/source/setup-install.rst rename to docs/source/setup-install.rst diff --git a/uncloud/docs/source/theory/summary.rst b/docs/source/theory/summary.rst similarity index 100% rename from uncloud/docs/source/theory/summary.rst rename to docs/source/theory/summary.rst diff --git a/uncloud/docs/source/troubleshooting.rst b/docs/source/troubleshooting.rst similarity index 100% rename from uncloud/docs/source/troubleshooting.rst rename to docs/source/troubleshooting.rst diff --git a/uncloud/docs/source/user-guide.rst b/docs/source/user-guide.rst similarity index 100% rename from uncloud/docs/source/user-guide.rst rename to docs/source/user-guide.rst diff --git a/uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst b/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst similarity index 100% rename from uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst rename to docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst From 8a451ff4ffd82ab382183eb1017704c8d4ea25d2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 12:40:37 +0100 Subject: [PATCH 48/75] [hack] phase in networking --- uncloud/hack/net.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 uncloud/hack/net.py diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py new file mode 100644 index 0000000..142eeb7 --- /dev/null +++ b/uncloud/hack/net.py @@ -0,0 +1,21 @@ +import subprocess + +class VXLANBridge(object): + def __init__(self, bridgedev=None, uplinkdev=None): + self.management_vni = 1 + + cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {netid} dstport 4789 group ff05::{netid} dev {uplinkdev} ttl 5" + cmd_up_dev = "ip link set {dev} up" + cmd_create_bridge="ip link add {bridgedev} type bridge" + cmd_add_to_bridge="ip link set {vxlandev} master {bridgedev} up" + cmd_add_addr="ip addr add {ip} dev {bridgedev}" + + def setup_networking(dev=wlan0, v6net): + ip=2a0a:e5c1:111:888::48/64 + vxlandev=vxlan${netid} + bridgedev=br${netid} + + +class DNSRA(object): + def __init__(self): + pass From 1b5a3f6d2ee71e75bdef9540ff204940b72a1f5c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 15 Jan 2020 13:26:05 +0100 Subject: [PATCH 49/75] Progress with networking --- docs/source/hacking.rst | 11 +++++++++++ uncloud/hack/main.py | 10 ++++++++++ uncloud/hack/net.py | 4 ++++ 3 files changed, 25 insertions(+) diff --git a/docs/source/hacking.rst b/docs/source/hacking.rst index d198126..1c750d6 100644 --- a/docs/source/hacking.rst +++ b/docs/source/hacking.rst @@ -23,3 +23,14 @@ You need to have python3 installed. python -m venv venv . ./venv/bin/activate ./bin/uncloud-run-reinstall + + + +Install os requirements +----------------------- +Install the following software packages: **dnsmasq**. + +If you already have a working IPv6 SLAAC and DNS setup, +this step can be skipped. + +Note that you need at least one /64 IPv6 network to run uncloud. diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 2980516..d7a4714 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -3,12 +3,18 @@ import argparse from uncloud.hack.vm import VM from uncloud.hack.config import Config from uncloud.hack.mac import MAC +from uncloud import UncloudException arg_parser = argparse.ArgumentParser('hack', add_help=False) #description="Commands that are unfinished - use at own risk") arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--last-used-mac', action='store_true') arg_parser.add_argument('--get-new-mac', action='store_true') +arg_parser.add_argument('--management-network', help="IPv6 management network") +arg_parser.add_argument('--run-dns-ra', action='store_true', + help="Provide router advertisements and DNS resolution via dnsmasq") + + def main(arguments): @@ -26,3 +32,7 @@ def main(arguments): if arguments['get_new_mac']: print(MAC(config).get_next()) + + if arguments['run_dns_ra']: + if not arguments['management_network']: + raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network. You can use fd00::/64 for testing (non production!)") diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 142eeb7..0a7819b 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -19,3 +19,7 @@ class VXLANBridge(object): class DNSRA(object): def __init__(self): pass + + +class Firewall(object): + pass From b8472607684a7ca9c73f86296144fe83a6d5e4f4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 09:16:29 +0100 Subject: [PATCH 50/75] ++network --- uncloud/hack/main.py | 13 +++++++++++++ uncloud/hack/net.py | 6 ++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index d7a4714..4ccb74a 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -3,6 +3,8 @@ import argparse from uncloud.hack.vm import VM from uncloud.hack.config import Config from uncloud.hack.mac import MAC +from uncloud.hack.net import VXLANBridge, DNSRA + from uncloud import UncloudException arg_parser = argparse.ArgumentParser('hack', add_help=False) @@ -10,6 +12,8 @@ arg_parser = argparse.ArgumentParser('hack', add_help=False) arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--last-used-mac', action='store_true') arg_parser.add_argument('--get-new-mac', action='store_true') + +arg_parser.add_argument('--init-network', help="Initialise networking") arg_parser.add_argument('--management-network', help="IPv6 management network") arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") @@ -33,6 +37,15 @@ def main(arguments): if arguments['get_new_mac']: print(MAC(config).get_next()) + if arguments['init_networking!']: + if not arguments['management_network']: + raise UncloudException("Initialising the network requires an IPv6 network. You can use fd00::/64 for testing (non production!)") + vb = VXLANBridge(arguments['management_network']) + vb.setup() + if arguments['run_dns_ra']: if not arguments['management_network']: raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network. You can use fd00::/64 for testing (non production!)") + + dnsra = DNSRA(arguments['management_network']) + dnsra.setup() diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 0a7819b..11649b8 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -1,7 +1,10 @@ import subprocess +class ManagementBridge(VXLANBridge): + pass + class VXLANBridge(object): - def __init__(self, bridgedev=None, uplinkdev=None): + def __init__(self, vni, bridgedev=None, uplinkdev=None): self.management_vni = 1 cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {netid} dstport 4789 group ff05::{netid} dev {uplinkdev} ttl 5" @@ -11,7 +14,6 @@ class VXLANBridge(object): cmd_add_addr="ip addr add {ip} dev {bridgedev}" def setup_networking(dev=wlan0, v6net): - ip=2a0a:e5c1:111:888::48/64 vxlandev=vxlan${netid} bridgedev=br${netid} From 2b8831784a4d22ec8f20216ccb54139e3da98aeb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 11:30:16 +0100 Subject: [PATCH 51/75] [pep440] improve versioning name for python --- bin/gen-version | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/gen-version b/bin/gen-version index a2e2882..06c3e22 100755 --- a/bin/gen-version +++ b/bin/gen-version @@ -1,22 +1,22 @@ #!/bin/sh # -*- coding: utf-8 -*- # -# 2019 Nico Schottelius (nico-ucloud at schottelius.org) +# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org) # -# This file is part of ucloud. +# This file is part of uncloud. # -# ucloud is free software: you can redistribute it and/or modify +# uncloud is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# ucloud is distributed in the hope that it will be useful, +# uncloud is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with ucloud. If not, see . +# along with uncloud. If not, see . # # @@ -26,4 +26,4 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py +printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py From 30be79131212cefb844d79afc86ffbb20ac921ab Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 11:30:30 +0100 Subject: [PATCH 52/75] Be less verbose when reinstalling --- bin/uncloud-run-reinstall | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/uncloud-run-reinstall b/bin/uncloud-run-reinstall index 18e95c0..b211613 100755 --- a/bin/uncloud-run-reinstall +++ b/bin/uncloud-run-reinstall @@ -24,6 +24,6 @@ dir=${0%/*} ${dir}/gen-version; -pip uninstall -y uncloud -python setup.py install +pip uninstall -y uncloud >/dev/null +python setup.py install >/dev/null ${dir}/uncloud "$@" From bd9dbb12b798a1bfe0651cfb7bcae22058ae456b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 11:30:41 +0100 Subject: [PATCH 53/75] Cleanup networking --- uncloud/hack/main.py | 23 +++++++++++++--------- uncloud/hack/net.py | 45 +++++++++++++++++++++++++++++++++++--------- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 4ccb74a..cb9fd7b 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -13,8 +13,11 @@ arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--last-used-mac', action='store_true') arg_parser.add_argument('--get-new-mac', action='store_true') -arg_parser.add_argument('--init-network', help="Initialise networking") -arg_parser.add_argument('--management-network', help="IPv6 management network") +arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true') +arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true') +arg_parser.add_argument('--network', help="/64 IPv6 network") +arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0") +arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") @@ -37,15 +40,17 @@ def main(arguments): if arguments['get_new_mac']: print(MAC(config).get_next()) - if arguments['init_networking!']: - if not arguments['management_network']: - raise UncloudException("Initialising the network requires an IPv6 network. You can use fd00::/64 for testing (non production!)") - vb = VXLANBridge(arguments['management_network']) - vb.setup() + #if arguments['init_network']: + if arguments['create_vxlan']: + if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: + raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + vb = VXLANBridge(vni=arguments['vni'], + uplinkdev=arguments['vxlan_uplink_device']) + vb._setup_vxlan() if arguments['run_dns_ra']: - if not arguments['management_network']: + if not arguments['network']: raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network. You can use fd00::/64 for testing (non production!)") - dnsra = DNSRA(arguments['management_network']) + dnsra = DNSRA(arguments['network']) dnsra.setup() diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 11649b8..170e7b9 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -1,21 +1,48 @@ import subprocess +import ipaddress + +from uncloud import UncloudException -class ManagementBridge(VXLANBridge): - pass class VXLANBridge(object): - def __init__(self, vni, bridgedev=None, uplinkdev=None): - self.management_vni = 1 - - cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {netid} dstport 4789 group ff05::{netid} dev {uplinkdev} ttl 5" + cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" cmd_up_dev = "ip link set {dev} up" cmd_create_bridge="ip link add {bridgedev} type bridge" cmd_add_to_bridge="ip link set {vxlandev} master {bridgedev} up" cmd_add_addr="ip addr add {ip} dev {bridgedev}" - def setup_networking(dev=wlan0, v6net): - vxlandev=vxlan${netid} - bridgedev=br${netid} + # VXLAN ids are at maximum 24 bit - use a /104 + multicast_network = ipaddress.IPv6Network("ff05::/104") + max_vni = (2**24)-1 + + def __init__(self, + vni, + uplinkdev): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + self.config['multicast_address'] = self.multicast_network[vni] + + self.config['uplinkdev'] = uplinkdev + self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + + def setup_networking(self): + pass + + def _setup_vxlan(self): + # check for device first (?) + cmd = self.cmd_create_vxlan.format(**self.config) + print(cmd) + subprocess.run(cmd.split()) + +class ManagementBridge(VXLANBridge): + pass class DNSRA(object): From 8888f5d9f7aaaf20d10f1657bb2df60df4a6f912 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 19 Jan 2020 12:55:06 +0100 Subject: [PATCH 54/75] add logging --- scripts/uncloud | 29 ++++++++++---------------- uncloud/hack/main.py | 7 ++++++- uncloud/hack/net.py | 49 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/scripts/uncloud b/scripts/uncloud index ab5b40d..d565954 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -11,17 +11,7 @@ from uncloud.common import settings from uncloud import UncloudException from uncloud.common.cli import resolve_otp_credentials - -def exception_hook(exc_type, exc_value, exc_traceback): - logging.getLogger(__name__).error( - 'Uncaught exception', - exc_info=(exc_type, exc_value, exc_traceback) - ) - - -sys.excepthook = exception_hook - -# the components that use etcd +# Components that use etcd ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure', 'hack'] @@ -30,10 +20,6 @@ ALL_COMPONENTS = ETCD_COMPONENTS.copy() if __name__ == '__main__': - # Setting up root logger - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - arg_parser = argparse.ArgumentParser() subparsers = arg_parser.add_subparsers(dest='command') @@ -84,11 +70,18 @@ if __name__ == '__main__': mod = importlib.import_module('uncloud.{}.main'.format(name)) main = getattr(mod, 'main') + if arguments['debug']: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + log = logging.getLogger() + try: main(arguments) except UncloudException as err: - logger.error(err) + log.error(err) # except ConnectionFailedError as err: -# logger.error('Cannot connect to etcd: {}'.format(err)) +# log.error('Cannot connect to etcd: {}'.format(err)) except Exception as err: - logger.exception(err) + log.exception(err) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index cb9fd7b..f275e62 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -20,6 +20,7 @@ arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") +arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') @@ -45,8 +46,12 @@ def main(arguments): if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") vb = VXLANBridge(vni=arguments['vni'], - uplinkdev=arguments['vxlan_uplink_device']) + route=arguments['network'], + uplinkdev=arguments['vxlan_uplink_device'], + use_sudo=arguments['use_sudo']) vb._setup_vxlan() + vb._setup_bridge() + vb._route_network() if arguments['run_dns_ra']: if not arguments['network']: diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 170e7b9..e18b36a 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -1,15 +1,20 @@ import subprocess import ipaddress +import logging + from uncloud import UncloudException +log = logging.getLogger(__name__) + class VXLANBridge(object): - cmd_create_vxlan = "ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" - cmd_up_dev = "ip link set {dev} up" - cmd_create_bridge="ip link add {bridgedev} type bridge" - cmd_add_to_bridge="ip link set {vxlandev} master {bridgedev} up" - cmd_add_addr="ip addr add {ip} dev {bridgedev}" + cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" + cmd_up_dev = "{sudo}ip link set {dev} up" + cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge" + cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up" + cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}" + cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}" # VXLAN ids are at maximum 24 bit - use a /104 multicast_network = ipaddress.IPv6Network("ff05::/104") @@ -17,16 +22,28 @@ class VXLANBridge(object): def __init__(self, vni, - uplinkdev): + uplinkdev, + route=None, + use_sudo=False): self.config = {} if vni > self.max_vni: raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + if use_sudo: + self.config['sudo'] = 'sudo ' + self.config['vni_dec'] = vni self.config['vni_hex'] = "{:x}".format(vni) self.config['multicast_address'] = self.multicast_network[vni] + #try: + self.config['route_network'] = ipaddress.IPv6Network(route) + #except Exception as e: + # print("Ahhhhhhhhhhhhhhhhh, die: {}".format(e)) + + self.config['route'] = route + self.config['uplinkdev'] = uplinkdev self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) @@ -36,9 +53,23 @@ class VXLANBridge(object): pass def _setup_vxlan(self): - # check for device first (?) - cmd = self.cmd_create_vxlan.format(**self.config) - print(cmd) + self._execute_cmd(self.cmd_create_vxlan) + self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev']) + + def _setup_bridge(self): + self._execute_cmd(self.cmd_create_bridge) + self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev']) + + def _route_network(self): + self._execute_cmd(self.cmd_add_route_dev) + + def _add_vxlan_to_bridge(self): + self._execute_cmd(self.cmd_add_to_bridge) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + print("Executing: {}".format(cmd)) subprocess.run(cmd.split()) class ManagementBridge(VXLANBridge): From 8e839aeb44ec47e82d446b3545cbe283f35c80ea Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 23 Jan 2020 18:41:59 +0100 Subject: [PATCH 55/75] commit stuff before dominique does --- uncloud/hack/main.py | 5 +++-- uncloud/hack/net.py | 5 ----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index f275e62..1e38c8a 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -1,4 +1,5 @@ import argparse +import logging from uncloud.hack.vm import VM from uncloud.hack.config import Config @@ -22,11 +23,11 @@ arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') - +log = logging.getLogger(__name__) def main(arguments): - print(arguments) + log.debug("args={}".format(arguments)) config = Config(arguments) if arguments['create_vm']: diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index e18b36a..e695dc8 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -37,11 +37,7 @@ class VXLANBridge(object): self.config['vni_hex'] = "{:x}".format(vni) self.config['multicast_address'] = self.multicast_network[vni] - #try: self.config['route_network'] = ipaddress.IPv6Network(route) - #except Exception as e: - # print("Ahhhhhhhhhhhhhhhhh, die: {}".format(e)) - self.config['route'] = route self.config['uplinkdev'] = uplinkdev @@ -69,7 +65,6 @@ class VXLANBridge(object): def _execute_cmd(self, cmd_string, **kwargs): cmd = cmd_string.format(**self.config, **kwargs) log.info("Executing: {}".format(cmd)) - print("Executing: {}".format(cmd)) subprocess.run(cmd.split()) class ManagementBridge(VXLANBridge): From 0982927c1bfe2a91b8244e148e3d7098b7c44ede Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Thu, 23 Jan 2020 18:43:41 +0100 Subject: [PATCH 56/75] Added DNSmasq ability for RA --- uncloud/hack/main.py | 10 ++++++---- uncloud/hack/net.py | 36 ++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index f275e62..281c251 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -54,8 +54,10 @@ def main(arguments): vb._route_network() if arguments['run_dns_ra']: - if not arguments['network']: - raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network. You can use fd00::/64 for testing (non production!)") + if not arguments['network'] or not arguments['vni']: + raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") - dnsra = DNSRA(arguments['network']) - dnsra.setup() + dnsra = DNSRA(route=arguments['network'], + vni=arguments['vni'], + use_sudo=arguments['use_sudo']) + dnsra._setup_dnsmasq() diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index e18b36a..b036198 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -77,9 +77,41 @@ class ManagementBridge(VXLANBridge): class DNSRA(object): - def __init__(self): - pass + # VXLAN ids are at maximum 24 bit + max_vni = (2**24)-1 + # Command to start dnsmasq + cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra" + + def __init__(self, + vni, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + + #TODO: remove if not needed + #self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + + # dnsmasq only wants the network without the prefix, therefore, cut it off + self.config['route'] = ipaddress.IPv6Network(route).network_address + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + def _setup_dnsmasq(self): + self._execute_cmd(self.cmd_start_dnsmasq) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + print("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + class Firewall(object): pass From c881c7ce4d65044f0b8bc63de981680f2bab9a1e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 23 Jan 2020 21:15:26 +0100 Subject: [PATCH 57/75] hack mac: be a proper python class --- uncloud/hack/mac.py | 50 +++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index e7f41a2..084df13 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -38,7 +38,8 @@ class MAC(object): self.config = config self.db = DB(config, prefix="/mac") - self.prefix = 0x002000000000 + self.prefix = 0x420000000000 + self._number = 0 # Not set by default @staticmethod def validate_mac(mac): @@ -56,35 +57,40 @@ class MAC(object): def last_used_mac(self): return self.int_to_mac(self.prefix + self.last_used_index()) - @staticmethod - def int_to_mac(number): - b = number.to_bytes(6, byteorder="big") + def to_colon_format(self): + b = self._number.to_bytes(6, byteorder="big") return ':'.join(format(s, '02x') for s in b) - def get_next(self, vmuuid=None, as_int=False): + def to_str_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ''.join(format(s, '02x') for s in b) + + def create(self): last_number = self.last_used_index() - # FIXME: compare to 48bit minus prefix length to the power of 2 - if last_number == int('0xffffff', 16): + if last_number == int('0xffffffff', 16): raise UncloudException("Exhausted all possible mac addresses - try to free some") next_number = last_number + 1 - next_number_string = "{:012x}".format(next_number) + self._number = self.prefix + next_number - next_mac_number = self.prefix + next_number - next_mac = self.int_to_mac(next_mac_number) - - db_entry = {} - db_entry['vm_uuid'] = vmuuid - db_entry['index'] = next_number - db_entry['mac_address'] = next_mac + #next_number_string = "{:012x}".format(next_number) + #next_mac = self.int_to_mac(next_mac_number) + # db_entry = {} + # db_entry['vm_uuid'] = vmuuid + # db_entry['index'] = next_number + # db_entry['mac_address'] = next_mac # should be one transaction - self.db.increment("last_used_index") - self.db.set("used/{}".format(next_mac), - db_entry, as_json=True) + # self.db.increment("last_used_index") + # self.db.set("used/{}".format(next_mac), + # db_entry, as_json=True) - if as_int: - return next_mac_number - else: - return next_mac + def __int__(self): + return self._number + + def __repr__(self): + return self.to_str_format() + + def __str__(self): + return self.to_colon_format() From 46a04048b54dc148d8b5538c0674d56038e00017 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Thu, 23 Jan 2020 21:17:09 +0100 Subject: [PATCH 58/75] small changes in vm.py to make it more generic --- uncloud/hack/vm.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index eb75902..e8038cc 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -32,27 +32,35 @@ class VM(object): self.config = config self.db = DB(config, prefix="/vm") - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" - self.qemu="/usr/bin/qemu-system-x86_64" - self.accel="kvm" + #TODO: Select generic + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm + self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config + self.accel="kvm" #TODO: should be config self.vm = {} - self.owner="nico" - self.bridge="br100" + #TODO: this should be generic + self.vm['owner']="nico" #TODO: Should in config.arguments + #self.config['vni_hex'] = "{:x}".format(self.config.vni) + #self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + self.vni_hex = "{:x}".format(self.config.arguments['vni']) + self.bridgedev = "br{}".format(self.vni_hex) + + #TODO: Touch later! (when necessary) self.ifup = os.path.join(self.hackprefix, "ifup.sh") self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") def create(self): self.uuid = uuid.uuid4() + #TODO: This all should be generic self.vm['uuid'] = str(self.uuid) self.vm['memory'] = 1024 self.vm['cores'] = 2 self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") - self.mac=MAC().next() + self.mac=MAC(self.config).get_next() self.vm['commandline' ] = [ "sudo", "{}".format(self.qemu), @@ -62,7 +70,7 @@ class VM(object): "-smp", "{}".format(self.vm['cores']), "-uuid", "{}".format(self.vm['uuid']), "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={}".format(self.ifup, self.ifdown), + "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown),self.mac, "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] From b5409552d8765afadada6c26793162beb8a5eda3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 23 Jan 2020 21:20:16 +0100 Subject: [PATCH 59/75] prepare vm.py for dominique --- uncloud/hack/vm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index e8038cc..1a531e0 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -32,7 +32,7 @@ class VM(object): self.config = config self.db = DB(config, prefix="/vm") - #TODO: Select generic + #TODO: Select generic self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config self.accel="kvm" #TODO: should be config @@ -60,7 +60,9 @@ class VM(object): self.vm['cores'] = 2 self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") - self.mac=MAC(self.config).get_next() + self.mac=MAC(self.config) + self.mac.create() + self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) self.vm['commandline' ] = [ "sudo", "{}".format(self.qemu), @@ -74,6 +76,8 @@ class VM(object): "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] + # TODO: Add ip link command afterwards (rouxdo) + self.db.set(str(self.vm['uuid']), self.vm, as_json=True) From 58daf8191e3c5d48a96b745a906fb4d9fa2a72e0 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 24 Jan 2020 13:56:08 +0100 Subject: [PATCH 60/75] refactored vm.py to create a VM --- uncloud/hack/mac.py | 14 ++++-- uncloud/hack/main.py | 6 ++- uncloud/hack/vm.py | 106 ++++++++++++++++++++++++------------------- 3 files changed, 75 insertions(+), 51 deletions(-) diff --git a/uncloud/hack/mac.py b/uncloud/hack/mac.py index 084df13..66286dd 100755 --- a/uncloud/hack/mac.py +++ b/uncloud/hack/mac.py @@ -36,7 +36,9 @@ log = logging.getLogger(__name__) class MAC(object): def __init__(self, config): self.config = config - self.db = DB(config, prefix="/mac") + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(config, prefix="/mac") self.prefix = 0x420000000000 self._number = 0 # Not set by default @@ -47,10 +49,14 @@ class MAC(object): raise Error("Not a valid mac address: %s" % mac) def last_used_index(self): - value = self.db.get("last_used_index") - if not value: - self.db.set("last_used_index", "0") + if not self.no_db: value = self.db.get("last_used_index") + if not value: + self.db.set("last_used_index", "0") + value = self.db.get("last_used_index") + + else: + value = "0" return int(value) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 2981184..4778ef6 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -22,6 +22,10 @@ arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') +arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) +arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) +arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') + log = logging.getLogger(__name__) @@ -33,7 +37,7 @@ def main(arguments): if arguments['create_vm']: print("Creating VM") vm = VM(config) - vm.create() + vm.commandline() if arguments['last_used_mac']: m = MAC(config) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 1a531e0..8e20e2e 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -29,58 +29,72 @@ from uncloud.hack.mac import MAC class VM(object): def __init__(self, config): - self.config = config - self.db = DB(config, prefix="/vm") + self.config = config + #TODO: Enable etcd lookup + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(self.config, prefix="/vm") - #TODO: Select generic - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm - self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config - self.accel="kvm" #TODO: should be config + #TODO: Select generic + #self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm + self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing + self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config + self.accel="kvm" #TODO: should be config - self.vm = {} + self.vm = {} + #TODO: Touch later! (when necessary) + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") - #TODO: this should be generic - self.vm['owner']="nico" #TODO: Should in config.arguments - #self.config['vni_hex'] = "{:x}".format(self.config.vni) - #self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) - self.vni_hex = "{:x}".format(self.config.arguments['vni']) - self.bridgedev = "br{}".format(self.vni_hex) + def commandline(self): + """This method is used to trigger / create a vm from the cli""" + #TODO: read arguments from cli + #TODO: create etcd json object + self.vm['owner']= "nico" + self.vm['memory'] = self.config.arguments['memory'] + self.vm['cores'] = self.config.arguments['cores'] + self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + self.create_template() + # mimics api call = this will already be in etcd + #self.vm['os_image'] = self.db.get("os_image") + self.create() + def create_template(self): + self.uuid = uuid.uuid4() + #TODO: This all should be generic + self.vm['uuid'] = str(self.uuid) + #self.vni_hex = "{:x}".format(self.config.arguments['vni']) + self.bridgedev = "br{}".format("{:x}".format(self.config.arguments['vni'])) + + #TODO: Enable sudo + if self.config.arguments['use_sudo']: + self.sudo = "sudo" + + self.mac=MAC(self.config) + self.mac.create() + self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) + + #self.vm['commandline'] = [ "{}".format(self.sudo), + self.vm['commandline'] = [ "{}".format(self.sudo), + "{}".format(self.qemu), + "-name", "uncloud-{}".format(self.vm['uuid']), + "-machine", "pc,accel={}".format(self.accel), + "-m", "{}".format(self.vm['memory']), + "-smp", "{}".format(self.vm['cores']), + "-uuid", "{}".format(self.vm['uuid']), + "-drive", "file={},media=cdrom".format(self.vm['os_image']), + "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.mac), + "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) + ] - #TODO: Touch later! (when necessary) - self.ifup = os.path.join(self.hackprefix, "ifup.sh") - self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") def create(self): - self.uuid = uuid.uuid4() - #TODO: This all should be generic - self.vm['uuid'] = str(self.uuid) - self.vm['memory'] = 1024 - self.vm['cores'] = 2 - self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + if not self.no_db: + self.db.set(str(self.vm['uuid']), + self.vm, + as_json=True) - self.mac=MAC(self.config) - self.mac.create() - self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) - - self.vm['commandline' ] = [ "sudo", - "{}".format(self.qemu), - "-name", "uncloud-{}".format(self.vm['uuid']), - "-machine", "pc,accel={}".format(self.accel), - "-m", "{}".format(self.vm['memory']), - "-smp", "{}".format(self.vm['cores']), - "-uuid", "{}".format(self.vm['uuid']), - "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown),self.mac, - "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) - ] - - # TODO: Add ip link command afterwards (rouxdo) - - self.db.set(str(self.vm['uuid']), - self.vm, - as_json=True) - - print(" ".join(self.vm['commandline'])) - subprocess.run(self.vm['commandline']) + print(" ".join(self.vm['commandline'])) + subprocess.run(self.vm['commandline']) #TODO: run in background + #TODO: Add interface ifname to bridge brXX (via net.py: public function add iface to bridge) From 7e91f60c0acf75c8d7bae75a8e1068cdbf4784cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 24 Jan 2020 14:10:08 +0100 Subject: [PATCH 61/75] sudo fix --- uncloud/hack/net.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 6e2a6ee..30d0c03 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -32,6 +32,8 @@ class VXLANBridge(object): if use_sudo: self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' self.config['vni_dec'] = vni self.config['vni_hex'] = "{:x}".format(vni) From 93d7a409b12e2cf7ab3c06a312cc6b4901816db0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 24 Jan 2020 14:10:49 +0100 Subject: [PATCH 62/75] Fix Dominique's sudo bug Totally not related to my previous commit --- uncloud/hack/net.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uncloud/hack/net.py b/uncloud/hack/net.py index 30d0c03..f28ab7f 100644 --- a/uncloud/hack/net.py +++ b/uncloud/hack/net.py @@ -92,6 +92,8 @@ class DNSRA(object): if use_sudo: self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' #TODO: remove if not needed #self.config['vni_dec'] = vni From b1319d654af20cd14c5f8f9b82a67a5e58d93098 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 24 Jan 2020 14:15:48 +0100 Subject: [PATCH 63/75] Make me and Dominique happy (aka add vxlan to bridge) --- uncloud/hack/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 4778ef6..fc54da1 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -56,13 +56,14 @@ def main(arguments): use_sudo=arguments['use_sudo']) vb._setup_vxlan() vb._setup_bridge() + vb._add_vxlan_to_bridge() vb._route_network() if arguments['run_dns_ra']: if not arguments['network'] or not arguments['vni']: raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") - dnsra = DNSRA(route=arguments['network'], + dnsra = DNSRA(route=arguments['network'], vni=arguments['vni'], use_sudo=arguments['use_sudo']) dnsra._setup_dnsmasq() From ae3482cc71350b8fc85a578798b73f500df45bd7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 24 Jan 2020 14:21:38 +0100 Subject: [PATCH 64/75] Fix and break some VM stuff --- uncloud/hack/vm.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 8e20e2e..c41fddc 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -36,8 +36,8 @@ class VM(object): self.db = DB(self.config, prefix="/vm") #TODO: Select generic - #self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm - self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing + self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm + #self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config self.accel="kvm" #TODO: should be config @@ -67,14 +67,19 @@ class VM(object): #self.vni_hex = "{:x}".format(self.config.arguments['vni']) self.bridgedev = "br{}".format("{:x}".format(self.config.arguments['vni'])) - #TODO: Enable sudo + #TODO: Enable sudo -- FIXME! if self.config.arguments['use_sudo']: self.sudo = "sudo" + else: + self.sudo = "" + self.mac=MAC(self.config) self.mac.create() self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) + # FIXME: TODO: turn this into a string and THEN + # .split() it later -- easier for using .format() #self.vm['commandline'] = [ "{}".format(self.sudo), self.vm['commandline'] = [ "{}".format(self.sudo), "{}".format(self.qemu), @@ -84,7 +89,7 @@ class VM(object): "-smp", "{}".format(self.vm['cores']), "-uuid", "{}".format(self.vm['uuid']), "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.mac), + "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] From 5711bf4770159821a50e5ef0b677bdba860780c8 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 24 Jan 2020 14:34:34 +0100 Subject: [PATCH 65/75] bugfixes in vm --- uncloud/hack/vm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 8e20e2e..24eb108 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -54,7 +54,7 @@ class VM(object): self.vm['owner']= "nico" self.vm['memory'] = self.config.arguments['memory'] self.vm['cores'] = self.config.arguments['cores'] - self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.2-x86_64.iso") + self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.3-x86_64.iso") self.create_template() # mimics api call = this will already be in etcd #self.vm['os_image'] = self.db.get("os_image") @@ -84,7 +84,7 @@ class VM(object): "-smp", "{}".format(self.vm['cores']), "-uuid", "{}".format(self.vm['uuid']), "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.mac), + "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) ] From 5d05e91335925def4ec4342874bef352db205cbd Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 24 Jan 2020 17:12:50 +0100 Subject: [PATCH 66/75] added hackerprefix argument, changed the commandline structure of vm to work better with sudo --- uncloud/hack/main.py | 1 + uncloud/hack/vm.py | 40 +++++++++++++++++++++++++--------------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index fc54da1..b6d8fad 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -25,6 +25,7 @@ arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root! arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') +arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") log = logging.getLogger(__name__) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index bb35348..4caa2fe 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -23,10 +23,14 @@ import subprocess import uuid import os +import logging from uncloud.hack.db import DB from uncloud.hack.mac import MAC + +log = logging.getLogger(__name__) + class VM(object): def __init__(self, config): self.config = config @@ -36,8 +40,9 @@ class VM(object): self.db = DB(self.config, prefix="/vm") #TODO: Select generic - self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm + #self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm #self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing + self.hackprefix=self.config.arguments['hackprefix'] self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config self.accel="kvm" #TODO: should be config @@ -69,30 +74,36 @@ class VM(object): #TODO: Enable sudo -- FIXME! if self.config.arguments['use_sudo']: - self.sudo = "sudo" + self.sudo = "sudo " else: self.sudo = "" self.mac=MAC(self.config) self.mac.create() + self.vm['mac'] = self.mac self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) # FIXME: TODO: turn this into a string and THEN # .split() it later -- easier for using .format() #self.vm['commandline'] = [ "{}".format(self.sudo), - self.vm['commandline'] = [ "{}".format(self.sudo), - "{}".format(self.qemu), - "-name", "uncloud-{}".format(self.vm['uuid']), - "-machine", "pc,accel={}".format(self.accel), - "-m", "{}".format(self.vm['memory']), - "-smp", "{}".format(self.vm['cores']), - "-uuid", "{}".format(self.vm['uuid']), - "-drive", "file={},media=cdrom".format(self.vm['os_image']), - "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), - "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.mac) - ] + self.vm['commandline'] = "{sudo}{qemu} -name uncloud-{uuid} -machine pc,accel={accel} -m {memory} -smp {cores} -uuid {uuid} -drive file={os_image},media=cdrom -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname} -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" +# self.vm['commandline'] = [ "{}".format(self.sudo), +# "{}".format(self.qemu), +# "-name", "uncloud-{}".format(self.vm['uuid']), +# "-machine", "pc,accel={}".format(self.accel), +# "-m", "{}".format(self.vm['memory']), +# "-smp", "{}".format(self.vm['cores']), +# "-uuid", "{}".format(self.vm['uuid']), +# "-drive", "file={},media=cdrom".format(self.vm['os_image']), +# "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), +# "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.vm['mac']) +# ] + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.vm, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) def create(self): if not self.no_db: @@ -100,6 +111,5 @@ class VM(object): self.vm, as_json=True) - print(" ".join(self.vm['commandline'])) - subprocess.run(self.vm['commandline']) #TODO: run in background + self._execute_cmd(self.vm['commandline'], sudo=self.sudo, qemu=self.qemu, accel=self.accel, ifup=self.ifup, ifdown=self.ifdown) #TODO: Add interface ifname to bridge brXX (via net.py: public function add iface to bridge) From cbcaf636506e138542e1580098d29057b9558b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 26 Jan 2020 12:04:37 +0100 Subject: [PATCH 67/75] Update VM images documentation (upstream images, uncloud-init) --- docs/source/{admin-guide => admin-guide.rst} | 39 ++---------- docs/source/index.rst | 5 +- docs/source/vm-images.rst | 66 ++++++++++++++++++++ 3 files changed, 74 insertions(+), 36 deletions(-) rename docs/source/{admin-guide => admin-guide.rst} (72%) create mode 100644 docs/source/vm-images.rst diff --git a/docs/source/admin-guide b/docs/source/admin-guide.rst similarity index 72% rename from docs/source/admin-guide rename to docs/source/admin-guide.rst index ec6597d..b62808d 100644 --- a/docs/source/admin-guide +++ b/docs/source/admin-guide.rst @@ -56,40 +56,13 @@ To start host we created earlier, execute the following command ucloud host ungleich.ch -Create OS Image ---------------- +File & image scanners +-------------------------- -Create ucloud-init ready OS image (Optional) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This step is optional if you just want to test ucloud. However, sooner or later -you want to create OS images with ucloud-init to properly -contexualize VMs. - -1. Start a VM with OS image on which you want to install ucloud-init -2. Execute the following command on the started VM - - .. code-block:: sh - - apk add git - git clone https://code.ungleich.ch/ucloud/ucloud-init.git - cd ucloud-init - sh ./install.sh -3. Congratulations. Your image is now ucloud-init ready. - - -Upload Sample OS Image -~~~~~~~~~~~~~~~~~~~~~~ -Execute the following to get the sample OS image file. - -.. code-block:: sh - - mkdir /var/www/admin - (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download) - -Run File Scanner and Image Scanner -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make -images from tracked files. So, we need to track the file by running File Scanner +Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our +uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by +ucloud. We can only make images from tracked files. So, we need to track the +file by running File Scanner .. code-block:: sh diff --git a/docs/source/index.rst b/docs/source/index.rst index b31cff3..fad1f88 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -11,14 +11,13 @@ Welcome to ucloud's documentation! :caption: Contents: introduction - user-guide setup-install + vm-images + user-guide admin-guide - user-guide/how-to-create-an-os-image-for-ucloud troubleshooting hacking - Indices and tables ================== diff --git a/docs/source/vm-images.rst b/docs/source/vm-images.rst new file mode 100644 index 0000000..4b2758a --- /dev/null +++ b/docs/source/vm-images.rst @@ -0,0 +1,66 @@ +VM images +================================== + +Overview +--------- + +ucloud tries to be least invasise towards VMs and only require +strictly necessary changes for running in a virtualised +environment. This includes configurations for: + +* Configuring the network +* Managing access via ssh keys +* Resizing the attached disk(s) + +Upstream images +--------------- + +The 'official' uncloud images are defined in the `uncloud/images +`_ repository. + +How to make you own Uncloud images +---------------------------------- + +.. note:: + It is fairly easy to create your own images for uncloud, as the common + operations (which are detailed below) can be automatically handled by the + `uncloud/uncloud-init `_ tool. + +Network configuration +~~~~~~~~~~~~~~~~~~~~~ +All VMs in ucloud are required to support IPv6. The primary network +configuration is always done using SLAAC. A VM thus needs only to be +configured to + +* accept router advertisements on all network interfaces +* use the router advertisements to configure the network interfaces +* accept the DNS entries from the router advertisements + + +Configuring SSH keys +~~~~~~~~~~~~~~~~~~~~ + +To be able to access the VM, ucloud support provisioning SSH keys. + +To accept ssh keys in your VM, request the URL +*http://metadata/ssh_keys*. Add the content to the appropriate user's +**authorized_keys** file. Below you find sample code to accomplish +this task: + +.. code-block:: sh + + tmp=$(mktemp) + curl -s http://metadata/ssk_keys > "$tmp" + touch ~/.ssh/authorized_keys # ensure it exists + cat ~/.ssh/authorized_keys >> "$tmp" + sort "$tmp" | uniq > ~/.ssh/authorized_keys + + +Disk resize +~~~~~~~~~~~ +In virtualised environments, the disk sizes might grow. The operating +system should detect disks that are bigger than the existing partition +table and resize accordingly. This task is os specific. + +ucloud does not support shrinking disks due to the complexity and +intra OS dependencies. From 2b71c1807de324be6a9bb707c561621548e3a48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 09:25:25 +0100 Subject: [PATCH 68/75] Wire uncloud-hack vm module to VMM --- uncloud/hack/main.py | 20 +++++-- uncloud/hack/vm.py | 130 +++++++++++++++++++++---------------------- 2 files changed, 80 insertions(+), 70 deletions(-) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index b6d8fad..351f582 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -10,7 +10,6 @@ from uncloud import UncloudException arg_parser = argparse.ArgumentParser('hack', add_help=False) #description="Commands that are unfinished - use at own risk") -arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--last-used-mac', action='store_true') arg_parser.add_argument('--get-new-mac', action='store_true') @@ -22,8 +21,15 @@ arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) arg_parser.add_argument('--run-dns-ra', action='store_true', help="Provide router advertisements and DNS resolution via dnsmasq") arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') + +arg_parser.add_argument('--create-vm', action='store_true') +arg_parser.add_argument('--destroy-vm', action='store_true') +arg_parser.add_argument('--get-vm-status', action='store_true') arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) +arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") +arg_parser.add_argument('--uuid', help="VM UUID") + arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") @@ -32,13 +38,19 @@ log = logging.getLogger(__name__) def main(arguments): - log.debug("args={}".format(arguments)) config = Config(arguments) if arguments['create_vm']: - print("Creating VM") vm = VM(config) - vm.commandline() + vm.create() + + if arguments['destroy_vm']: + vm = VM(config) + vm.stop() + + if arguments['get_vm_status']: + vm = VM(config) + vm.status() if arguments['last_used_mac']: m = MAC(config) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 4caa2fe..ce96fbf 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -27,89 +27,87 @@ import logging from uncloud.hack.db import DB from uncloud.hack.mac import MAC - +from uncloud.vmm import VMM log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) class VM(object): def __init__(self, config): self.config = config + #TODO: Enable etcd lookup self.no_db = self.config.arguments['no_db'] if not self.no_db: self.db = DB(self.config, prefix="/vm") - #TODO: Select generic - #self.hackprefix="/home/nico/vcs/uncloud/uncloud/hack/hackcloud" #TODO: Should be removed midterm - #self.hackprefix="/home/rouxdo/Work/ungleich/uncloud/uncloud/hack/hackcloud" #TODO: Dominique testing - self.hackprefix=self.config.arguments['hackprefix'] - self.qemu="/usr/bin/qemu-system-x86_64" #TODO: should be in config - self.accel="kvm" #TODO: should be config + # General CLI arguments. + self.hackprefix = self.config.arguments['hackprefix'] + self.uuid = self.config.arguments['uuid'] + self.memory = self.config.arguments['memory'] or '1024M' + self.cores = self.config.arguments['cores'] or 1 + if self.config.arguments['image']: + self.image = os.path.join(self.hackprefix, self.config.arguments['image']) + else: + self.image = None - self.vm = {} + # External components. + self.vmm = VMM(vmm_backend=self.hackprefix) + self.mac = MAC(self.config) - #TODO: Touch later! (when necessary) + # Harcoded & generated values. + self.owner = 'uncoud' + self.image_format='qcow2' + self.accel = 'kvm' + self.threads = 1 self.ifup = os.path.join(self.hackprefix, "ifup.sh") self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + self.ifname = "uc{}".format(self.mac.to_str_format()) - def commandline(self): - """This method is used to trigger / create a vm from the cli""" - #TODO: read arguments from cli - #TODO: create etcd json object - self.vm['owner']= "nico" - self.vm['memory'] = self.config.arguments['memory'] - self.vm['cores'] = self.config.arguments['cores'] - self.vm['os_image'] = os.path.join(self.hackprefix, "alpine-virt-3.11.3-x86_64.iso") - self.create_template() - # mimics api call = this will already be in etcd - #self.vm['os_image'] = self.db.get("os_image") - self.create() + def get_qemu_args(self): + command = ( + "-name {owner}-{name}" + " -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}" + " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" + ).format( + owner=self.owner, name=self.uuid, + accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname, + mac=self.mac + ) - def create_template(self): - self.uuid = uuid.uuid4() - #TODO: This all should be generic - self.vm['uuid'] = str(self.uuid) - #self.vni_hex = "{:x}".format(self.config.arguments['vni']) - self.bridgedev = "br{}".format("{:x}".format(self.config.arguments['vni'])) - - #TODO: Enable sudo -- FIXME! - if self.config.arguments['use_sudo']: - self.sudo = "sudo " - else: - self.sudo = "" - - - self.mac=MAC(self.config) - self.mac.create() - self.vm['mac'] = self.mac - self.vm['ifname'] = "uc{}".format(self.mac.to_str_format()) - - # FIXME: TODO: turn this into a string and THEN - # .split() it later -- easier for using .format() - #self.vm['commandline'] = [ "{}".format(self.sudo), - self.vm['commandline'] = "{sudo}{qemu} -name uncloud-{uuid} -machine pc,accel={accel} -m {memory} -smp {cores} -uuid {uuid} -drive file={os_image},media=cdrom -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname} -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" -# self.vm['commandline'] = [ "{}".format(self.sudo), -# "{}".format(self.qemu), -# "-name", "uncloud-{}".format(self.vm['uuid']), -# "-machine", "pc,accel={}".format(self.accel), -# "-m", "{}".format(self.vm['memory']), -# "-smp", "{}".format(self.vm['cores']), -# "-uuid", "{}".format(self.vm['uuid']), -# "-drive", "file={},media=cdrom".format(self.vm['os_image']), -# "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), -# "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.vm['mac']) -# ] - - def _execute_cmd(self, cmd_string, **kwargs): - cmd = cmd_string.format(**self.vm, **kwargs) - log.info("Executing: {}".format(cmd)) - subprocess.run(cmd.split()) + return command.split(" ") def create(self): - if not self.no_db: - self.db.set(str(self.vm['uuid']), - self.vm, - as_json=True) + # New VM: new UUID, new MAC. + self.uuid = str(uuid.uuid4()) + self.mac.create() + + qemu_args = self.get_qemu_args() + log.debug("QEMU args passed to VMM: {}".format(qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + self.vmm.stop(self.uuid) + + def status(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_status(self.uuid)) - self._execute_cmd(self.vm['commandline'], sudo=self.sudo, qemu=self.qemu, accel=self.accel, ifup=self.ifup, ifdown=self.ifdown) - #TODO: Add interface ifname to bridge brXX (via net.py: public function add iface to bridge) From 4c6a126d8b0a59a454ec69cbbc867786f0b7b04c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 11:02:18 +0100 Subject: [PATCH 69/75] Hack/VM: wire get_vnc and list_vms --- uncloud/hack/main.py | 10 ++++++++++ uncloud/hack/vm.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index 351f582..9607ec2 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -25,6 +25,8 @@ arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root! arg_parser.add_argument('--create-vm', action='store_true') arg_parser.add_argument('--destroy-vm', action='store_true') arg_parser.add_argument('--get-vm-status', action='store_true') +arg_parser.add_argument('--get-vm-vnc', action='store_true') +arg_parser.add_argument('--list-vms', action='store_true') arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int) arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int) arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") @@ -52,6 +54,14 @@ def main(arguments): vm = VM(config) vm.status() + if arguments['get_vm_vnc']: + vm = VM(config) + vm.vnc_addr() + + if arguments['list_vms']: + vm = VM(config) + vm.list() + if arguments['last_used_mac']: m = MAC(config) print(m.last_used_mac()) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index ce96fbf..e9b7719 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -111,3 +111,13 @@ class VM(object): print(self.vmm.get_status(self.uuid)) + def vnc_addr(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_vnc(self.uuid)) + + def list(self): + print(self.vmm.discover()) + From a759b8aa39ae96a08904119b15c5306048c34c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 12:24:26 +0100 Subject: [PATCH 70/75] VMM: make use of socket_dir --- uncloud/vmm/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/uncloud/vmm/__init__.py b/uncloud/vmm/__init__.py index 719bdbe..6db61eb 100644 --- a/uncloud/vmm/__init__.py +++ b/uncloud/vmm/__init__.py @@ -125,7 +125,7 @@ class VMM: os.makedirs(self.socket_dir, exist_ok=True) def is_running(self, uuid): - sock_path = os.path.join(self.vmm_backend, uuid) + sock_path = os.path.join(self.socket_dir, uuid) try: sock = socket.socket(socket.AF_UNIX) sock.connect(sock_path) @@ -163,7 +163,7 @@ class VMM: qmp_arg = ( "-qmp", "unix:{},server,nowait".format( - join_path(self.vmm_backend, uuid) + join_path(self.socket_dir, uuid) ), ) vnc_arg = ( @@ -212,7 +212,7 @@ class VMM: def execute_command(self, uuid, command, **kwargs): # execute_command -> sucess?, output try: - with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as ( + with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as ( sock_handle, file_handle, ): @@ -255,8 +255,8 @@ class VMM: def discover(self): vms = [ uuid - for uuid in os.listdir(self.vmm_backend) - if not isdir(join_path(self.vmm_backend, uuid)) + for uuid in os.listdir(self.socket_dir) + if not isdir(join_path(self.socket_dir, uuid)) ] return vms From 1758629ca1b861c0406e80591ff35073d7d6331f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 12:33:36 +0100 Subject: [PATCH 71/75] Add minimal doc to hack/vm.py --- uncloud/hack/vm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index e9b7719..f9cd31a 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -17,8 +17,21 @@ # # You should have received a copy of the GNU General Public License # along with uncloud. If not, see . + +# This module is directly called from the hack module, and can be used as follow: # +# Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix. +# `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2` # +# List running VMs (returns a list of UUIDs). +# `uncloud hack --hackprefix /tmp/hackcloud --list-vms +# +# Get VM status: +# `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid` +# +# Stop a VM: +# `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` + `` import subprocess import uuid From e2cd44826b9c307f816d170ed93b3a172edcf712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 13:45:20 +0100 Subject: [PATCH 72/75] Fix typo in hack/vm.py --- uncloud/hack/vm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index f9cd31a..ac403d8 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -31,7 +31,7 @@ # # Stop a VM: # `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` - `` +# `` import subprocess import uuid From 618fecb73fe3bc77f43d567219a88f3c5cb19b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 14:38:07 +0100 Subject: [PATCH 73/75] Initial implementation (no networking) of uncloud-oneshot --- scripts/uncloud | 1 + uncloud/oneshot/__init__.py | 3 ++ uncloud/oneshot/main.py | 65 ++++++++++++++++++++++++++++ uncloud/oneshot/virtualmachine.py | 70 +++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+) create mode 100644 uncloud/oneshot/__init__.py create mode 100644 uncloud/oneshot/main.py create mode 100644 uncloud/oneshot/virtualmachine.py diff --git a/scripts/uncloud b/scripts/uncloud index d565954..7d38e42 100755 --- a/scripts/uncloud +++ b/scripts/uncloud @@ -16,6 +16,7 @@ ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure', 'hack'] ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('oneshot') #ALL_COMPONENTS.append('cli') diff --git a/uncloud/oneshot/__init__.py b/uncloud/oneshot/__init__.py new file mode 100644 index 0000000..eea436a --- /dev/null +++ b/uncloud/oneshot/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/uncloud/oneshot/main.py b/uncloud/oneshot/main.py new file mode 100644 index 0000000..20f22e4 --- /dev/null +++ b/uncloud/oneshot/main.py @@ -0,0 +1,65 @@ +import argparse +import os + +from pathlib import Path +from uncloud.vmm import VMM + +from . import virtualmachine, logger + +arg_parser = argparse.ArgumentParser('oneshot', add_help=False) +arg_parser.add_argument('--workdir', default=Path.home()) +arg_parser.add_argument('--list-vms', action='store_true') +arg_parser.add_argument('--start-vm', action='store_true') +arg_parser.add_argument('--stop-vm', action='store_true') +arg_parser.add_argument('--name') +arg_parser.add_argument('--image') +arg_parser.add_argument('--uuid') +arg_parser.add_argument('--mac') +arg_parser.add_argument('--get_vm_status', action='store_true') +arg_parser.add_argument('--setup-network') + +def setup_network(): + print("Not implemented yet.") + exit(1) + +def require_with(arguments, required, mode): + if not arguments[required]: + print("--{} is required with the {} flag. Exiting.".format(required, mode)) + exit(1) + +def main(arguments): + # Initialize VMM + workdir = arguments['workdir'] + vmm = VMM(vmm_backend=workdir) + + # Initialize workdir directory. + # TODO: copy ifup, ifdown. + + # Build VM configuration. + vm_config = {} + for spec in ['uuid', 'memory', 'cores', 'threads', 'image', 'image_format', 'name']: + if arguments.get(spec): + vm_config[spec] = arguments[spec] + + # Execute requested VM action. + vm = virtualmachine.VM(vmm, vm_config) + if arguments['setup_network']: + setup_network() + elif arguments['start_vm']: + require_with(arguments, 'image', 'start_vm') + vm.start() + logger.info("Created VM {}".format(vm.get_uuid)) + elif arguments['get_vm_status']: + require_with(arguments, 'uuid', 'get_vm_status') + print("VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status())) + elif arguments['stop_vm']: + require_with(arguments, 'uuid', 'stop_vm') + vm.stop() + elif arguments['list_vms']: + discovered = vmm.discover() + print("Found {} VMs.".format(len(discovered))) + for uuid in vmm.discover(): + vmi = virtualmachine.VM(vmm, {'uuid': uuid}) + print("VM: {} {} {}".format(vmi.get_uuid, vmi.get_name, vmi.get_status)) + else: + print('No action requested. Exiting.') diff --git a/uncloud/oneshot/virtualmachine.py b/uncloud/oneshot/virtualmachine.py new file mode 100644 index 0000000..47365d5 --- /dev/null +++ b/uncloud/oneshot/virtualmachine.py @@ -0,0 +1,70 @@ +import uuid +import os + +from uncloud.oneshot import logger + +class VM(object): + def __init__(self, vmm, config): + self.config = config + self.vmm = vmm + + # Extract VM specs/metadata from configuration. + self.name = config.get('name') + self.memory = config.get('memory', 1024) + self.cores = config.get('cores', 1) + self.threads = config.get('threads', 1) + self.image_format = config.get('image_format', 'qcow2') + self.image = config.get('image') + self.uuid = config.get('uuid', uuid.uuid4()) + self.mac = config.get('mac', 'spuik') + + # Harcoded & generated values. + self.image_format='qcow2' + self.accel = 'kvm' + + def get_qemu_args(self): + command = ( + "-uuid {uuid} -name {name}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + ).format( + uuid=self.uuid, name=self.name, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + ) + + return command.split(" ") + + def start(self): + # Check that VM image is available. + if not os.path.isfile(self.image): + logger.error("Image {} does not exist. Aborting.".format(self.image)) + + # Generate config for and run QEMU. + qemu_args = self.get_qemu_args() + logger.warning("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + self.vmm.stop(self.uuid) + + def get_status(self): + return self.vmm.get_status(self.uuid) + + def get_vnc_addr(self): + return self.vmm.get_vnc(self.uuid) + + def get_uuid(self): + return self.uuid + + def get_name(self): + success, json = self.vmm.execute_command(uuid, 'query-name') + if success: + return json['return']['name'] + + return None From 3e69fb275fb152cc842582e3a173cdbea8e2e155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 Jan 2020 17:44:53 +0100 Subject: [PATCH 74/75] Oneshot: cleanup CLI, initial networking support --- uncloud/oneshot/main.py | 140 +++++++++++++++++++++--------- uncloud/oneshot/virtualmachine.py | 27 ++++-- 2 files changed, 119 insertions(+), 48 deletions(-) diff --git a/uncloud/oneshot/main.py b/uncloud/oneshot/main.py index 20f22e4..0e94a81 100644 --- a/uncloud/oneshot/main.py +++ b/uncloud/oneshot/main.py @@ -1,65 +1,123 @@ import argparse import os + from pathlib import Path from uncloud.vmm import VMM +from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap from . import virtualmachine, logger +### +# Argument parser loaded by scripts/uncloud. arg_parser = argparse.ArgumentParser('oneshot', add_help=False) -arg_parser.add_argument('--workdir', default=Path.home()) -arg_parser.add_argument('--list-vms', action='store_true') -arg_parser.add_argument('--start-vm', action='store_true') -arg_parser.add_argument('--stop-vm', action='store_true') -arg_parser.add_argument('--name') -arg_parser.add_argument('--image') -arg_parser.add_argument('--uuid') -arg_parser.add_argument('--mac') -arg_parser.add_argument('--get_vm_status', action='store_true') -arg_parser.add_argument('--setup-network') -def setup_network(): - print("Not implemented yet.") - exit(1) +# Actions. +arg_parser.add_argument('--list', action='store_true', + help='list UUID and name of running VMs') +arg_parser.add_argument('--start', nargs=3, + metavar=('IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'), + help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix') +arg_parser.add_argument('--stop', metavar='UUID', + help='stop a VM') +arg_parser.add_argument('--get-status', metavar='UUID', + help='return the status of the VM') +arg_parser.add_argument('--get-vnc', metavar='UUID', + help='return the path of the VNC socket of the VM') +arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK', + help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix') -def require_with(arguments, required, mode): - if not arguments[required]: - print("--{} is required with the {} flag. Exiting.".format(required, mode)) - exit(1) +# Arguments. +arg_parser.add_argument('--workdir', default=Path.home(), + help='Working directory, defaulting to $HOME') +arg_parser.add_argument('--mac', + help='MAC address of the VM to create (--start)') +arg_parser.add_argument('--memory', type=int, + help='Memory (MB) to allocate (--start)') +arg_parser.add_argument('--cores', type=int, + help='Number of cores to allocate (--start)') +arg_parser.add_argument('--threads', type=int, + help='Number of threads to allocate (--start)') +arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], + help='Format of OS image (--start)') +arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='tcg', + help='QEMU acceleration to use (--start)') +arg_parser.add_argument('--upstream-interface', default='eth0', + help='Name of upstream interface (--start)') + +### +# Helpers. + +# XXX: check if it is possible to use the type returned by ETCD queries. +class UncloudEntryWrapper: + def __init__(self, value): + self.value = value + + def value(self): + return self.value + +def status_line(vm): + return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status()) + +### +# Entrypoint. def main(arguments): - # Initialize VMM + # Initialize VMM. workdir = arguments['workdir'] vmm = VMM(vmm_backend=workdir) - # Initialize workdir directory. - # TODO: copy ifup, ifdown. + # Harcoded debug values. + net_id = 0 # Build VM configuration. vm_config = {} - for spec in ['uuid', 'memory', 'cores', 'threads', 'image', 'image_format', 'name']: - if arguments.get(spec): - vm_config[spec] = arguments[spec] + vm_options = [ + 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', + '--upstream_interface', 'upstream_interface', 'network' + ] + for option in vm_options: + if arguments.get(option): + vm_config[option] = arguments[option] + + vm_config['net_id'] = net_id # Execute requested VM action. - vm = virtualmachine.VM(vmm, vm_config) - if arguments['setup_network']: - setup_network() - elif arguments['start_vm']: - require_with(arguments, 'image', 'start_vm') + if arguments['reconfigure_radvd']: + # TODO: check that RADVD is available. + prefix = arguments['reconfigure_radvd'] + network = UncloudEntryWrapper({ + 'id': net_id, + 'ipv6': prefix + }) + + # Make use of uncloud.host.virtualmachine for network configuration. + update_radvd_conf([network]) + elif arguments['start']: + # Extract from --start positional arguments. Quite fragile. + vm_config['image'] = arguments['start'][0] + vm_config['network'] = arguments['start'][1] + vm_config['upstream_interface'] = arguments['start'][2] + + vm_config['tap_interface'] = "uc{}".format(len(vmm.discover())) + vm = virtualmachine.VM(vmm, vm_config) vm.start() - logger.info("Created VM {}".format(vm.get_uuid)) - elif arguments['get_vm_status']: - require_with(arguments, 'uuid', 'get_vm_status') - print("VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status())) - elif arguments['stop_vm']: - require_with(arguments, 'uuid', 'stop_vm') + elif arguments['stop']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']}) + vm = virtualmachine.VM(vmm, vm_config) vm.stop() - elif arguments['list_vms']: - discovered = vmm.discover() - print("Found {} VMs.".format(len(discovered))) - for uuid in vmm.discover(): - vmi = virtualmachine.VM(vmm, {'uuid': uuid}) - print("VM: {} {} {}".format(vmi.get_uuid, vmi.get_name, vmi.get_status)) + elif arguments['get_status']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']}) + print(status_line(vm)) + elif arguments['get_vnc']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']}) + print(vm.get_vnc_addr()) + elif arguments['list']: + vms = vmm.discover() + print("Found {} VMs.".format(len(vms))) + for uuid in vms: + vm = virtualmachine.VM(vmm, {'uuid': uuid}) + print(status_line(vm)) else: - print('No action requested. Exiting.') + print('Please specify an action: --start, --stop, --list,\ +--get-status, --get-vnc, --reconfigure-radvd') diff --git a/uncloud/oneshot/virtualmachine.py b/uncloud/oneshot/virtualmachine.py index 47365d5..1388d49 100644 --- a/uncloud/oneshot/virtualmachine.py +++ b/uncloud/oneshot/virtualmachine.py @@ -1,6 +1,7 @@ import uuid import os +from uncloud.host.virtualmachine import create_vxlan_br_tap from uncloud.oneshot import logger class VM(object): @@ -9,29 +10,36 @@ class VM(object): self.vmm = vmm # Extract VM specs/metadata from configuration. - self.name = config.get('name') + self.name = config.get('name', 'no-name') self.memory = config.get('memory', 1024) self.cores = config.get('cores', 1) self.threads = config.get('threads', 1) self.image_format = config.get('image_format', 'qcow2') self.image = config.get('image') - self.uuid = config.get('uuid', uuid.uuid4()) - self.mac = config.get('mac', 'spuik') + self.uuid = config.get('uuid', str(uuid.uuid4())) + self.mac = config.get('mac') + + self.net_id = config.get('net_id', 0) + self.upstream_interface = config.get('upstream_interface', 'eth0') + self.tap_interface = config.get('tap_interface', 'uc0') + self.network = config.get('network') # Harcoded & generated values. - self.image_format='qcow2' self.accel = 'kvm' def get_qemu_args(self): command = ( - "-uuid {uuid} -name {name}" + "-uuid {uuid} -name {name} -machine pc,accel={accel}" " -drive file={image},format={image_format},if=virtio" " -device virtio-rng-pci" " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" ).format( - uuid=self.uuid, name=self.name, + uuid=self.uuid, name=self.name, accel=self.accel, image=self.image, image_format=self.image_format, memory=self.memory, cores=self.cores, threads=self.threads, + net_id=self.net_id, tap=self.tap_interface, mac=self.mac ) return command.split(" ") @@ -41,9 +49,14 @@ class VM(object): if not os.path.isfile(self.image): logger.error("Image {} does not exist. Aborting.".format(self.image)) + # Create Bridge, VXLAN and tap interface for VM. + create_vxlan_br_tap( + self.net_id, self.upstream_interface, self.tap_interface, self.network + ) + # Generate config for and run QEMU. qemu_args = self.get_qemu_args() - logger.warning("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) + logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) self.vmm.start( uuid=self.uuid, migration=False, From e14ac947c14bbad0cfeef66b5397a2feb2a3b043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 30 Jan 2020 08:44:41 +0100 Subject: [PATCH 75/75] Fix --accel parameter for oneshot --- uncloud/oneshot/main.py | 4 ++-- uncloud/oneshot/virtualmachine.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/uncloud/oneshot/main.py b/uncloud/oneshot/main.py index 0e94a81..5b9b61c 100644 --- a/uncloud/oneshot/main.py +++ b/uncloud/oneshot/main.py @@ -40,7 +40,7 @@ arg_parser.add_argument('--threads', type=int, help='Number of threads to allocate (--start)') arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], help='Format of OS image (--start)') -arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='tcg', +arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm', help='QEMU acceleration to use (--start)') arg_parser.add_argument('--upstream-interface', default='eth0', help='Name of upstream interface (--start)') @@ -74,7 +74,7 @@ def main(arguments): vm_config = {} vm_options = [ 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', - '--upstream_interface', 'upstream_interface', 'network' + '--upstream_interface', 'upstream_interface', 'network', 'accel' ] for option in vm_options: if arguments.get(option): diff --git a/uncloud/oneshot/virtualmachine.py b/uncloud/oneshot/virtualmachine.py index 1388d49..c8c2909 100644 --- a/uncloud/oneshot/virtualmachine.py +++ b/uncloud/oneshot/virtualmachine.py @@ -18,15 +18,13 @@ class VM(object): self.image = config.get('image') self.uuid = config.get('uuid', str(uuid.uuid4())) self.mac = config.get('mac') + self.accel = config.get('accel', 'kvm') self.net_id = config.get('net_id', 0) self.upstream_interface = config.get('upstream_interface', 'eth0') self.tap_interface = config.get('tap_interface', 'uc0') self.network = config.get('network') - # Harcoded & generated values. - self.accel = 'kvm' - def get_qemu_args(self): command = ( "-uuid {uuid} -name {name} -machine pc,accel={accel}"