From a4bedb01f60346b9abde13552bfdaef5fbbc4638 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:00:42 +0100 Subject: [PATCH 1/3] [api] begin to move to configparser --- ucloud/api/create_image_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ucloud/api/create_image_store.py b/ucloud/api/create_image_store.py index 17fa63c..259b9c8 100755 --- a/ucloud/api/create_image_store.py +++ b/ucloud/api/create_image_store.py @@ -3,7 +3,7 @@ import os from uuid import uuid4 -from ucloud.config import etcd_client, env_vars +from ucloud.config import etcd_client, config data = { "is_public": True, @@ -13,4 +13,4 @@ data = { "attributes": {"list": [], "key": [], "pool": "images"}, } -etcd_client.put(os.path.join(env_vars.get('IMAGE_STORE_PREFIX'), uuid4().hex), json.dumps(data)) +etcd_client.put(os.path.join(config['api']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data)) From cdbfb96e712f51970729bdd999aeb531ba072336 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:09:52 +0100 Subject: [PATCH 2/3] [api] config updates and add default values --- conf/ucloud.conf | 6 ++++++ ucloud/api/common_fields.py | 5 ++--- ucloud/api/helper.py | 26 +++++++++++++------------- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 7f83d80..0ebb705 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -1,8 +1,14 @@ # This section contains default values for all other sections [DEFAULT] +AUTH_NAME = "replace me" +AUTH_SEED = "replace me" +AUTH_REALM = "replace me" + NETWORK_PREFIX = moo +OTP_VERIFY_ENDPOINT = verify/ + [api] NETWORK_PREFIX = foo diff --git a/ucloud/api/common_fields.py b/ucloud/api/common_fields.py index e9903ac..1ceb1b0 100755 --- a/ucloud/api/common_fields.py +++ b/ucloud/api/common_fields.py @@ -1,7 +1,6 @@ import os -from ucloud.config import etcd_client, env_vars - +from ucloud.config import etcd_client, config class Optional: pass @@ -48,6 +47,6 @@ class VmUUIDField(Field): self.validation = self.vm_uuid_validation def vm_uuid_validation(self): - r = etcd_client.get(os.path.join(env_vars.get('VM_PREFIX'), self.uuid)) + r = etcd_client.get(os.path.join(config['api']['VM_PREFIX'], self.uuid)) if not r: self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/ucloud/api/helper.py b/ucloud/api/helper.py index 63d2f90..6735f05 100755 --- a/ucloud/api/helper.py +++ b/ucloud/api/helper.py @@ -7,15 +7,15 @@ import requests from pyotp import TOTP -from ucloud.config import vm_pool, env_vars +from ucloud.config import vm_pool, config def check_otp(name, realm, token): try: data = { - "auth_name": env_vars.get("AUTH_NAME"), - "auth_token": TOTP(env_vars.get("AUTH_SEED")).now(), - "auth_realm": env_vars.get("AUTH_REALM"), + "auth_name": config['api']["AUTH_NAME"], + "auth_token": TOTP(config['api']["AUTH_SEED"]).now(), + "auth_realm": config['api']["AUTH_REALM"], "name": name, "realm": realm, "token": token, @@ -25,8 +25,8 @@ def check_otp(name, realm, token): response = requests.post( "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( - OTP_SERVER=env_vars.get("OTP_SERVER", ""), - OTP_VERIFY_ENDPOINT=env_vars.get("OTP_VERIFY_ENDPOINT", "verify/"), + OTP_SERVER=config['api']["OTP_SERVER"], + OTP_VERIFY_ENDPOINT=config['api']["OTP_VERIFY_ENDPOINT"] ), json=data, ) @@ -35,7 +35,7 @@ def check_otp(name, realm, token): def resolve_vm_name(name, owner): """Return UUID of Virtual Machine of name == name and owner == owner - + Input: name of vm, owner of vm. Output: uuid of vm if found otherwise None """ @@ -54,7 +54,7 @@ def resolve_vm_name(name, owner): def resolve_image_name(name, etcd_client): """Return image uuid given its name and its store - + * If the provided name is not in correct format i.e {store_name}:{image_name} return ValueError * If no such image found then return KeyError @@ -70,9 +70,9 @@ def resolve_image_name(name, etcd_client): """ Examples, where it would work and where it would raise exception "images:alpine" --> ["images", "alpine"] - + "images" --> ["images"] it would raise Exception as non enough value to unpack - + "images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception as too many values to unpack """ @@ -80,7 +80,7 @@ def resolve_image_name(name, etcd_client): except Exception: raise ValueError("Image name not in correct format i.e {store_name}:{image_name}") - images = etcd_client.get_prefix(env_vars.get('IMAGE_PREFIX'), value_in_json=True) + images = etcd_client.get_prefix(config['api']['IMAGE_PREFIX'], value_in_json=True) # Try to find image with name == image_name and store_name == store_name try: @@ -119,14 +119,14 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt=' def get_ip_addr(mac_address, device): """Return IP address of a device provided its mac address / link local address and the device with which it is connected. - + For Example, if we call get_ip_addr(mac_address="52:54:00:12:34:56", device="br0") the following two scenarios can happen 1. It would return None if we can't be able to find device whose mac_address is equal to the arg:mac_address or the mentioned arg:device does not exists or the ip address we found is local. 2. It would return ip_address of device whose mac_address is equal to arg:mac_address - and is connected/neighbor of arg:device + and is connected/neighbor of arg:device """ try: output = sp.check_output(['ip', '-6', 'neigh', 'show', 'dev', device], stderr=sp.PIPE) From 76f63633ca84fff61147e0537430f998cce09c68 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 8 Dec 2019 13:29:24 +0100 Subject: [PATCH 3/3] [api] done -> configparser --- conf/ucloud.conf | 21 +++++++++++++--- ucloud/api/main.py | 56 +++++++++++++++++++++---------------------- ucloud/api/schemas.py | 12 +++++----- 3 files changed, 52 insertions(+), 37 deletions(-) diff --git a/conf/ucloud.conf b/conf/ucloud.conf index 0ebb705..60d5042 100644 --- a/conf/ucloud.conf +++ b/conf/ucloud.conf @@ -12,7 +12,22 @@ OTP_VERIFY_ENDPOINT = verify/ [api] NETWORK_PREFIX = foo -[woo] -NETWORK_PREFIX = foo +[network] +PREFIX_LENGTH = 64 +PREFIX = 2001:db8::/48 -[noval] \ No newline at end of file +[netbox] +NETBOX_URL = https://replace-me.example.com +NETBOX_TOKEN = replace me + +[etcd] + +FILE_PREFIX = file/ +HOST_PREFIx = host/ +IMAGE_PREFIX = image/ +IMAGE_STORE_PREFIX = imagestore/ + +NETWORK_PREFIX = network/ +REQUEST_PREFIX = request/ +USER_PREFIX = user/ +VM_PREFIX = vm/ diff --git a/ucloud/api/main.py b/ucloud/api/main.py index 1475fb0..d3ddd5d 100644 --- a/ucloud/api/main.py +++ b/ucloud/api/main.py @@ -10,7 +10,7 @@ from flask_restful import Resource, Api from ucloud.common import counters from ucloud.common.vm import VMStatus from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, env_vars, image_storage_handler) +from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, config, image_storage_handler) from . import schemas from .helper import generate_mac, mac2ipv6 from . import logger @@ -28,7 +28,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(env_vars.get("VM_PREFIX"), vm_uuid) + vm_key = join_path(config['api']["VM_PREFIX"), vm_uuid) specs = { "cpu": validator.specs["cpu"], "ram": validator.specs["ram"], @@ -56,7 +56,7 @@ class CreateVM(Resource): # Create ScheduleVM Request r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=env_vars.get("REQUEST_PREFIX") + request_prefix=config['api']["REQUEST_PREFIX") ) request_pool.put(r) @@ -71,7 +71,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = vm_pool.get( - join_path(env_vars.get("VM_PREFIX"), data["uuid"]) + join_path(config['api']["VM_PREFIX"), data["uuid"]) ) vm_value = vm.value.copy() vm_value["ip"] = [] @@ -79,7 +79,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = etcd_client.get( join_path( - env_vars.get("NETWORK_PREFIX"), + config['api']["NETWORK_PREFIX"), data["name"], network_name, ), @@ -100,7 +100,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = etcd_client.get( - join_path(env_vars.get("FILE_PREFIX"), data["uuid"]) + join_path(config['api']["FILE_PREFIX"), data["uuid"]) ) file_entry_value = json.loads(file_entry.value) @@ -113,7 +113,7 @@ class CreateImage(Resource): "visibility": "public", } etcd_client.put( - join_path(env_vars.get("IMAGE_PREFIX"), data["uuid"]), + join_path(config['etcd']["IMAGE_PREFIX"), data["uuid"]), json.dumps(image_entry_json), ) @@ -125,7 +125,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = etcd_client.get_prefix( - env_vars.get("IMAGE_PREFIX"), value_in_json=True + config['etcd']["IMAGE_PREFIX"), value_in_json=True ) r = { "images": [] @@ -148,7 +148,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = vm_pool.get( - join_path(env_vars.get("VM_PREFIX"), data["uuid"]) + join_path(config['etcd']["VM_PREFIX"), data["uuid"]) ) action = data["action"] @@ -172,7 +172,7 @@ class VMAction(Resource): type="{}VM".format(action.title()), uuid=data["uuid"], hostname=vm_entry.hostname, - request_prefix=env_vars.get("REQUEST_PREFIX") + request_prefix=config['etcd']["REQUEST_PREFIX"] ) request_pool.put(r) return {"message": "VM {} Queued".format(action.title())}, 200 @@ -193,10 +193,10 @@ class VMMigration(Resource): type=RequestType.ScheduleVM, uuid=vm.uuid, destination=join_path( - env_vars.get("HOST_PREFIX"), validator.destination.value + config['etcd']["HOST_PREFIX"], validator.destination.value ), migration=True, - request_prefix=env_vars.get("REQUEST_PREFIX") + request_prefix=config['etcd']["REQUEST_PREFIX"] ) request_pool.put(r) return {"message": "VM Migration Initialization Queued"}, 200 @@ -212,7 +212,7 @@ class ListUserVM(Resource): if validator.is_valid(): vms = etcd_client.get_prefix( - env_vars.get("VM_PREFIX"), value_in_json=True + config['etcd']["VM_PREFIX"], value_in_json=True ) return_vms = [] user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) @@ -246,7 +246,7 @@ class ListUserFiles(Resource): if validator.is_valid(): files = etcd_client.get_prefix( - env_vars.get("FILE_PREFIX"), value_in_json=True + config['etcd']["FILE_PREFIX"], value_in_json=True ) return_files = [] user_files = list( @@ -270,7 +270,7 @@ class CreateHost(Resource): data = request.json validator = schemas.CreateHostSchema(data) if validator.is_valid(): - host_key = join_path(env_vars.get("HOST_PREFIX"), uuid4().hex) + host_key = join_path(config['etcd']["HOST_PREFIX"], uuid4().hex) host_entry = { "specs": data["specs"], "hostname": data["hostname"], @@ -309,7 +309,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - env_vars.get('USER_PREFIX'), + config['etcd']['USER_PREFIX'], data["realm"], data["name"], "key", @@ -326,7 +326,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - env_vars.get('USER_PREFIX'), + config['etcd']['USER_PREFIX'), data["realm"], data["name"], "key", @@ -355,7 +355,7 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - env_vars.get("USER_PREFIX"), + config['etcd']["USER_PREFIX"], data["realm"], data["name"], "key", @@ -385,7 +385,7 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - env_vars.get("USER_PREFIX"), + config['etcd']["USER_PREFIX"], data["realm"], data["name"], "key", @@ -421,17 +421,17 @@ class CreateNetwork(Resource): } if validator.user.value: nb = pynetbox.api( - url=env_vars.get("NETBOX_URL"), - token=env_vars.get("NETBOX_TOKEN"), + url=config['netbox']["NETBOX_URL"], + token=config['netbox']["NETBOX_TOKEN"], ) nb_prefix = nb.ipam.prefixes.get( - prefix=env_vars.get("PREFIX") + prefix=config['network']["PREFIX"] ) prefix = nb_prefix.available_prefixes.create( data={ - "prefix_length": env_vars.get( - "PREFIX_LENGTH", cast=int + "prefix_length": config['network'][ + "PREFIX_LENGTH"] ), "description": '{}\'s network "{}"'.format( data["name"], data["network_name"] @@ -444,7 +444,7 @@ class CreateNetwork(Resource): network_entry["ipv6"] = "fd00::/64" network_key = join_path( - env_vars.get("NETWORK_PREFIX"), + config['network']["NETWORK_PREFIX"], data["name"], data["network_name"], ) @@ -462,7 +462,7 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - env_vars.get("NETWORK_PREFIX"), data["name"] + config['network']["NETWORK_PREFIX"], data["name"] ) networks = etcd_client.get_prefix(prefix, value_in_json=True) user_networks = [] @@ -498,7 +498,7 @@ api.add_resource(CreateNetwork, "/network/create") def main(): - image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'), value_in_json=True)) + image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True)) if len(image_stores) == 0: data = { "is_public": True, @@ -508,7 +508,7 @@ def main(): "attributes": {"list": [], "key": [], "pool": "images"}, } - etcd_client.put(join_path(env_vars.get('IMAGE_STORE_PREFIX'), uuid4().hex), json.dumps(data)) + etcd_client.put(join_path(config['etcd']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data)) app.run(host="::", debug=True) diff --git a/ucloud/api/schemas.py b/ucloud/api/schemas.py index c4f60ca..23db184 100755 --- a/ucloud/api/schemas.py +++ b/ucloud/api/schemas.py @@ -21,7 +21,7 @@ import bitmath from ucloud.common.host import HostStatus from ucloud.common.vm import VMStatus -from ucloud.config import etcd_client, env_vars, vm_pool, host_pool +from ucloud.config import etcd_client, config, vm_pool, host_pool from . import helper from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -102,14 +102,14 @@ class CreateImageSchema(BaseSchema): super().__init__(data, fields) def file_uuid_validation(self): - file_entry = etcd_client.get(os.path.join(env_vars.get('FILE_PREFIX'), self.uuid.value)) + file_entry = etcd_client.get(os.path.join(config['etcd']['FILE_PREFIX'], self.uuid.value)) if file_entry is None: self.add_error( "Image File with uuid '{}' Not Found".format(self.uuid.value) ) def image_store_name_validation(self): - image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'))) + image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'])) image_store = next( filter( @@ -235,7 +235,7 @@ class CreateVMSchema(OTPSchema): if _network: for net in _network: - network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'), + network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'], self.name.value, net), value_in_json=True) if not network: @@ -400,7 +400,7 @@ class VmMigrationSchema(OTPSchema): if vm.status != VMStatus.running: self.add_error("Can't migrate non-running VM") - if vm.hostname == os.path.join(env_vars.get('HOST_PREFIX'), self.destination.value): + if vm.hostname == os.path.join(config['etcd']['HOST_PREFIX'], self.destination.value): self.add_error("Destination host couldn't be same as Source Host") @@ -442,7 +442,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'), + network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'], self.name.value, self.network_name.value), value_in_json=True)