From f18dbbe62b971d34c994dce4354e0a1eba92653d Mon Sep 17 00:00:00 2001 From: Ahmed Bilal Khalid Date: Sun, 22 Sep 2019 23:51:33 +0500 Subject: [PATCH] Implemented support for VM operations through VM name, Allow user of realm ungleich-admin to do operation on someone's else VM (for providing support etc), Fix few hidden issues, Refactoring/Formating + Little bit Documenting --- Pipfile.lock | 14 ++-- common_fields.py | 6 +- config.py | 9 +++ create_image_store.py | 11 +-- helper.py | 17 ++++ main.py | 132 +++++++++++++++++------------- schemas.py | 183 ++++++++++++++++++++++++++++++------------ 7 files changed, 250 insertions(+), 122 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index a93d91a..dce94fd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -54,7 +54,7 @@ "etcd3-wrapper": { "editable": true, "git": "https://code.ungleich.ch/ungleich-public/etcd3_wrapper.git", - "ref": "b1893fc286e9ed59876a526d81094edd796d6815" + "ref": "76fb0bdf797199e9ea161dad1d004eea9b4520f8" }, "flask": { "hashes": [ @@ -240,17 +240,17 @@ }, "urllib3": { "hashes": [ - "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", - "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" + "sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb", + "sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47" ], - "version": "==1.25.3" + "version": "==1.25.5" }, "werkzeug": { "hashes": [ - "sha256:00d32beac38fcd48d329566f80d39f10ec2ed994efbecfb8dd4b320062d05902", - "sha256:0a24d43be6a7dce81bae05292356176d6c46d63e42a0dd3f9504b210a9cfaa43" + "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7", + "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4" ], - "version": "==0.15.6" + "version": "==0.16.0" } }, "develop": { diff --git a/common_fields.py b/common_fields.py index 80eec87..6b5cd54 100755 --- a/common_fields.py +++ b/common_fields.py @@ -62,5 +62,7 @@ class SpecsField(Field): def specs_validation(self): if not specs_parser.transform_specs(self.specs): - self.add_error("Invalid unit - " - "Please use following units {}".format(specs_parser.get_allowed_units())) + self.add_error( + "Invalid unit - " + "Please use following units {}".format(specs_parser.get_allowed_units()) + ) diff --git a/config.py b/config.py index ca75c72..4ae1b66 100644 --- a/config.py +++ b/config.py @@ -3,6 +3,11 @@ import logging from etcd3_wrapper import Etcd3Wrapper from decouple import config + +from ucloud_common.vm import VmPool +from ucloud_common.host import HostPool +from ucloud_common.request import RequestPool + logging.basicConfig( level=logging.DEBUG, filename="log.txt", @@ -21,3 +26,7 @@ IMAGE_PREFIX = config("IMAGE_PREFIX") IMAGE_STORE_PREFIX = config("IMAGE_STORE_PREFIX") etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) + +VM_POOL = VmPool(etcd_client, VM_PREFIX) +HOST_POOL = HostPool(etcd_client, HOST_PREFIX) +REQUEST_POOL = RequestPool(etcd_client, REQUEST_PREFIX) diff --git a/create_image_store.py b/create_image_store.py index 5a1f382..796cc43 100755 --- a/create_image_store.py +++ b/create_image_store.py @@ -11,14 +11,7 @@ data = { "type": "ceph", "name": "images", "description": "first ever public image-store", - "attributes": { - "list": [], - "key": [], - "pool": "images", - } + "attributes": {"list": [], "key": [], "pool": "images"}, } -client.put( - os.path.join(IMAGE_STORE_PREFIX, uuid4().hex), - json.dumps(data), -) +client.put(os.path.join(IMAGE_STORE_PREFIX, uuid4().hex), json.dumps(data)) diff --git a/helper.py b/helper.py index 819ffe6..76f4b59 100755 --- a/helper.py +++ b/helper.py @@ -3,6 +3,7 @@ import requests from decouple import config from pyotp import TOTP +from config import VM_POOL def check_otp(name, realm, token): @@ -26,3 +27,19 @@ def check_otp(name, realm, token): data=data, ) return response.status_code + + +def resolve_vm_name(name, owner): + """ + Input: name of vm, owner of vm + Output: uuid of vm if found otherwise None + """ + result = next( + filter( + lambda vm: vm.value["owner"] == owner and vm.value["name"] == name, + VM_POOL.vms, + ), + None, + ) + if result: + return result.key.split("/")[-1] diff --git a/main.py b/main.py index 9f48c05..77da1af 100755 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ -# TODO -# 1. Allow user of realm ungleich-admin to perform any action on -# any user vm. - +# TODO: realm should be part of user's name. Because, there can be multiple +# name but with different realms. For Example, nico exists in both +# ungleich-admin and ungleich-twitter realm. So, to differentiate between +# them, we need to make realm part of user's name import json import subprocess import os @@ -10,29 +10,34 @@ from uuid import uuid4 from flask import Flask, request from flask_restful import Resource, Api -from ucloud_common.vm import VmPool, VMStatus -from ucloud_common.host import HostPool -from ucloud_common.request import (RequestEntry, - RequestPool, - RequestType) +from ucloud_common.vm import VMStatus +from ucloud_common.request import RequestEntry, RequestType from config import etcd_client as client -from config import (WITHOUT_CEPH, VM_PREFIX, - HOST_PREFIX, REQUEST_PREFIX, - FILE_PREFIX, IMAGE_PREFIX, - logging) -from schemas import (CreateVMSchema, VMStatusSchema, - CreateImageSchema, VmActionSchema, - OTPSchema, CreateHostSchema, - VmMigrationSchema) +from config import ( + WITHOUT_CEPH, + VM_PREFIX, + HOST_PREFIX, + FILE_PREFIX, + IMAGE_PREFIX, + logging, + REQUEST_POOL, + VM_POOL, + HOST_POOL, +) +from schemas import ( + CreateVMSchema, + VMStatusSchema, + CreateImageSchema, + VmActionSchema, + OTPSchema, + CreateHostSchema, + VmMigrationSchema, +) app = Flask(__name__) api = Api(app) -VM_POOL = VmPool(client, VM_PREFIX) -HOST_POOL = HostPool(client, HOST_PREFIX) -REQUEST_POOL = RequestPool(client, REQUEST_PREFIX) - class CreateVM(Resource): @staticmethod @@ -43,6 +48,7 @@ class CreateVM(Resource): vm_uuid = uuid4().hex vm_key = os.path.join(VM_PREFIX, vm_uuid) vm_entry = { + "name": data["vm_name"], "owner": data["name"], "specs": data["specs"], "hostname": "", @@ -50,13 +56,12 @@ class CreateVM(Resource): "image_uuid": data["image_uuid"], "log": [], "storage_attachment": [], - "vnc_socket": "" + "vnc_socket": "", } client.put(vm_key, vm_entry, value_in_json=True) # Create ScheduleVM Request - r = RequestEntry.from_scratch(type=RequestType.ScheduleVM, - uuid=vm_uuid) + r = RequestEntry.from_scratch(type=RequestType.ScheduleVM, uuid=vm_uuid) REQUEST_POOL.put(r) return {"message": "VM Creation Queued"}, 200 @@ -69,7 +74,7 @@ class VmStatus(Resource): data = request.json validator = VMStatusSchema(data) if validator.is_valid(): - vm = VM_POOL.get(os.path.join(VM_PREFIX, data['uuid'])) + vm = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"])) return str(vm) else: return validator.get_errors(), 400 @@ -81,7 +86,7 @@ class CreateImage(Resource): data = request.json validator = CreateImageSchema(data) if validator.is_valid(): - file_entry = client.get(os.path.join(FILE_PREFIX, data['uuid'])) + file_entry = client.get(os.path.join(FILE_PREFIX, data["uuid"])) file_entry_value = json.loads(file_entry.value) image_entry_json = { @@ -92,7 +97,9 @@ class CreateImage(Resource): "store_name": data["image_store"], "visibility": "public", } - client.put(os.path.join(IMAGE_PREFIX, data['uuid']), json.dumps(image_entry_json)) + client.put( + os.path.join(IMAGE_PREFIX, data["uuid"]), json.dumps(image_entry_json) + ) return {"message": "Image successfully created"} return validator.get_errors(), 400 @@ -115,7 +122,7 @@ class VMAction(Resource): validator = VmActionSchema(data) if validator.is_valid(): - vm_entry = VM_POOL.get(os.path.join(VM_PREFIX, data['uuid'])) + vm_entry = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"])) action = data["action"] if action == "start": @@ -125,10 +132,13 @@ class VMAction(Resource): if action == "delete" and vm_entry.hostname == "": try: - path_without_protocol = vm_entry.path[vm_entry.path.find(":")+1:] + path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1 :] if WITHOUT_CEPH: - command_to_delete = ["rm", os.path.join("/var/vm", vm_entry.uuid)] + command_to_delete = [ + "rm", + os.path.join("/var/vm", vm_entry.uuid), + ] else: command_to_delete = ["rbd", "rm", path_without_protocol] @@ -144,9 +154,11 @@ class VMAction(Resource): client.client.delete(vm_entry.key) return {"message": "VM successfully deleted"} - r = RequestEntry.from_scratch(type="{}VM".format(action.title()), - uuid=data['uuid'], - hostname=vm_entry.hostname) + r = RequestEntry.from_scratch( + type="{}VM".format(action.title()), + uuid=data["uuid"], + hostname=vm_entry.hostname, + ) REQUEST_POOL.put(r) return {"message": "VM {} Queued".format(action.title())}, 200 else: @@ -160,12 +172,14 @@ class VMMigration(Resource): validator = VmMigrationSchema(data) if validator.is_valid(): - vm = VM_POOL.get(data['uuid']) + vm = VM_POOL.get(data["uuid"]) - r = RequestEntry.from_scratch(type=RequestType.ScheduleVM, - uuid=vm.uuid, - destination=os.path.join(HOST_PREFIX, data["destination"]), - migration=True) + r = RequestEntry.from_scratch( + type=RequestType.ScheduleVM, + uuid=vm.uuid, + destination=os.path.join(HOST_PREFIX, data["destination"]), + migration=True, + ) REQUEST_POOL.put(r) return {"message": "VM Migration Initialization Queued"}, 200 else: @@ -180,22 +194,25 @@ class ListUserVM(Resource): if validator.is_valid(): vms = client.get_prefix(VM_PREFIX, value_in_json=True) - if vms: - return_vms = [] - user_vms = list(filter(lambda v: v.value["owner"] == data["name"], vms)) - for vm in user_vms: - return_vms.append( - { - "vm_uuid": vm.key.split("/")[-1], - "specs": vm.value["specs"], - "status": vm.value["status"], - "hostname": vm.value["hostname"], - "vnc_socket": None or vm.value["vnc_socket"] - } - ) + return_vms = [] + user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) + for vm in user_vms: + return_vms.append( + { + "name": vm.value["name"], + "vm_uuid": vm.key.split("/")[-1], + "specs": vm.value["specs"], + "status": vm.value["status"], + "hostname": vm.value["hostname"], + "vnc_socket": None + if vm.value.get("vnc_socket", None) == None + else vm.value["vnc_socket"], + } + ) + if return_vms: return {"message": return_vms}, 200 - else: - return {"message": "No VM found"}, 404 + return {"message": "No VM found"}, 404 + else: return validator.get_errors(), 400 @@ -246,7 +263,14 @@ class ListHost(Resource): @staticmethod def get(): hosts = HOST_POOL.hosts - r = {host.key: {"status": host.status, "specs": host.specs, "hostname": host.hostname} for host in hosts} + r = { + host.key: { + "status": host.status, + "specs": host.specs, + "hostname": host.hostname, + } + for host in hosts + } return r, 200 diff --git a/schemas.py b/schemas.py index 1e9438c..446b3cb 100755 --- a/schemas.py +++ b/schemas.py @@ -1,3 +1,19 @@ +""" +This module contain classes thats validates and intercept/modify +data coming from ucloud-cli (user) + +It was primarily developed as an alternative to argument parser +of Flask_Restful which is going to be deprecated. I also tried +marshmallow for that purpose but it was an overkill (because it +do validation + serialization + deserialization) and little +inflexible for our purpose. +""" + +# TODO: Fix error message when user's mentioned VM (referred by name) +# does not exists. +# +# Currently, it says uuid is a required field. + import json import os @@ -5,10 +21,9 @@ from ucloud_common.host import HostPool, HostStatus from ucloud_common.vm import VmPool, VMStatus from common_fields import Field, VmUUIDField, SpecsField -from helper import check_otp +from helper import check_otp, resolve_vm_name from config import etcd_client as client -from config import (HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX, - FILE_PREFIX, IMAGE_STORE_PREFIX) +from config import HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX, FILE_PREFIX, IMAGE_STORE_PREFIX HOST_POOL = HostPool(client, HOST_PREFIX) VM_POOL = VmPool(client, VM_PREFIX) @@ -70,30 +85,7 @@ class OTPSchema(BaseSchema): self.add_error("Wrong Credentials") -class CreateVMSchema(OTPSchema): - def __init__(self, data): - self.specs = SpecsField(data) - - self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError)) - self.image_uuid.validation = self.image_uuid_validation - - fields = [self.specs, self.image_uuid] - super().__init__(data=data, fields=fields) - - def image_uuid_validation(self): - images = client.get_prefix(IMAGE_PREFIX) - - if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]: - self.add_error("Image UUID not valid") - - -class VMStatusSchema(BaseSchema): - def __init__(self, data): - self.uuid = VmUUIDField(data) - - fields = [self.uuid] - - super().__init__(data, fields) +########################## Image Operations ############################################### class CreateImageSchema(BaseSchema): @@ -114,46 +106,149 @@ class CreateImageSchema(BaseSchema): def file_uuid_validation(self): file_entry = client.get(os.path.join(FILE_PREFIX, self.uuid.value)) if file_entry is None: - self.add_error("Image File with uuid '{}' Not Found".format(self.uuid.value)) + self.add_error( + "Image File with uuid '{}' Not Found".format(self.uuid.value) + ) def image_store_name_validation(self): image_stores = list(client.get_prefix(IMAGE_STORE_PREFIX)) - image_store = next(filter(lambda s: json.loads(s.value)["name"] == self.image_store.value, - image_stores), None) + image_store = next( + filter( + lambda s: json.loads(s.value)["name"] == self.image_store.value, + image_stores, + ), + None, + ) if not image_store: self.add_error("Store '{}' does not exists".format(self.image_store.value)) +########################## Host Operations ############################################### + + +class CreateHostSchema(OTPSchema): + def __init__(self, data): + self.specs = SpecsField(data) + self.hostname = Field("hostname", str, data.get("hostname", KeyError)) + + fields = [self.specs, self.hostname] + + super().__init__(data=data, fields=fields) + + def validation(self): + if self.realm.value != "ungleich-admin": + self.add_error("Invalid Credentials/Insufficient Permission") + + +########################## VM Operations ############################################### + + +class CreateVMSchema(OTPSchema): + def __init__(self, data): + self.specs = SpecsField(data) + + self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError)) + self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError)) + + self.image_uuid.validation = self.image_uuid_validation + self.vm_name.validation = self.vm_name_validation + + fields = [self.vm_name, self.specs, self.image_uuid] + super().__init__(data=data, fields=fields) + + def image_uuid_validation(self): + images = client.get_prefix(IMAGE_PREFIX) + + if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]: + self.add_error("Image UUID not valid") + + def vm_name_validation(self): + if resolve_vm_name(name=self.vm_name.value, owner=self.name.value): + self.add_error( + 'VM with same name "{}" already exists'.format(self.vm_name.value) + ) + + +class VMStatusSchema(OTPSchema): + def __init__(self, data): + data["uuid"] = ( + resolve_vm_name( + name=data.get("vm_name", None), + owner=(data.get("in_support_of", None) or data.get("name", None)), + ) + or KeyError + ) + self.uuid = VmUUIDField(data) + + fields = [self.uuid] + + super().__init__(data, fields) + + def validation(self): + vm = VM_POOL.get(self.uuid.value) + if not ( + vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + ): + self.add_error("Invalid User") + + class VmActionSchema(OTPSchema): def __init__(self, data): + data["uuid"] = ( + resolve_vm_name( + name=data.get("vm_name", None), + owner=(data.get("in_support_of", None) or data.get("name", None)), + ) + or KeyError + ) self.uuid = VmUUIDField(data) self.action = Field("action", str, data.get("action", KeyError)) self.action.validation = self.action_validation _fields = [self.uuid, self.action] + super().__init__(data=data, fields=_fields) def action_validation(self): allowed_actions = ["start", "stop", "delete"] if self.action.value not in allowed_actions: - self.add_error("Invalid Action. Allowed Actions are {}".format(allowed_actions)) + self.add_error( + "Invalid Action. Allowed Actions are {}".format(allowed_actions) + ) def validation(self): vm = VM_POOL.get(self.uuid.value) - if vm.value["owner"] != self.name.value: + if not ( + vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + ): self.add_error("Invalid User") - if self.action.value == "start" and vm.status == VMStatus.running and vm.hostname != "": + if ( + self.action.value == "start" + and vm.status == VMStatus.running + and vm.hostname != "" + ): self.add_error("VM Already Running") - if self.action.value == "stop" and vm.status == VMStatus.stopped: - self.add_error("VM Already Stopped") + if self.action.value == "stop": + if vm.status == VMStatus.stopped: + self.add_error("VM Already Stopped") + elif vm.status != VMStatus.running: + self.add_error("Cannot stop non-running VM") class VmMigrationSchema(OTPSchema): def __init__(self, data): + data["uuid"] = ( + resolve_vm_name( + name=data.get("vm_name", None), + owner=(data.get("in_support_of", None) or data.get("name", None)), + ) + or KeyError + ) + self.uuid = VmUUIDField(data) self.destination = Field("destination", str, data.get("destination", KeyError)) @@ -172,7 +267,9 @@ class VmMigrationSchema(OTPSchema): def validation(self): vm = VM_POOL.get(self.uuid.value) - if vm.owner != self.name.value: + if not ( + vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + ): self.add_error("Invalid User") if vm.status != VMStatus.running: @@ -180,17 +277,3 @@ class VmMigrationSchema(OTPSchema): if vm.hostname == os.path.join(HOST_PREFIX, self.destination.value): self.add_error("Destination host couldn't be same as Source Host") - - -class CreateHostSchema(OTPSchema): - def __init__(self, data): - self.specs = SpecsField(data) - self.hostname = Field("hostname", str, data.get("hostname", KeyError)) - - fields = [self.specs, self.hostname] - - super().__init__(data=data, fields=fields) - - def validation(self): - if self.realm.value != "ungleich-admin": - self.add_error("Invalid Credentials/Insufficient Permission")