From 47b0ba77195ab6e95fa1510e332b3c39b4c999d9 Mon Sep 17 00:00:00 2001 From: meow Date: Thu, 1 Aug 2019 15:04:40 +0500 Subject: [PATCH] Cleaning --- .gitignore | 2 + common_fields.py | 64 ++++++++ create_image_store.py | 23 +++ helper.py | 12 -- main.py | 343 +++++++++++------------------------------- schemas.py | 153 +++++++++++++++++++ 6 files changed, 326 insertions(+), 271 deletions(-) create mode 100644 common_fields.py create mode 100755 create_image_store.py create mode 100644 schemas.py diff --git a/.gitignore b/.gitignore index 5929a22..209604a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ __pycache__/ venv/ settings.json +ucloud_common +etcd3_wrapper diff --git a/common_fields.py b/common_fields.py new file mode 100644 index 0000000..4450bb6 --- /dev/null +++ b/common_fields.py @@ -0,0 +1,64 @@ +from etcd3_wrapper import Etcd3Wrapper +from specs_parser import SpecsParser + +specs_parser = SpecsParser(exceptional_devices=["cpu"]) + + +class Field(object): + def __init__(self, _name, _type, _value=None): + self.name = _name + self.value = _value + self.type = _type + self.__errors = [] + + def validation(self): + return True + + def is_valid(self): + if self.value == KeyError: + self.add_error(f"'{self.name}' field is a required field") + else: + if not isinstance(self.value, self.type): + self.add_error(f"Incorrect Type for '{self.name}' field") + else: + self.validation() + + if self.__errors: + return False + return True + + def get_errors(self): + return self.__errors + + def add_error(self, error): + self.__errors.append(error) + + +class VmUUIDField(Field): + def __init__(self, data): + self.uuid = data.get("uuid", KeyError) + + super().__init__("uuid", str, self.uuid) + + self.validation = self.vm_uuid_validation + + def vm_uuid_validation(self): + client = Etcd3Wrapper() + + r = client.get(f"/v1/vm/{self.uuid}") + if not r: + self.add_error(f"VM with uuid {self.uuid} does not exists") + + +class SpecsField(Field): + def __init__(self, data): + self.specs = data.get("specs", KeyError) + + super().__init__("specs", dict, self.specs) + + self.validation = self.specs_validation + + def specs_validation(self): + if not specs_parser.transform_specs(self.specs): + self.add_error("Invalid unit - " + f"Please use following units {specs_parser.get_allowed_units()}") diff --git a/create_image_store.py b/create_image_store.py new file mode 100755 index 0000000..7d5fc56 --- /dev/null +++ b/create_image_store.py @@ -0,0 +1,23 @@ +import json +from uuid import uuid4 +from etcd3_wrapper import Etcd3Wrapper + +client = Etcd3Wrapper() + +data = { + "is_public": True, + "type": "ceph", + "name": "images", + "description": "first ever public image-store", + "attributes": { + "list": [], + "key": [], + "pool": "images", + } +} + +client.put( + f"/v1/image_store/{uuid4().hex}", + json.dumps(data), +) + diff --git a/helper.py b/helper.py index 9d3a254..819ffe6 100644 --- a/helper.py +++ b/helper.py @@ -26,15 +26,3 @@ def check_otp(name, realm, token): data=data, ) return response.status_code - - -def add_otp_args(parser): - parser.add_argument("name", required=True) - parser.add_argument("realm", required=True) - parser.add_argument("token", required=True) - return parser - - -def add_vmid_args(parser): - parser.add_argument("vmid", required=True) - return parser diff --git a/main.py b/main.py index 3755cdb..8bad54e 100644 --- a/main.py +++ b/main.py @@ -4,176 +4,74 @@ import json -from helper import check_otp, add_otp_args, add_vmid_args -from flask import Flask -from flask_restful import Resource, Api, reqparse +from flask import Flask, request +from flask_restful import Resource, Api from uuid import uuid4 from etcd3_wrapper import Etcd3Wrapper -from specs_parser import SpecsParser + +from ucloud_common.vm import VmPool, VMStatus +from schemas import (CreateVMSchema, VMStatusSchema, + CreateImageSchema, VmActionSchema, + OTPSchema, CreateHostSchema) + app = Flask(__name__) api = Api(app) client = Etcd3Wrapper() - -# CreateVM argparser -createvm_argparser = reqparse.RequestParser() -createvm_argparser.add_argument("specs", type=dict, required=True) -createvm_argparser.add_argument("image_uuid", type=str, required=True) -add_otp_args(createvm_argparser) - -# CreateImage argparser -createimage_argparser = reqparse.RequestParser() -createimage_argparser.add_argument("uuid", type=str, required=True) -createimage_argparser.add_argument("name", type=str, required=True) -createimage_argparser.add_argument("image_store", type=str, required=True) - -# DeleteVM argparser -deletevm_argparser = reqparse.RequestParser() -add_vmid_args(add_otp_args(deletevm_argparser)) - -# VMStatus argparser -vmstatus_argparser = reqparse.RequestParser() -add_vmid_args(vmstatus_argparser) - -# StartVM argparser -startvm_argparser = reqparse.RequestParser() -add_vmid_args(add_otp_args(startvm_argparser)) - -# UserVM argparser -uservm_argparser = reqparse.RequestParser() -add_otp_args(uservm_argparser) - -# Specs parser -specs_parser = SpecsParser(exceptional_devices=["cpu"]) - -# AddHost argparser -addhost_argparser = reqparse.RequestParser() -addhost_argparser.add_argument("hostname", type=str, required=True) -addhost_argparser.add_argument("specs", type=dict, required=True) -add_otp_args(addhost_argparser) - - -def is_image_valid(image_uuid): - images = client.get_prefix("/v1/image/") - return image_uuid in [i.key.split("/")[-1] for i in images] +vm_pool = VmPool(client, "/v1/vm") class CreateVM(Resource): def post(self): - createvm_args = createvm_argparser.parse_args() - name, realm, token, specs, image_uuid = ( - createvm_args.name, - createvm_args.realm, - createvm_args.token, - createvm_args.specs, - createvm_args.image_uuid, - ) - - if check_otp(name, realm, token) == 200: - # User is good - if is_image_valid(image_uuid): - if not specs_parser.transform_specs(specs): - return ( - { - "message": f"""Invalid unit - Please use following units {specs_parser.get_allowed_units()}""" - }, - 400, - ) - - print(specs) - vm_key = f"/v1/vm/{uuid4().hex}" - vm_entry = { - "owner": name, - "specs": specs, - "hostname": "", - "status": "REQUESTED_NEW", - "image_uuid": image_uuid, - } - - client.put(vm_key, vm_entry, value_in_json=True) - - return {"message": "VM Creation Queued"}, 200 - else: - return {"message": "Image uuid not valid"} + data = request.json + validator = CreateVMSchema(data) + if validator.is_valid(): + vm_key = f"/v1/vm/{uuid4().hex}" + vm_entry = { + "owner": data["name"], + "specs": data["specs"], + "hostname": "", + "status": "REQUESTED_NEW", + "image_uuid": data["image_uuid"], + } + client.put(vm_key, vm_entry, value_in_json=True) + return {"message": "VM Creation Queued"}, 200 else: - return {"message": "Invalid Credentials"}, 400 - - -class DeleteVM(Resource): - def post(self): - deletevm_args = deletevm_argparser.parse_args() - name, realm, token, vmid = ( - deletevm_args.name, - deletevm_args.realm, - deletevm_args.token, - deletevm_args.vmid, - ) - - if check_otp(name, realm, token) == 200: - # User is good - - vmentry_etcd = client.get(f"/v1/vm/{vmid}").value - if vmentry_etcd: - vmentry_etcd = json.loads(vmentry_etcd) - vmentry_etcd["status"] = "REQUESTED_DELETE" - - client.put(f"/v1/vm/{vmid}", vmentry_etcd, - value_in_json=True) - - return {"message": "VM Deletion Queued"}, 200 - else: - return {"message": "Invalid VM ID"} - else: - return {"message": "Invalid Credentials"}, 400 + return validator.get_errors(), 400 class VmStatus(Resource): def get(self): - args = vmstatus_argparser.parse_args() - r = client.get(f"/v1/vm/{args.vmid}").value - print(r) - if r: - r = dict(json.loads(r.decode("utf-8"))) - return r - return {"Message": "Not Found"} + data = request.json + validator = VMStatusSchema(data) + if validator.is_valid(): + vm = vm_pool.get(f"/v1/vm/{data['uuid']}") + return str(vm) + else: + return validator.get_errors(), 400 class CreateImage(Resource): def post(self): - image_stores = list(client.get_prefix("/v1/image_store/")) - args = createimage_argparser.parse_args() - image_file_uuid = args.uuid - image_store_name = args.image_store + data = request.json + validator = CreateImageSchema(data) + if validator.is_valid(): + file_entry = client.get(f"/v1/file/{data['uuid']}") + file_entry_value = json.loads(file_entry.value) - file_entry = client.get(f"/v1/file/{image_file_uuid}") - if file_entry is None: - return ( - {"Message": f"Image File with uuid '{image_file_uuid}' Not Found"}, - 400, - ) + image_entry_json = { + "status": "TO_BE_CREATED", + "owner": file_entry_value["owner"], + "filename": file_entry_value["filename"], + "name": data["name"], + "store_name": data["image_store"], + "visibility": "public", + } + client.put(f"/v1/image/{data['uuid']}", json.dumps(image_entry_json)) - file_entry_value = json.loads(file_entry.value) - - image_store = list( - filter( - lambda s: json.loads(s.value)["name"] == image_store_name, image_stores - ) - ) - if not image_store: - return {"Message": f"Store '{image_store_name}' does not exists"}, 400 - - image_store = image_store[0] - image_entry_json = { - "status": "TO_BE_CREATED", - "owner": file_entry_value["owner"], - "filename": file_entry_value["filename"], - "name": args.name, - "store_name": image_store_name, - "visibility": "public", - } - client.put(f"/v1/image/{image_file_uuid}", json.dumps(image_entry_json)) - - return {"Message": "Image successfully created"} + return {"Message": "Image successfully created"} + else: + return validator.get_errors(), 400 class ListPublicImages(Resource): @@ -186,93 +84,41 @@ class ListPublicImages(Resource): return r, 200 -class StartVM(Resource): +class VMAction(Resource): def post(self): - args = startvm_argparser.parse_args() - name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid + data = request.json + validator = VmActionSchema(data) + action = data["action"] - if check_otp(name, realm, token) == 200: - vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm: - if vm.value["owner"] != name: - return {"message": "Invalid User"} - vm.value["status"] = "REQUESTED_START" - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Start Queued"} - else: - return {"message": "No such VM found"} + if validator.is_valid(): + vm = vm_pool.get(f"/v1/vm/{data['uuid']}") + if action == "start": + vm.status = VMStatus.requested_start + elif action == "suspend": + vm.status = VMStatus.requested_suspend + elif action == "resume": + vm.status = VMStatus.requested_resume + elif action == "shutdown": + vm.status = VMStatus.requested_shutdown + elif action == "delete": + vm.status = VMStatus.requested_delete + + client.put(vm.key, json.dumps(vm.value)) + return {"message": f"VM Start Queued"} else: - return {"message": "Invalid Credentials"}, 400 - - -class SuspendVM(Resource): - def post(self): - args = startvm_argparser.parse_args() - name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid - - if check_otp(name, realm, token) == 200: - vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm: - if vm.value["owner"] != name: - return {"message": "Invalid User"} - vm.value["status"] = "REQUESTED_SUSPEND" - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Suspension Queued"} - else: - return {"message": "No such VM found"} - else: - return {"message": "Invalid Credentials"}, 400 - - -class ResumeVM(Resource): - def post(self): - args = startvm_argparser.parse_args() - name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid - - if check_otp(name, realm, token) == 200: - vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm: - if vm.value["owner"] != name: - return {"message": "Invalid User"} - vm.value["status"] = "REQUESTED_RESUME" - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Resume Queued"} - else: - return {"message": "No such VM found"} - else: - return {"message": "Invalid Credentials"}, 400 - - -class ShutdownVM(Resource): - def post(self): - args = startvm_argparser.parse_args() - name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid - - if check_otp(name, realm, token) == 200: - vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm: - if vm.value["owner"] != name: - return {"message": "Invalid User"} - - vm.value["status"] = "REQUESTED_SHUTDOWN" - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Shutdown Queued"} - else: - return {"message": "No such VM found"} - else: - return {"message": "Invalid Credentials"}, 400 + return validator.get_errors(), 400 class ListUserVM(Resource): def get(self): - args = uservm_argparser.parse_args() - name, realm, token = args.name, args.realm, args.token + data = request.json + validator = OTPSchema(data) - if check_otp(name, realm, token) == 200: + if validator.is_valid(): vms = client.get_prefix(f"/v1/vm/", value_in_json=True) if vms: return_vms = [] - user_vms = list(filter(lambda v: v.value["owner"] == name, vms)) + user_vms = list(filter(lambda v: v.value["owner"] == data["name"], vms)) for vm in user_vms: return_vms.append( { @@ -285,19 +131,19 @@ class ListUserVM(Resource): else: return {"message": "No VM found"}, 404 else: - return {"message": "Invalid Credentials"}, 400 + return validator.get_errors(), 400 class ListUserFiles(Resource): def get(self): - args = uservm_argparser.parse_args() - name, realm, token = args.name, args.realm, args.token + data = request.json + validator = OTPSchema(data) - if check_otp(name, realm, token) == 200: + if validator.is_valid(): files = client.get_prefix(f"/v1/file/", value_in_json=True) if files: return_files = [] - user_files = list(filter(lambda f: f.value["owner"] == name, files)) + user_files = list(filter(lambda f: f.value["owner"] == data["name"], files)) for file in user_files: return_files.append( { @@ -309,35 +155,18 @@ class ListUserFiles(Resource): else: return {"message": "No File found"}, 404 else: - return {"message": "Invalid Credentials"}, 400 + return validator.get_errors(), 400 class CreateHost(Resource): def post(self): - args = addhost_argparser.parse_args() - name, realm, token, specs, hostname = ( - args.name, - args.realm, - args.token, - args.specs, - args.hostname, - ) - - if realm == "ungleich-admin" and check_otp(name, realm, token) == 200: - # User is good - if not specs_parser.transform_specs(specs): - return ( - { - "message": f"""Invalid unit - Please use following units {specs_parser.get_allowed_units()}""" - }, - 400, - ) - - print(specs) + data = request.json + validator = CreateHostSchema(data) + if validator.is_valid(): host_key = f"/v1/host/{uuid4().hex}" host_entry = { - "specs": specs, - "hostname": hostname, + "specs": data["specs"], + "hostname": data["hostname"], "status": "DEAD", "last_heartbeat": "", } @@ -345,17 +174,13 @@ class CreateHost(Resource): return {"message": "Host Created"}, 200 else: - return {"message": "Invalid Credentials/Insufficient Permission"}, 400 + return validator.get_errors(), 400 api.add_resource(CreateVM, "/vm/create") -api.add_resource(DeleteVM, "/vm/delete") api.add_resource(VmStatus, "/vm/status") -api.add_resource(StartVM, "/vm/start") -api.add_resource(SuspendVM, "/vm/suspend") -api.add_resource(ResumeVM, "/vm/resume") -api.add_resource(ShutdownVM, "/vm/shutdown") +api.add_resource(VMAction, "/vm/action") api.add_resource(CreateImage, "/image/create") api.add_resource(ListPublicImages, "/image/list-public") diff --git a/schemas.py b/schemas.py new file mode 100644 index 0000000..a11d900 --- /dev/null +++ b/schemas.py @@ -0,0 +1,153 @@ +import json + +from common_fields import Field, VmUUIDField, SpecsField +from helper import check_otp +from etcd3_wrapper import Etcd3Wrapper + +client = Etcd3Wrapper() + + +class BaseSchema(object): + def __init__(self, data, fields=None): + _ = data + + self.__errors = [] + if fields is None: + self.fields = [] + else: + self.fields = fields + + def validation(self): + # custom validation is optional + return True + + def is_valid(self): + for field in self.fields: + field.is_valid() + self.add_field_errors(field) + + for parent in self.__class__.__bases__: + try: + parent.validation(self) + except AttributeError: + pass + if not self.__errors: + self.validation() + + if self.__errors: + return False + return True + + def get_errors(self): + return {"Message": self.__errors} + + def add_field_errors(self, field: Field): + self.__errors += field.get_errors() + + def add_error(self, error): + self.__errors.append(error) + + +class OTPSchema(BaseSchema): + def __init__(self, data: dict, fields=None): + self.name = Field("name", str, data.get("name", KeyError)) + self.realm = Field("realm", str, data.get("realm", KeyError)) + self.token = Field("token", str, data.get("token", KeyError)) + + _fields = [self.name, self.realm, self.token] + if fields: + _fields += fields + super().__init__(data=data, fields=_fields) + + def validation(self): + rc = check_otp(self.name.value, self.realm.value, self.token.value) + if rc != 200: + self.add_error("Wrong Credentials") + + +class CreateVMSchema(OTPSchema): + def __init__(self, data): + self.specs = SpecsField(data) + + self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError)) + self.image_uuid.validation = self.image_uuid_validation + + fields = [self.specs, self.image_uuid] + super().__init__(data=data, fields=fields) + + def image_uuid_validation(self): + images = client.get_prefix("/v1/image/") + + if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]: + self.add_error("Image UUID not valid") + + +class VMStatusSchema(BaseSchema): + def __init__(self, data): + self.uuid = VmUUIDField(data) + + fields = [self.uuid] + + super().__init__(data, fields) + + +class CreateImageSchema(BaseSchema): + def __init__(self, data): + # Fields + self.uuid: Field = Field("uuid", str, data.get("uuid", KeyError)) + self.name = Field("name", str, data.get("name", KeyError)) + self.image_store = Field("image_store", str, data.get("image_store", KeyError)) + # Validations + self.uuid.validation = self.file_uuid_validation + + # All Fields + fields = [self.uuid, self.name, self.image_store] + super().__init__(data, fields) + + def file_uuid_validation(self): + file_entry = client.get(f"/v1/file/{self.uuid.value}") + if file_entry is None: + self.add_error(f"Image File with uuid '{self.uuid.value}' Not Found") + + def image_store_name_validation(self): + image_stores = list(client.get_prefix("/v1/image_store/")) + + image_store = next(filter(lambda s: json.loads(s.value)["name"] == self.image_store.value, + image_stores)) + if not image_store: + self.add_error(f"Store '{self.image_store.value}' does not exists") + + +class VmActionSchema(OTPSchema): + def __init__(self, data): + self.uuid = VmUUIDField(data) + self.action = Field("action", str, data.get("action", KeyError)) + + self.action.validation = self.action_validation + + fields = [self.uuid, self.action] + super().__init__(data=data, fields=fields) + + def action_validation(self): + allowed_actions = ["start", "shutdown", "suspend", "resume", "delete"] + if self.action.value not in allowed_actions: + self.add_error(f"Invalid Action. Allowed Actions are {allowed_actions}") + + def validation(self): + vm = client.get(f"/v1/vm/{self.uuid.value}", value_in_json=True) + if vm.value["owner"] != self.name: + self.add_error("Invalid User") + + +class CreateHostSchema(OTPSchema): + def __init__(self, data): + self.specs = SpecsField(data) + self.hostname = Field("hostname", str, data.get("hostname", KeyError)) + + fields = [self.specs, self.hostname] + + super().__init__(data=data, fields=fields) + + def validation(self): + if self.realm.value != "ungleich-admin": + self.add_error("Invalid Credentials/Insufficient Permission")